/ Check-in [031feebc]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Bring in all the latest enhancements from trunk.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | apple-osx
Files: files | file ages | folders
SHA3-256: 031feebc94b0cfe1b0ae96ede1684fe54cd79c1d340b8b35cca1ef885f82dde4
User & Date: drh 2017-04-24 16:14:58
Context
2017-05-11
18:49
Merge recent enhancements from trunk. check-in: b55c0f14 user: drh tags: apple-osx
2017-04-24
16:14
Bring in all the latest enhancements from trunk. check-in: 031feebc user: drh tags: apple-osx
14:16
When building an ephemeral table of integers, do not assume that the table does not already contain N if the insert cursor points to N-1. Fix for ticket [61fe9745]. check-in: 1c1b0862 user: dan tags: trunk
2017-04-08
18:24
Merge all recent changes from trunk. check-in: c23ef5e2 user: drh tags: apple-osx
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to Makefile.msc.

    17     17   USE_AMALGAMATION = 1
    18     18   !ENDIF
    19     19   # <</mark>>
    20     20   
    21     21   # Set this non-0 to enable full warnings (-W4, etc) when compiling.
    22     22   #
    23     23   !IFNDEF USE_FULLWARN
    24         -USE_FULLWARN = 0
           24  +USE_FULLWARN = 1
    25     25   !ENDIF
    26     26   
    27     27   # Set this non-0 to enable treating warnings as errors (-WX, etc) when
    28     28   # compiling.
    29     29   #
    30     30   !IFNDEF USE_FATAL_WARN
    31     31   USE_FATAL_WARN = 0

Changes to ext/fts3/fts3.c.

   368    368       if( (c & 0x80)==0 ) break;
   369    369     }
   370    370     *v = b;
   371    371     return (int)(p - pStart);
   372    372   }
   373    373   
   374    374   /*
   375         -** Similar to sqlite3Fts3GetVarint(), except that the output is truncated to a
   376         -** 32-bit integer before it is returned.
          375  +** Similar to sqlite3Fts3GetVarint(), except that the output is truncated to 
          376  +** a non-negative 32-bit integer before it is returned.
   377    377   */
   378    378   int sqlite3Fts3GetVarint32(const char *p, int *pi){
   379    379     u32 a;
   380    380   
   381    381   #ifndef fts3GetVarint32
   382    382     GETVARINT_INIT(a, p, 0,  0x00,     0x80, *pi, 1);
   383    383   #else
................................................................................
   385    385     assert( a & 0x80 );
   386    386   #endif
   387    387   
   388    388     GETVARINT_STEP(a, p, 7,  0x7F,     0x4000, *pi, 2);
   389    389     GETVARINT_STEP(a, p, 14, 0x3FFF,   0x200000, *pi, 3);
   390    390     GETVARINT_STEP(a, p, 21, 0x1FFFFF, 0x10000000, *pi, 4);
   391    391     a = (a & 0x0FFFFFFF );
   392         -  *pi = (int)(a | ((u32)(*p & 0x0F) << 28));
          392  +  *pi = (int)(a | ((u32)(*p & 0x07) << 28));
          393  +  assert( 0==(a & 0x80000000) );
          394  +  assert( *pi>=0 );
   393    395     return 5;
   394    396   }
   395    397   
   396    398   /*
   397    399   ** Return the number of bytes required to encode v as a varint
   398    400   */
   399    401   int sqlite3Fts3VarintLen(sqlite3_uint64 v){
................................................................................
  1215   1217         }else{
  1216   1218           for(iOpt=0; iOpt<SizeofArray(aFts4Opt); iOpt++){
  1217   1219             struct Fts4Option *pOp = &aFts4Opt[iOpt];
  1218   1220             if( nKey==pOp->nOpt && !sqlite3_strnicmp(z, pOp->zOpt, pOp->nOpt) ){
  1219   1221               break;
  1220   1222             }
  1221   1223           }
  1222         -        if( iOpt==SizeofArray(aFts4Opt) ){
  1223         -          sqlite3Fts3ErrMsg(pzErr, "unrecognized parameter: %s", z);
  1224         -          rc = SQLITE_ERROR;
  1225         -        }else{
  1226         -          switch( iOpt ){
  1227         -            case 0:               /* MATCHINFO */
  1228         -              if( strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "fts3", 4) ){
  1229         -                sqlite3Fts3ErrMsg(pzErr, "unrecognized matchinfo: %s", zVal);
  1230         -                rc = SQLITE_ERROR;
  1231         -              }
  1232         -              bNoDocsize = 1;
  1233         -              break;
  1234         -
  1235         -            case 1:               /* PREFIX */
  1236         -              sqlite3_free(zPrefix);
  1237         -              zPrefix = zVal;
  1238         -              zVal = 0;
  1239         -              break;
  1240         -
  1241         -            case 2:               /* COMPRESS */
  1242         -              sqlite3_free(zCompress);
  1243         -              zCompress = zVal;
  1244         -              zVal = 0;
  1245         -              break;
  1246         -
  1247         -            case 3:               /* UNCOMPRESS */
  1248         -              sqlite3_free(zUncompress);
  1249         -              zUncompress = zVal;
  1250         -              zVal = 0;
  1251         -              break;
  1252         -
  1253         -            case 4:               /* ORDER */
  1254         -              if( (strlen(zVal)!=3 || sqlite3_strnicmp(zVal, "asc", 3)) 
  1255         -               && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 4)) 
  1256         -              ){
  1257         -                sqlite3Fts3ErrMsg(pzErr, "unrecognized order: %s", zVal);
  1258         -                rc = SQLITE_ERROR;
  1259         -              }
  1260         -              bDescIdx = (zVal[0]=='d' || zVal[0]=='D');
  1261         -              break;
  1262         -
  1263         -            case 5:              /* CONTENT */
  1264         -              sqlite3_free(zContent);
  1265         -              zContent = zVal;
  1266         -              zVal = 0;
  1267         -              break;
  1268         -
  1269         -            case 6:              /* LANGUAGEID */
  1270         -              assert( iOpt==6 );
  1271         -              sqlite3_free(zLanguageid);
  1272         -              zLanguageid = zVal;
  1273         -              zVal = 0;
  1274         -              break;
  1275         -
  1276         -            case 7:              /* NOTINDEXED */
  1277         -              azNotindexed[nNotindexed++] = zVal;
  1278         -              zVal = 0;
  1279         -              break;
  1280         -          }
         1224  +        switch( iOpt ){
         1225  +          case 0:               /* MATCHINFO */
         1226  +            if( strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "fts3", 4) ){
         1227  +              sqlite3Fts3ErrMsg(pzErr, "unrecognized matchinfo: %s", zVal);
         1228  +              rc = SQLITE_ERROR;
         1229  +            }
         1230  +            bNoDocsize = 1;
         1231  +            break;
         1232  +
         1233  +          case 1:               /* PREFIX */
         1234  +            sqlite3_free(zPrefix);
         1235  +            zPrefix = zVal;
         1236  +            zVal = 0;
         1237  +            break;
         1238  +
         1239  +          case 2:               /* COMPRESS */
         1240  +            sqlite3_free(zCompress);
         1241  +            zCompress = zVal;
         1242  +            zVal = 0;
         1243  +            break;
         1244  +
         1245  +          case 3:               /* UNCOMPRESS */
         1246  +            sqlite3_free(zUncompress);
         1247  +            zUncompress = zVal;
         1248  +            zVal = 0;
         1249  +            break;
         1250  +
         1251  +          case 4:               /* ORDER */
         1252  +            if( (strlen(zVal)!=3 || sqlite3_strnicmp(zVal, "asc", 3)) 
         1253  +             && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 4)) 
         1254  +            ){
         1255  +              sqlite3Fts3ErrMsg(pzErr, "unrecognized order: %s", zVal);
         1256  +              rc = SQLITE_ERROR;
         1257  +            }
         1258  +            bDescIdx = (zVal[0]=='d' || zVal[0]=='D');
         1259  +            break;
         1260  +
         1261  +          case 5:              /* CONTENT */
         1262  +            sqlite3_free(zContent);
         1263  +            zContent = zVal;
         1264  +            zVal = 0;
         1265  +            break;
         1266  +
         1267  +          case 6:              /* LANGUAGEID */
         1268  +            assert( iOpt==6 );
         1269  +            sqlite3_free(zLanguageid);
         1270  +            zLanguageid = zVal;
         1271  +            zVal = 0;
         1272  +            break;
         1273  +
         1274  +          case 7:              /* NOTINDEXED */
         1275  +            azNotindexed[nNotindexed++] = zVal;
         1276  +            zVal = 0;
         1277  +            break;
         1278  +
         1279  +          default:
         1280  +            assert( iOpt==SizeofArray(aFts4Opt) );
         1281  +            sqlite3Fts3ErrMsg(pzErr, "unrecognized parameter: %s", z);
         1282  +            rc = SQLITE_ERROR;
         1283  +            break;
  1281   1284           }
  1282   1285           sqlite3_free(zVal);
  1283   1286         }
  1284   1287       }
  1285   1288   
  1286   1289       /* Otherwise, the argument is a column name. */
  1287   1290       else {
................................................................................
  1842   1845       ** the size of zBuffer if required.  */
  1843   1846       if( !isFirstTerm ){
  1844   1847         zCsr += fts3GetVarint32(zCsr, &nPrefix);
  1845   1848       }
  1846   1849       isFirstTerm = 0;
  1847   1850       zCsr += fts3GetVarint32(zCsr, &nSuffix);
  1848   1851       
  1849         -    if( nPrefix<0 || nSuffix<0 || &zCsr[nSuffix]>zEnd ){
         1852  +    assert( nPrefix>=0 && nSuffix>=0 );
         1853  +    if( &zCsr[nSuffix]>zEnd ){
  1850   1854         rc = FTS_CORRUPT_VTAB;
  1851   1855         goto finish_scan;
  1852   1856       }
  1853   1857       if( nPrefix+nSuffix>nAlloc ){
  1854   1858         char *zNew;
  1855   1859         nAlloc = (nPrefix+nSuffix) * 2;
  1856   1860         zNew = (char *)sqlite3_realloc(zBuffer, nAlloc);
................................................................................
  2652   2656         nOut += sqlite3Fts3PutVarint(&pOut[nOut], iDelta);
  2653   2657         pOut[nOut++] = 0x02;
  2654   2658         bWritten = 1;
  2655   2659       }
  2656   2660       fts3ColumnlistCopy(0, &p);
  2657   2661     }
  2658   2662   
  2659         -  while( p<pEnd && *p==0x01 ){
         2663  +  while( p<pEnd ){
  2660   2664       sqlite3_int64 iCol;
  2661   2665       p++;
  2662   2666       p += sqlite3Fts3GetVarint(p, &iCol);
  2663   2667       if( *p==0x02 ){
  2664   2668         if( bWritten==0 ){
  2665   2669           nOut += sqlite3Fts3PutVarint(&pOut[nOut], iDelta);
  2666   2670           bWritten = 1;
................................................................................
  3332   3336     int rc = SQLITE_OK;             /* Return Code */
  3333   3337     Fts3Cursor *pCsr = (Fts3Cursor *) pCursor;
  3334   3338     Fts3Table *p = (Fts3Table *)pCursor->pVtab;
  3335   3339   
  3336   3340     /* The column value supplied by SQLite must be in range. */
  3337   3341     assert( iCol>=0 && iCol<=p->nColumn+2 );
  3338   3342   
  3339         -  if( iCol==p->nColumn+1 ){
  3340         -    /* This call is a request for the "docid" column. Since "docid" is an 
  3341         -    ** alias for "rowid", use the xRowid() method to obtain the value.
  3342         -    */
  3343         -    sqlite3_result_int64(pCtx, pCsr->iPrevId);
  3344         -  }else if( iCol==p->nColumn ){
  3345         -    /* The extra column whose name is the same as the table.
  3346         -    ** Return a blob which is a pointer to the cursor.  */
  3347         -    sqlite3_result_blob(pCtx, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT);
  3348         -  }else if( iCol==p->nColumn+2 && pCsr->pExpr ){
  3349         -    sqlite3_result_int64(pCtx, pCsr->iLangid);
  3350         -  }else{
  3351         -    /* The requested column is either a user column (one that contains 
  3352         -    ** indexed data), or the language-id column.  */
  3353         -    rc = fts3CursorSeek(0, pCsr);
  3354         -
  3355         -    if( rc==SQLITE_OK ){
  3356         -      if( iCol==p->nColumn+2 ){
  3357         -        int iLangid = 0;
  3358         -        if( p->zLanguageid ){
  3359         -          iLangid = sqlite3_column_int(pCsr->pStmt, p->nColumn+1);
  3360         -        }
  3361         -        sqlite3_result_int(pCtx, iLangid);
  3362         -      }else if( sqlite3_data_count(pCsr->pStmt)>(iCol+1) ){
         3343  +  switch( iCol-p->nColumn ){
         3344  +    case 0:
         3345  +      /* The special 'table-name' column */
         3346  +      sqlite3_result_blob(pCtx, &pCsr, sizeof(Fts3Cursor*), SQLITE_TRANSIENT);
         3347  +      sqlite3_result_subtype(pCtx, SQLITE_BLOB);
         3348  +      break;
         3349  +
         3350  +    case 1:
         3351  +      /* The docid column */
         3352  +      sqlite3_result_int64(pCtx, pCsr->iPrevId);
         3353  +      break;
         3354  +
         3355  +    case 2:
         3356  +      if( pCsr->pExpr ){
         3357  +        sqlite3_result_int64(pCtx, pCsr->iLangid);
         3358  +        break;
         3359  +      }else if( p->zLanguageid==0 ){
         3360  +        sqlite3_result_int(pCtx, 0);
         3361  +        break;
         3362  +      }else{
         3363  +        iCol = p->nColumn;
         3364  +        /* fall-through */
         3365  +      }
         3366  +
         3367  +    default:
         3368  +      /* A user column. Or, if this is a full-table scan, possibly the
         3369  +      ** language-id column. Seek the cursor. */
         3370  +      rc = fts3CursorSeek(0, pCsr);
         3371  +      if( rc==SQLITE_OK && sqlite3_data_count(pCsr->pStmt)-1>iCol ){
  3363   3372           sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pStmt, iCol+1));
  3364   3373         }
  3365         -    }
         3374  +      break;
  3366   3375     }
  3367   3376   
  3368   3377     assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
  3369   3378     return rc;
  3370   3379   }
  3371   3380   
  3372   3381   /* 
................................................................................
  3438   3447   ** table (if p->bHasStat==2), attempt to determine this (set p->bHasStat
  3439   3448   ** to 0 or 1). Return SQLITE_OK if successful, or an SQLite error code
  3440   3449   ** if an error occurs.
  3441   3450   */
  3442   3451   static int fts3SetHasStat(Fts3Table *p){
  3443   3452     int rc = SQLITE_OK;
  3444   3453     if( p->bHasStat==2 ){
  3445         -    const char *zFmt ="SELECT 1 FROM %Q.sqlite_master WHERE tbl_name='%q_stat'";
  3446         -    char *zSql = sqlite3_mprintf(zFmt, p->zDb, p->zName);
  3447         -    if( zSql ){
  3448         -      sqlite3_stmt *pStmt = 0;
  3449         -      rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
  3450         -      if( rc==SQLITE_OK ){
  3451         -        int bHasStat = (sqlite3_step(pStmt)==SQLITE_ROW);
  3452         -        rc = sqlite3_finalize(pStmt);
  3453         -        if( rc==SQLITE_OK ) p->bHasStat = (u8)bHasStat;
  3454         -      }
  3455         -      sqlite3_free(zSql);
         3454  +    char *zTbl = sqlite3_mprintf("%s_stat", p->zName);
         3455  +    if( zTbl ){
         3456  +      int res = sqlite3_table_column_metadata(p->db, p->zDb, zTbl, 0,0,0,0,0,0);
         3457  +      sqlite3_free(zTbl);
         3458  +      p->bHasStat = (res==SQLITE_OK);
  3456   3459       }else{
  3457   3460         rc = SQLITE_NOMEM;
  3458   3461       }
  3459   3462     }
  3460   3463     return rc;
  3461   3464   }
  3462   3465   
................................................................................
  3555   3558   */
  3556   3559   static int fts3FunctionArg(
  3557   3560     sqlite3_context *pContext,      /* SQL function call context */
  3558   3561     const char *zFunc,              /* Function name */
  3559   3562     sqlite3_value *pVal,            /* argv[0] passed to function */
  3560   3563     Fts3Cursor **ppCsr              /* OUT: Store cursor handle here */
  3561   3564   ){
  3562         -  Fts3Cursor *pRet;
  3563         -  if( sqlite3_value_type(pVal)!=SQLITE_BLOB 
  3564         -   || sqlite3_value_bytes(pVal)!=sizeof(Fts3Cursor *)
  3565         -  ){
         3565  +  int rc = SQLITE_OK;
         3566  +  if( sqlite3_value_subtype(pVal)==SQLITE_BLOB ){
         3567  +    *ppCsr = *(Fts3Cursor**)sqlite3_value_blob(pVal);
         3568  +  }else{
  3566   3569       char *zErr = sqlite3_mprintf("illegal first argument to %s", zFunc);
  3567   3570       sqlite3_result_error(pContext, zErr, -1);
  3568   3571       sqlite3_free(zErr);
  3569         -    return SQLITE_ERROR;
         3572  +    rc = SQLITE_ERROR;
  3570   3573     }
  3571         -  memcpy(&pRet, sqlite3_value_blob(pVal), sizeof(Fts3Cursor *));
  3572         -  *ppCsr = pRet;
  3573         -  return SQLITE_OK;
         3574  +  return rc;
  3574   3575   }
  3575   3576   
  3576   3577   /*
  3577   3578   ** Implementation of the snippet() function for FTS3
  3578   3579   */
  3579   3580   static void fts3SnippetFunc(
  3580   3581     sqlite3_context *pContext,      /* SQLite function call context */
................................................................................
  3953   3954   #ifdef SQLITE_TEST
  3954   3955     if( rc==SQLITE_OK ){
  3955   3956       rc = sqlite3Fts3ExprInitTestInterface(db);
  3956   3957     }
  3957   3958   #endif
  3958   3959   
  3959   3960     /* Create the virtual table wrapper around the hash-table and overload 
  3960         -  ** the two scalar functions. If this is successful, register the
         3961  +  ** the four scalar functions. If this is successful, register the
  3961   3962     ** module with sqlite.
  3962   3963     */
  3963   3964     if( SQLITE_OK==rc 
  3964   3965      && SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer"))
  3965   3966      && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1))
  3966   3967      && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", 1))
  3967   3968      && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 1))
................................................................................
  4536   4537     Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
  4537   4538     u8 bEof = 0;
  4538   4539   
  4539   4540     /* This is only called if it is guaranteed that the phrase has at least
  4540   4541     ** one incremental token. In which case the bIncr flag is set. */
  4541   4542     assert( p->bIncr==1 );
  4542   4543   
  4543         -  if( p->nToken==1 && p->bIncr ){
         4544  +  if( p->nToken==1 ){
  4544   4545       rc = sqlite3Fts3MsrIncrNext(pTab, p->aToken[0].pSegcsr, 
  4545   4546           &pDL->iDocid, &pDL->pList, &pDL->nList
  4546   4547       );
  4547   4548       if( pDL->pList==0 ) bEof = 1;
  4548   4549     }else{
  4549   4550       int bDescDoclist = pCsr->bDesc;
  4550   4551       struct TokenDoclist a[MAX_INCR_PHRASE_TOKENS];
................................................................................
  4769   4770   ** The average document size in pages is calculated by first calculating 
  4770   4771   ** determining the average size in bytes, B. If B is less than the amount
  4771   4772   ** of data that will fit on a single leaf page of an intkey table in
  4772   4773   ** this database, then the average docsize is 1. Otherwise, it is 1 plus
  4773   4774   ** the number of overflow pages consumed by a record B bytes in size.
  4774   4775   */
  4775   4776   static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){
         4777  +  int rc = SQLITE_OK;
  4776   4778     if( pCsr->nRowAvg==0 ){
  4777   4779       /* The average document size, which is required to calculate the cost
  4778   4780       ** of each doclist, has not yet been determined. Read the required 
  4779   4781       ** data from the %_stat table to calculate it.
  4780   4782       **
  4781   4783       ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 
  4782   4784       ** varints, where nCol is the number of columns in the FTS3 table.
................................................................................
  4808   4810         return FTS_CORRUPT_VTAB;
  4809   4811       }
  4810   4812   
  4811   4813       pCsr->nDoc = nDoc;
  4812   4814       pCsr->nRowAvg = (int)(((nByte / nDoc) + p->nPgsz) / p->nPgsz);
  4813   4815       assert( pCsr->nRowAvg>0 ); 
  4814   4816       rc = sqlite3_reset(pStmt);
  4815         -    if( rc!=SQLITE_OK ) return rc;
  4816   4817     }
  4817   4818   
  4818   4819     *pnPage = pCsr->nRowAvg;
  4819         -  return SQLITE_OK;
         4820  +  return rc;
  4820   4821   }
  4821   4822   
  4822   4823   /*
  4823   4824   ** This function is called to select the tokens (if any) that will be 
  4824   4825   ** deferred. The array aTC[] has already been populated when this is
  4825   4826   ** called.
  4826   4827   **
................................................................................
  5162   5163               }else{
  5163   5164                 fts3EvalNextRow(pCsr, pRight, pRc);
  5164   5165               }
  5165   5166             }
  5166   5167             pExpr->iDocid = pLeft->iDocid;
  5167   5168             pExpr->bEof = (pLeft->bEof || pRight->bEof);
  5168   5169             if( pExpr->eType==FTSQUERY_NEAR && pExpr->bEof ){
  5169         -            if( pRight->pPhrase && pRight->pPhrase->doclist.aAll ){
         5170  +            assert( pRight->eType==FTSQUERY_PHRASE );
         5171  +            if( pRight->pPhrase->doclist.aAll ){
  5170   5172                 Fts3Doclist *pDl = &pRight->pPhrase->doclist;
  5171   5173                 while( *pRc==SQLITE_OK && pRight->bEof==0 ){
  5172   5174                   memset(pDl->pList, 0, pDl->nList);
  5173   5175                   fts3EvalNextRow(pCsr, pRight, pRc);
  5174   5176                 }
  5175   5177               }
  5176   5178               if( pLeft->pPhrase && pLeft->pPhrase->doclist.aAll ){
................................................................................
  5191   5193           sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
  5192   5194   
  5193   5195           assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid );
  5194   5196           assert( pRight->bStart || pLeft->iDocid==pRight->iDocid );
  5195   5197   
  5196   5198           if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){
  5197   5199             fts3EvalNextRow(pCsr, pLeft, pRc);
  5198         -        }else if( pLeft->bEof || (pRight->bEof==0 && iCmp>0) ){
         5200  +        }else if( pLeft->bEof || iCmp>0 ){
  5199   5201             fts3EvalNextRow(pCsr, pRight, pRc);
  5200   5202           }else{
  5201   5203             fts3EvalNextRow(pCsr, pLeft, pRc);
  5202   5204             fts3EvalNextRow(pCsr, pRight, pRc);
  5203   5205           }
  5204   5206   
  5205   5207           pExpr->bEof = (pLeft->bEof && pRight->bEof);
................................................................................
  5283   5285     **
  5284   5286     ** The right-hand child of a NEAR node is always a phrase. The 
  5285   5287     ** left-hand child may be either a phrase or a NEAR node. There are
  5286   5288     ** no exceptions to this - it's the way the parser in fts3_expr.c works.
  5287   5289     */
  5288   5290     if( *pRc==SQLITE_OK 
  5289   5291      && pExpr->eType==FTSQUERY_NEAR 
  5290         -   && pExpr->bEof==0
  5291   5292      && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR)
  5292   5293     ){
  5293   5294       Fts3Expr *p; 
  5294   5295       int nTmp = 0;                 /* Bytes of temp space */
  5295   5296       char *aTmp;                   /* Temp space for PoslistNearMerge() */
  5296   5297   
  5297   5298       /* Allocate temporary working space. */
  5298   5299       for(p=pExpr; p->pLeft; p=p->pLeft){
         5300  +      assert( p->pRight->pPhrase->doclist.nList>0 );
  5299   5301         nTmp += p->pRight->pPhrase->doclist.nList;
  5300   5302       }
  5301   5303       nTmp += p->pPhrase->doclist.nList;
  5302         -    if( nTmp==0 ){
         5304  +    aTmp = sqlite3_malloc(nTmp*2);
         5305  +    if( !aTmp ){
         5306  +      *pRc = SQLITE_NOMEM;
  5303   5307         res = 0;
  5304   5308       }else{
  5305         -      aTmp = sqlite3_malloc(nTmp*2);
  5306         -      if( !aTmp ){
  5307         -        *pRc = SQLITE_NOMEM;
  5308         -        res = 0;
  5309         -      }else{
  5310         -        char *aPoslist = p->pPhrase->doclist.pList;
  5311         -        int nToken = p->pPhrase->nToken;
  5312         -
  5313         -        for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){
  5314         -          Fts3Phrase *pPhrase = p->pRight->pPhrase;
  5315         -          int nNear = p->nNear;
  5316         -          res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase);
  5317         -        }
  5318         -
  5319         -        aPoslist = pExpr->pRight->pPhrase->doclist.pList;
  5320         -        nToken = pExpr->pRight->pPhrase->nToken;
  5321         -        for(p=pExpr->pLeft; p && res; p=p->pLeft){
  5322         -          int nNear;
  5323         -          Fts3Phrase *pPhrase;
  5324         -          assert( p->pParent && p->pParent->pLeft==p );
  5325         -          nNear = p->pParent->nNear;
  5326         -          pPhrase = (
  5327         -              p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase
  5328         -              );
  5329         -          res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase);
  5330         -        }
         5309  +      char *aPoslist = p->pPhrase->doclist.pList;
         5310  +      int nToken = p->pPhrase->nToken;
         5311  +
         5312  +      for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){
         5313  +        Fts3Phrase *pPhrase = p->pRight->pPhrase;
         5314  +        int nNear = p->nNear;
         5315  +        res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase);
         5316  +      }
         5317  +
         5318  +      aPoslist = pExpr->pRight->pPhrase->doclist.pList;
         5319  +      nToken = pExpr->pRight->pPhrase->nToken;
         5320  +      for(p=pExpr->pLeft; p && res; p=p->pLeft){
         5321  +        int nNear;
         5322  +        Fts3Phrase *pPhrase;
         5323  +        assert( p->pParent && p->pParent->pLeft==p );
         5324  +        nNear = p->pParent->nNear;
         5325  +        pPhrase = (
         5326  +            p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase
         5327  +        );
         5328  +        res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase);
  5331   5329         }
         5330  +    }
  5332   5331   
  5333         -      sqlite3_free(aTmp);
  5334         -    }
         5332  +    sqlite3_free(aTmp);
  5335   5333     }
  5336   5334   
  5337   5335     return res;
  5338   5336   }
  5339   5337   
  5340   5338   /*
  5341   5339   ** This function is a helper function for sqlite3Fts3EvalTestDeferred().

Changes to ext/fts5/fts5Int.h.

   650    650     const char *p;                  /* Token text (not NULL terminated) */
   651    651     int n;                          /* Size of buffer p in bytes */
   652    652   };
   653    653   
   654    654   /* Parse a MATCH expression. */
   655    655   int sqlite3Fts5ExprNew(
   656    656     Fts5Config *pConfig, 
          657  +  int iCol,                       /* Column on LHS of MATCH operator */
   657    658     const char *zExpr,
   658    659     Fts5Expr **ppNew, 
   659    660     char **pzErr
   660    661   );
   661    662   
   662    663   /*
   663    664   ** for(rc = sqlite3Fts5ExprFirst(pExpr, pIdx, bDesc);
................................................................................
   734    735   );
   735    736   
   736    737   void sqlite3Fts5ParsePhraseFree(Fts5ExprPhrase*);
   737    738   void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset*);
   738    739   void sqlite3Fts5ParseNodeFree(Fts5ExprNode*);
   739    740   
   740    741   void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*);
   741         -void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNearset*, Fts5Colset*);
          742  +void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNode*, Fts5Colset*);
   742    743   Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse*, Fts5Colset*);
   743    744   void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p);
   744    745   void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*);
   745    746   
   746    747   /*
   747    748   ** End of interface to code in fts5_expr.c.
   748    749   **************************************************************************/

Changes to ext/fts5/fts5_expr.c.

   209    209   }
   210    210   
   211    211   static void *fts5ParseAlloc(u64 t){ return sqlite3_malloc((int)t); }
   212    212   static void fts5ParseFree(void *p){ sqlite3_free(p); }
   213    213   
   214    214   int sqlite3Fts5ExprNew(
   215    215     Fts5Config *pConfig,            /* FTS5 Configuration */
          216  +  int iCol,
   216    217     const char *zExpr,              /* Expression text */
   217    218     Fts5Expr **ppNew, 
   218    219     char **pzErr
   219    220   ){
   220    221     Fts5Parse sParse;
   221    222     Fts5Token token;
   222    223     const char *z = zExpr;
................................................................................
   232    233     sParse.pConfig = pConfig;
   233    234   
   234    235     do {
   235    236       t = fts5ExprGetToken(&sParse, &z, &token);
   236    237       sqlite3Fts5Parser(pEngine, t, token, &sParse);
   237    238     }while( sParse.rc==SQLITE_OK && t!=FTS5_EOF );
   238    239     sqlite3Fts5ParserFree(pEngine, fts5ParseFree);
          240  +
          241  +  /* If the LHS of the MATCH expression was a user column, apply the
          242  +  ** implicit column-filter.  */
          243  +  if( iCol<pConfig->nCol && sParse.pExpr && sParse.rc==SQLITE_OK ){
          244  +    int n = sizeof(Fts5Colset);
          245  +    Fts5Colset *pColset = (Fts5Colset*)sqlite3Fts5MallocZero(&sParse.rc, n);
          246  +    if( pColset ){
          247  +      pColset->nCol = 1;
          248  +      pColset->aiCol[0] = iCol;
          249  +      sqlite3Fts5ParseSetColset(&sParse, sParse.pExpr, pColset);
          250  +    }
          251  +  }
   239    252   
   240    253     assert( sParse.rc!=SQLITE_OK || sParse.zErr==0 );
   241    254     if( sParse.rc==SQLITE_OK ){
   242    255       *ppNew = pNew = sqlite3_malloc(sizeof(Fts5Expr));
   243    256       if( pNew==0 ){
   244    257         sParse.rc = SQLITE_NOMEM;
   245    258         sqlite3Fts5ParseNodeFree(sParse.pExpr);
................................................................................
  1882   1895       assert( pParse->rc!=SQLITE_OK );
  1883   1896       sqlite3_free(pColset);
  1884   1897     }
  1885   1898   
  1886   1899     return pRet;
  1887   1900   }
  1888   1901   
         1902  +/*
         1903  +** If argument pOrig is NULL, or if (*pRc) is set to anything other than
         1904  +** SQLITE_OK when this function is called, NULL is returned. 
         1905  +**
         1906  +** Otherwise, a copy of (*pOrig) is made into memory obtained from
         1907  +** sqlite3Fts5MallocZero() and a pointer to it returned. If the allocation
         1908  +** fails, (*pRc) is set to SQLITE_NOMEM and NULL is returned.
         1909  +*/
         1910  +static Fts5Colset *fts5CloneColset(int *pRc, Fts5Colset *pOrig){
         1911  +  Fts5Colset *pRet;
         1912  +  if( pOrig ){
         1913  +    int nByte = sizeof(Fts5Colset) + (pOrig->nCol-1) * sizeof(int);
         1914  +    pRet = (Fts5Colset*)sqlite3Fts5MallocZero(pRc, nByte);
         1915  +    if( pRet ){ 
         1916  +      memcpy(pRet, pOrig, nByte);
         1917  +    }
         1918  +  }else{
         1919  +    pRet = 0;
         1920  +  }
         1921  +  return pRet;
         1922  +}
         1923  +
         1924  +/*
         1925  +** Remove from colset pColset any columns that are not also in colset pMerge.
         1926  +*/
         1927  +static void fts5MergeColset(Fts5Colset *pColset, Fts5Colset *pMerge){
         1928  +  int iIn = 0;          /* Next input in pColset */
         1929  +  int iMerge = 0;       /* Next input in pMerge */
         1930  +  int iOut = 0;         /* Next output slot in pColset */
         1931  +
         1932  +  while( iIn<pColset->nCol && iMerge<pMerge->nCol ){
         1933  +    int iDiff = pColset->aiCol[iIn] - pMerge->aiCol[iMerge];
         1934  +    if( iDiff==0 ){
         1935  +      pColset->aiCol[iOut++] = pMerge->aiCol[iMerge];
         1936  +      iMerge++;
         1937  +      iIn++;
         1938  +    }else if( iDiff>0 ){
         1939  +      iMerge++;
         1940  +    }else{
         1941  +      iIn++;
         1942  +    }
         1943  +  }
         1944  +  pColset->nCol = iOut;
         1945  +}
         1946  +
         1947  +/*
         1948  +** Recursively apply colset pColset to expression node pNode and all of
         1949  +** its decendents. If (*ppFree) is not NULL, it contains a spare copy
         1950  +** of pColset. This function may use the spare copy and set (*ppFree) to
         1951  +** zero, or it may create copies of pColset using fts5CloneColset().
         1952  +*/
         1953  +static void fts5ParseSetColset(
         1954  +  Fts5Parse *pParse, 
         1955  +  Fts5ExprNode *pNode, 
         1956  +  Fts5Colset *pColset,
         1957  +  Fts5Colset **ppFree
         1958  +){
         1959  +  if( pParse->rc==SQLITE_OK ){
         1960  +    assert( pNode->eType==FTS5_TERM || pNode->eType==FTS5_STRING 
         1961  +         || pNode->eType==FTS5_AND  || pNode->eType==FTS5_OR
         1962  +         || pNode->eType==FTS5_NOT  || pNode->eType==FTS5_EOF
         1963  +    );
         1964  +    if( pNode->eType==FTS5_STRING || pNode->eType==FTS5_TERM ){
         1965  +      Fts5ExprNearset *pNear = pNode->pNear;
         1966  +      if( pNear->pColset ){
         1967  +        fts5MergeColset(pNear->pColset, pColset);
         1968  +        if( pNear->pColset->nCol==0 ){
         1969  +          pNode->eType = FTS5_EOF;
         1970  +          pNode->xNext = 0;
         1971  +        }
         1972  +      }else if( *ppFree ){
         1973  +        pNear->pColset = pColset;
         1974  +        *ppFree = 0;
         1975  +      }else{
         1976  +        pNear->pColset = fts5CloneColset(&pParse->rc, pColset);
         1977  +      }
         1978  +    }else{
         1979  +      int i;
         1980  +      assert( pNode->eType!=FTS5_EOF || pNode->nChild==0 );
         1981  +      for(i=0; i<pNode->nChild; i++){
         1982  +        fts5ParseSetColset(pParse, pNode->apChild[i], pColset, ppFree);
         1983  +      }
         1984  +    }
         1985  +  }
         1986  +}
         1987  +
         1988  +/*
         1989  +** Apply colset pColset to expression node pExpr and all of its descendents.
         1990  +*/
  1889   1991   void sqlite3Fts5ParseSetColset(
  1890   1992     Fts5Parse *pParse, 
  1891         -  Fts5ExprNearset *pNear, 
         1993  +  Fts5ExprNode *pExpr, 
  1892   1994     Fts5Colset *pColset 
  1893   1995   ){
         1996  +  Fts5Colset *pFree = pColset;
  1894   1997     if( pParse->pConfig->eDetail==FTS5_DETAIL_NONE ){
  1895   1998       pParse->rc = SQLITE_ERROR;
  1896   1999       pParse->zErr = sqlite3_mprintf(
  1897   2000         "fts5: column queries are not supported (detail=none)"
  1898   2001       );
  1899         -    sqlite3_free(pColset);
  1900         -    return;
  1901         -  }
  1902         -
  1903         -  if( pNear ){
  1904         -    pNear->pColset = pColset;
  1905   2002     }else{
  1906         -    sqlite3_free(pColset);
         2003  +    fts5ParseSetColset(pParse, pExpr, pColset, &pFree);
  1907   2004     }
         2005  +  sqlite3_free(pFree);
  1908   2006   }
  1909   2007   
  1910   2008   static void fts5ExprAssignXNext(Fts5ExprNode *pNode){
  1911   2009     switch( pNode->eType ){
  1912   2010       case FTS5_STRING: {
  1913   2011         Fts5ExprNearset *pNear = pNode->pNear;
  1914   2012         if( pNear->nPhrase==1 && pNear->apPhrase[0]->nTerm==1 
................................................................................
  2354   2452       azConfig[i++] = (const char*)sqlite3_value_text(apVal[iArg]);
  2355   2453     }
  2356   2454   
  2357   2455     zExpr = (const char*)sqlite3_value_text(apVal[0]);
  2358   2456   
  2359   2457     rc = sqlite3Fts5ConfigParse(pGlobal, db, nConfig, azConfig, &pConfig, &zErr);
  2360   2458     if( rc==SQLITE_OK ){
  2361         -    rc = sqlite3Fts5ExprNew(pConfig, zExpr, &pExpr, &zErr);
         2459  +    rc = sqlite3Fts5ExprNew(pConfig, pConfig->nCol, zExpr, &pExpr, &zErr);
  2362   2460     }
  2363   2461     if( rc==SQLITE_OK ){
  2364   2462       char *zText;
  2365   2463       if( pExpr->pRoot->xNext==0 ){
  2366   2464         zText = sqlite3_mprintf("");
  2367   2465       }else if( bTcl ){
  2368   2466         zText = fts5ExprPrintTcl(pConfig, zNearsetCmd, pExpr->pRoot);

Changes to ext/fts5/fts5_index.c.

  2874   2874   static void fts5MultiIterNext2(
  2875   2875     Fts5Index *p, 
  2876   2876     Fts5Iter *pIter,
  2877   2877     int *pbNewTerm                  /* OUT: True if *might* be new term */
  2878   2878   ){
  2879   2879     assert( pIter->bSkipEmpty );
  2880   2880     if( p->rc==SQLITE_OK ){
  2881         -    do {
         2881  +    *pbNewTerm = 0;
         2882  +    do{
  2882   2883         int iFirst = pIter->aFirst[1].iFirst;
  2883   2884         Fts5SegIter *pSeg = &pIter->aSeg[iFirst];
  2884   2885         int bNewTerm = 0;
  2885   2886   
  2886   2887         assert( p->rc==SQLITE_OK );
  2887   2888         pSeg->xNext(p, pSeg, &bNewTerm);
  2888   2889         if( pSeg->pLeaf==0 || bNewTerm 
  2889   2890          || fts5MultiIterAdvanceRowid(pIter, iFirst, &pSeg)
  2890   2891         ){
  2891   2892           fts5MultiIterAdvanced(p, pIter, iFirst, 1);
  2892   2893           fts5MultiIterSetEof(pIter);
  2893   2894           *pbNewTerm = 1;
  2894         -      }else{
  2895         -        *pbNewTerm = 0;
  2896   2895         }
  2897   2896         fts5AssertMultiIterSetup(p, pIter);
  2898   2897   
  2899   2898       }while( fts5MultiIterIsEmpty(p, pIter) );
  2900   2899     }
  2901   2900   }
  2902   2901   
................................................................................
  3154   3153     while( p<pEnd && *p!=0x01 ){
  3155   3154       while( *p++ & 0x80 );
  3156   3155     }
  3157   3156   
  3158   3157     return p - (*pa);
  3159   3158   }
  3160   3159   
  3161         -static int fts5IndexExtractColset (
         3160  +static void fts5IndexExtractColset(
         3161  +  int *pRc,
  3162   3162     Fts5Colset *pColset,            /* Colset to filter on */
  3163   3163     const u8 *pPos, int nPos,       /* Position list */
  3164   3164     Fts5Buffer *pBuf                /* Output buffer */
  3165   3165   ){
  3166         -  int rc = SQLITE_OK;
  3167         -  int i;
  3168         -
  3169         -  fts5BufferZero(pBuf);
  3170         -  for(i=0; i<pColset->nCol; i++){
  3171         -    const u8 *pSub = pPos;
  3172         -    int nSub = fts5IndexExtractCol(&pSub, nPos, pColset->aiCol[i]);
  3173         -    if( nSub ){
  3174         -      fts5BufferAppendBlob(&rc, pBuf, nSub, pSub);
         3166  +  if( *pRc==SQLITE_OK ){
         3167  +    int i;
         3168  +    fts5BufferZero(pBuf);
         3169  +    for(i=0; i<pColset->nCol; i++){
         3170  +      const u8 *pSub = pPos;
         3171  +      int nSub = fts5IndexExtractCol(&pSub, nPos, pColset->aiCol[i]);
         3172  +      if( nSub ){
         3173  +        fts5BufferAppendBlob(pRc, pBuf, nSub, pSub);
         3174  +      }
  3175   3175       }
  3176   3176     }
  3177         -  return rc;
  3178   3177   }
  3179   3178   
  3180   3179   /*
  3181   3180   ** xSetOutputs callback used by detail=none tables.
  3182   3181   */
  3183   3182   static void fts5IterSetOutputs_None(Fts5Iter *pIter, Fts5SegIter *pSeg){
  3184   3183     assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_NONE );
................................................................................
  3294   3293       /* All data is stored on the current page. Populate the output 
  3295   3294       ** variables to point into the body of the page object. */
  3296   3295       const u8 *a = &pSeg->pLeaf->p[pSeg->iLeafOffset];
  3297   3296       if( pColset->nCol==1 ){
  3298   3297         pIter->base.nData = fts5IndexExtractCol(&a, pSeg->nPos,pColset->aiCol[0]);
  3299   3298         pIter->base.pData = a;
  3300   3299       }else{
         3300  +      int *pRc = &pIter->pIndex->rc;
  3301   3301         fts5BufferZero(&pIter->poslist);
  3302         -      fts5IndexExtractColset(pColset, a, pSeg->nPos, &pIter->poslist);
         3302  +      fts5IndexExtractColset(pRc, pColset, a, pSeg->nPos, &pIter->poslist);
  3303   3303         pIter->base.pData = pIter->poslist.p;
  3304   3304         pIter->base.nData = pIter->poslist.n;
  3305   3305       }
  3306   3306     }else{
  3307   3307       /* The data is distributed over two or more pages. Copy it into the
  3308   3308       ** Fts5Iter.poslist buffer and then set the output pointer to point
  3309   3309       ** to this buffer.  */
................................................................................
  3840   3840   }
  3841   3841   
  3842   3842   static void fts5WriteFlushLeaf(Fts5Index *p, Fts5SegWriter *pWriter){
  3843   3843     static const u8 zero[] = { 0x00, 0x00, 0x00, 0x00 };
  3844   3844     Fts5PageWriter *pPage = &pWriter->writer;
  3845   3845     i64 iRowid;
  3846   3846   
  3847         -static int nCall = 0;
  3848         -nCall++;
  3849         -
  3850   3847     assert( (pPage->pgidx.n==0)==(pWriter->bFirstTermInPage) );
  3851   3848   
  3852   3849     /* Set the szLeaf header field. */
  3853   3850     assert( 0==fts5GetU16(&pPage->buf.p[2]) );
  3854   3851     fts5PutU16(&pPage->buf.p[2], (u16)pPage->buf.n);
  3855   3852   
  3856   3853     if( pWriter->bFirstTermInPage ){

Changes to ext/fts5/fts5_main.c.

   502    502   **       * An == rowid constraint:       cost=10.0
   503    503   **
   504    504   ** Costs are not modified by the ORDER BY clause.
   505    505   */
   506    506   static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
   507    507     Fts5Table *pTab = (Fts5Table*)pVTab;
   508    508     Fts5Config *pConfig = pTab->pConfig;
          509  +  const int nCol = pConfig->nCol;
   509    510     int idxFlags = 0;               /* Parameter passed through to xFilter() */
   510    511     int bHasMatch;
   511    512     int iNext;
   512    513     int i;
   513    514   
   514    515     struct Constraint {
   515    516       int op;                       /* Mask against sqlite3_index_constraint.op */
................................................................................
   527    528                                       FTS5_BI_ROWID_LE, 0, 0, -1},
   528    529       {SQLITE_INDEX_CONSTRAINT_GT|SQLITE_INDEX_CONSTRAINT_GE, 
   529    530                                       FTS5_BI_ROWID_GE, 0, 0, -1},
   530    531     };
   531    532   
   532    533     int aColMap[3];
   533    534     aColMap[0] = -1;
   534         -  aColMap[1] = pConfig->nCol;
   535         -  aColMap[2] = pConfig->nCol+1;
          535  +  aColMap[1] = nCol;
          536  +  aColMap[2] = nCol+1;
   536    537   
   537    538     /* Set idxFlags flags for all WHERE clause terms that will be used. */
   538    539     for(i=0; i<pInfo->nConstraint; i++){
   539    540       struct sqlite3_index_constraint *p = &pInfo->aConstraint[i];
   540         -    int j;
   541         -    for(j=0; j<ArraySize(aConstraint); j++){
   542         -      struct Constraint *pC = &aConstraint[j];
   543         -      if( p->iColumn==aColMap[pC->iCol] && p->op & pC->op ){
   544         -        if( p->usable ){
          541  +    int iCol = p->iColumn;
          542  +
          543  +    if( (p->op==SQLITE_INDEX_CONSTRAINT_MATCH && iCol>=0 && iCol<=nCol)
          544  +     || (p->op==SQLITE_INDEX_CONSTRAINT_EQ && iCol==nCol)
          545  +    ){
          546  +      /* A MATCH operator or equivalent */
          547  +      if( p->usable ){
          548  +        idxFlags = (idxFlags & 0xFFFF) | FTS5_BI_MATCH | (iCol << 16);
          549  +        aConstraint[0].iConsIndex = i;
          550  +      }else{
          551  +        /* As there exists an unusable MATCH constraint this is an 
          552  +        ** unusable plan. Set a prohibitively high cost. */
          553  +        pInfo->estimatedCost = 1e50;
          554  +        return SQLITE_OK;
          555  +      }
          556  +    }else{
          557  +      int j;
          558  +      for(j=1; j<ArraySize(aConstraint); j++){
          559  +        struct Constraint *pC = &aConstraint[j];
          560  +        if( iCol==aColMap[pC->iCol] && p->op & pC->op && p->usable ){
   545    561             pC->iConsIndex = i;
   546    562             idxFlags |= pC->fts5op;
   547         -        }else if( j==0 ){
   548         -          /* As there exists an unusable MATCH constraint this is an 
   549         -          ** unusable plan. Set a prohibitively high cost. */
   550         -          pInfo->estimatedCost = 1e50;
   551         -          return SQLITE_OK;
   552    563           }
   553    564         }
   554    565       }
   555    566     }
   556    567   
   557    568     /* Set idxFlags flags for the ORDER BY clause */
   558    569     if( pInfo->nOrderBy==1 ){
................................................................................
  1119   1130     int bDesc;                      /* True if ORDER BY [rank|rowid] DESC */
  1120   1131     int bOrderByRank;               /* True if ORDER BY rank */
  1121   1132     sqlite3_value *pMatch = 0;      /* <tbl> MATCH ? expression (or NULL) */
  1122   1133     sqlite3_value *pRank = 0;       /* rank MATCH ? expression (or NULL) */
  1123   1134     sqlite3_value *pRowidEq = 0;    /* rowid = ? expression (or NULL) */
  1124   1135     sqlite3_value *pRowidLe = 0;    /* rowid <= ? expression (or NULL) */
  1125   1136     sqlite3_value *pRowidGe = 0;    /* rowid >= ? expression (or NULL) */
         1137  +  int iCol;                       /* Column on LHS of MATCH operator */
  1126   1138     char **pzErrmsg = pConfig->pzErrmsg;
  1127   1139   
  1128   1140     UNUSED_PARAM(zUnused);
  1129   1141     UNUSED_PARAM(nVal);
  1130   1142   
  1131   1143     if( pCsr->ePlan ){
  1132   1144       fts5FreeCursorComponents(pCsr);
................................................................................
  1149   1161     ** order as the corresponding entries in the struct at the top of
  1150   1162     ** fts5BestIndexMethod().  */
  1151   1163     if( BitFlagTest(idxNum, FTS5_BI_MATCH) ) pMatch = apVal[iVal++];
  1152   1164     if( BitFlagTest(idxNum, FTS5_BI_RANK) ) pRank = apVal[iVal++];
  1153   1165     if( BitFlagTest(idxNum, FTS5_BI_ROWID_EQ) ) pRowidEq = apVal[iVal++];
  1154   1166     if( BitFlagTest(idxNum, FTS5_BI_ROWID_LE) ) pRowidLe = apVal[iVal++];
  1155   1167     if( BitFlagTest(idxNum, FTS5_BI_ROWID_GE) ) pRowidGe = apVal[iVal++];
         1168  +  iCol = (idxNum>>16);
         1169  +  assert( iCol>=0 && iCol<=pConfig->nCol );
  1156   1170     assert( iVal==nVal );
  1157   1171     bOrderByRank = ((idxNum & FTS5_BI_ORDER_RANK) ? 1 : 0);
  1158   1172     pCsr->bDesc = bDesc = ((idxNum & FTS5_BI_ORDER_DESC) ? 1 : 0);
  1159   1173   
  1160   1174     /* Set the cursor upper and lower rowid limits. Only some strategies 
  1161   1175     ** actually use them. This is ok, as the xBestIndex() method leaves the
  1162   1176     ** sqlite3_index_constraint.omit flag clear for range constraints
................................................................................
  1195   1209         if( zExpr[0]=='*' ){
  1196   1210           /* The user has issued a query of the form "MATCH '*...'". This
  1197   1211           ** indicates that the MATCH expression is not a full text query,
  1198   1212           ** but a request for an internal parameter.  */
  1199   1213           rc = fts5SpecialMatch(pTab, pCsr, &zExpr[1]);
  1200   1214         }else{
  1201   1215           char **pzErr = &pTab->base.zErrMsg;
  1202         -        rc = sqlite3Fts5ExprNew(pConfig, zExpr, &pCsr->pExpr, pzErr);
         1216  +        rc = sqlite3Fts5ExprNew(pConfig, iCol, zExpr, &pCsr->pExpr, pzErr);
  1203   1217           if( rc==SQLITE_OK ){
  1204   1218             if( bOrderByRank ){
  1205   1219               pCsr->ePlan = FTS5_PLAN_SORTED_MATCH;
  1206   1220               rc = fts5CursorFirstSorted(pTab, pCsr, bDesc);
  1207   1221             }else{
  1208   1222               pCsr->ePlan = FTS5_PLAN_MATCH;
  1209   1223               rc = fts5CursorFirst(pTab, pCsr, bDesc);

Changes to ext/fts5/fts5parse.y.

    85     85   %type cnearset    {Fts5ExprNode*}
    86     86   %type expr        {Fts5ExprNode*}
    87     87   %type exprlist    {Fts5ExprNode*}
    88     88   %destructor cnearset { sqlite3Fts5ParseNodeFree($$); }
    89     89   %destructor expr     { sqlite3Fts5ParseNodeFree($$); }
    90     90   %destructor exprlist { sqlite3Fts5ParseNodeFree($$); }
    91     91   
    92         -expr(A) ::= expr(X) AND expr(Y). {
    93         -  A = sqlite3Fts5ParseNode(pParse, FTS5_AND, X, Y, 0);
    94         -}
    95         -expr(A) ::= expr(X) OR expr(Y). {
    96         -  A = sqlite3Fts5ParseNode(pParse, FTS5_OR, X, Y, 0);
    97         -}
    98         -expr(A) ::= expr(X) NOT expr(Y). {
    99         -  A = sqlite3Fts5ParseNode(pParse, FTS5_NOT, X, Y, 0);
   100         -}
   101         -
   102         -expr(A) ::= LP expr(X) RP. {A = X;}
   103         -expr(A) ::= exprlist(X).   {A = X;}
   104         -
   105         -exprlist(A) ::= cnearset(X). {A = X;}
   106         -exprlist(A) ::= exprlist(X) cnearset(Y). {
   107         -  A = sqlite3Fts5ParseImplicitAnd(pParse, X, Y);
   108         -}
   109         -
   110         -cnearset(A) ::= nearset(X). { 
   111         -  A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, X); 
   112         -}
   113         -cnearset(A) ::= colset(X) COLON nearset(Y). { 
   114         -  sqlite3Fts5ParseSetColset(pParse, Y, X);
   115         -  A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, Y); 
   116         -}
   117         -
   118     92   %type colset {Fts5Colset*}
   119     93   %destructor colset { sqlite3_free($$); }
   120     94   %type colsetlist {Fts5Colset*}
   121     95   %destructor colsetlist { sqlite3_free($$); }
   122     96   
   123     97   colset(A) ::= MINUS LCP colsetlist(X) RCP. { 
   124     98       A = sqlite3Fts5ParseColsetInvert(pParse, X);
................................................................................
   133    107   }
   134    108   
   135    109   colsetlist(A) ::= colsetlist(Y) STRING(X). { 
   136    110     A = sqlite3Fts5ParseColset(pParse, Y, &X); }
   137    111   colsetlist(A) ::= STRING(X). { 
   138    112     A = sqlite3Fts5ParseColset(pParse, 0, &X); 
   139    113   }
          114  +
          115  +expr(A) ::= expr(X) AND expr(Y). {
          116  +  A = sqlite3Fts5ParseNode(pParse, FTS5_AND, X, Y, 0);
          117  +}
          118  +expr(A) ::= expr(X) OR expr(Y). {
          119  +  A = sqlite3Fts5ParseNode(pParse, FTS5_OR, X, Y, 0);
          120  +}
          121  +expr(A) ::= expr(X) NOT expr(Y). {
          122  +  A = sqlite3Fts5ParseNode(pParse, FTS5_NOT, X, Y, 0);
          123  +}
          124  +
          125  +expr(A) ::= colset(X) COLON LP expr(Y) RP. {
          126  +  sqlite3Fts5ParseSetColset(pParse, Y, X);
          127  +  A = Y;
          128  +}
          129  +expr(A) ::= LP expr(X) RP. {A = X;}
          130  +expr(A) ::= exprlist(X).   {A = X;}
          131  +
          132  +exprlist(A) ::= cnearset(X). {A = X;}
          133  +exprlist(A) ::= exprlist(X) cnearset(Y). {
          134  +  A = sqlite3Fts5ParseImplicitAnd(pParse, X, Y);
          135  +}
          136  +
          137  +cnearset(A) ::= nearset(X). { 
          138  +  A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, X); 
          139  +}
          140  +cnearset(A) ::= colset(X) COLON nearset(Y). { 
          141  +  A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, Y); 
          142  +  sqlite3Fts5ParseSetColset(pParse, A, X);
          143  +}
          144  +
   140    145   
   141    146   %type nearset     {Fts5ExprNearset*}
   142    147   %type nearphrases {Fts5ExprNearset*}
   143    148   %destructor nearset { sqlite3Fts5ParseNearsetFree($$); }
   144    149   %destructor nearphrases { sqlite3Fts5ParseNearsetFree($$); }
   145    150   
   146    151   nearset(A) ::= phrase(X). { A = sqlite3Fts5ParseNearset(pParse, 0, X); }

Changes to ext/fts5/test/fts5aa.test.

   572    572   
   573    573   do_execsql_test 21.1 {
   574    574     BEGIN;
   575    575       INSERT INTO ft VALUES('a b c');
   576    576       DROP TABLE t8;
   577    577     COMMIT;
   578    578   }
          579  +
          580  +do_execsql_test 22.0 {
          581  +  CREATE VIRTUAL TABLE t9 USING fts5(x, detail=%DETAIL%);
          582  +  INSERT INTO t9(rowid, x) VALUES(2, 'bbb');
          583  +  BEGIN;
          584  +    INSERT INTO t9(rowid, x) VALUES(1, 'aaa');
          585  +    DELETE FROM t9 WHERE rowid = 2;
          586  +    INSERT INTO t9(rowid, x) VALUES(3, 'bbb');
          587  +  COMMIT;
          588  +}
          589  +
          590  +do_execsql_test 22.1 {
          591  +  SELECT rowid FROM t9('a*')
          592  +} {1}
   579    593   
   580    594   }
   581    595   
   582    596   
   583    597   finish_test
   584    598   
   585    599   

Changes to ext/fts5/test/fts5colset.test.

    40     40       5 " - {d d c} : a" {1 2}
    41     41       6 "- {d c b a} : a" {}
    42     42       7 "-{\"a\"} : b" {1 2 3}
    43     43       8 "- c : a" {1 2 4}
    44     44       9 "-c : a"  {1 2 4}
    45     45       10 "-\"c\" : a"  {1 2 4}
    46     46     } {
    47         -  breakpoint
    48     47       do_execsql_test 1.$tn {
    49     48         SELECT rowid FROM t1($q)
    50     49       } $res
    51     50     }
    52     51   
           52  +  foreach {tn q res} {
           53  +    0 {{a} : (a AND ":")}     {}
           54  +    1 "{a b c} : (a AND d)"   {2 3}
           55  +    2 "{a b c} : (a AND b:d)" {3}
           56  +    3 "{a b c} : (a AND d:d)" {}
           57  +    4 "{b} : ( {b a} : ( {c b a} : ( {d b c a} : ( d OR c ) ) ) )" {3 4}
           58  +    5 "{a} : ( {b a} : ( {c b a} : ( {d b c a} : ( d OR c ) ) ) )" {2 3}
           59  +    6 "{a} : ( {b a} : ( {c b} : ( {d b c a} : ( d OR c ) ) ) )" {}
           60  +    7 "{a b c} : (b:a AND c:b)" {2}
           61  +  } {
           62  +    do_execsql_test 2.$tn {
           63  +      SELECT rowid FROM t1($q)
           64  +    } $res
           65  +  }
           66  +
           67  +  foreach {tn w res} {
           68  +    0 "a MATCH 'a'" {1}
           69  +    1 "b MATCH 'a'" {2}
           70  +    2 "b MATCH '{a b c} : a'" {2}
           71  +    3 "b MATCH 'a OR b'"      {1 2}
           72  +    4 "b MATCH 'a OR a:b'"    {2}
           73  +    5 "b MATCH 'a OR b:b'"    {1 2}
           74  +  } {
           75  +    do_execsql_test 3.$tn "
           76  +      SELECT rowid FROM t1 WHERE $w
           77  +    " $res
           78  +  }
    53     79   
           80  +  do_catchsql_test 4.1 {
           81  +    SELECT * FROM t1 WHERE rowid MATCH 'a'
           82  +  } {1 {unable to use function MATCH in the requested context}}
    54     83   }
    55     84   
    56     85   
    57     86   finish_test
    58     87   
    59     88   

Changes to ext/fts5/test/fts5faultB.test.

   102    102   do_faultsim_test 3.3 -faults oom* -body {
   103    103     execsql {
   104    104       SELECT rowid FROM x1('c') WHERE rowid>1;
   105    105     }
   106    106   } -test {
   107    107     faultsim_test_result {0 {2 3}}
   108    108   }
          109  +
          110  +#-------------------------------------------------------------------------
          111  +# Test OOM injection with nested colsets.
          112  +#
          113  +reset_db
          114  +do_execsql_test 4.0 {
          115  +  CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, d);
          116  +  INSERT INTO t1 VALUES('a', 'b', 'c', 'd');  -- 1
          117  +  INSERT INTO t1 VALUES('d', 'a', 'b', 'c');  -- 2
          118  +  INSERT INTO t1 VALUES('c', 'd', 'a', 'b');  -- 3
          119  +  INSERT INTO t1 VALUES('b', 'c', 'd', 'a');  -- 4
          120  +}
          121  +do_faultsim_test 4.1 -faults oom* -body {
          122  +  execsql { SELECT rowid FROM t1('{a b c} : (b:a AND c:b)'); }
          123  +} -test {
          124  +  faultsim_test_result {0 2}
          125  +}
          126  +
          127  +do_faultsim_test 4.2 -faults oom* -body {
          128  +  execsql { SELECT rowid FROM t1('{a b c} : (a AND d)') }
          129  +} -test {
          130  +  faultsim_test_result {0 {2 3}}
          131  +}
          132  +
   109    133   
   110    134   finish_test
   111    135   

Changes to ext/fts5/test/fts5plan.test.

    26     26     CREATE VIRTUAL TABLE f1 USING fts5(ff);
    27     27   }
    28     28   
    29     29   do_eqp_test 1.1 {
    30     30     SELECT * FROM t1, f1 WHERE f1 MATCH t1.x
    31     31   } {
    32     32     0 0 0 {SCAN TABLE t1} 
    33         -  0 1 1 {SCAN TABLE f1 VIRTUAL TABLE INDEX 1:}
           33  +  0 1 1 {SCAN TABLE f1 VIRTUAL TABLE INDEX 65537:}
    34     34   }
    35     35   
    36     36   do_eqp_test 1.2 {
    37     37     SELECT * FROM t1, f1 WHERE f1 > t1.x
    38     38   } {
    39     39     0 0 1 {SCAN TABLE f1 VIRTUAL TABLE INDEX 0:}
    40     40     0 1 0 {SCAN TABLE t1} 
    41     41   }
    42     42   
    43     43   do_eqp_test 1.3 {
    44     44     SELECT * FROM f1 WHERE f1 MATCH ? ORDER BY ff
    45     45   } {
    46         -  0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 1:}
           46  +  0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 65537:}
    47     47     0 0 0 {USE TEMP B-TREE FOR ORDER BY}
    48     48   }
    49     49   
    50     50   do_eqp_test 1.4 {
    51     51     SELECT * FROM f1 ORDER BY rank
    52     52   } {
    53     53     0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 0:}

Added ext/misc/anycollseq.c.

            1  +/*
            2  +** 2017-04-16
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +*************************************************************************
           12  +**
           13  +** This file implements a run-time loadable extension to SQLite that
           14  +** registers a sqlite3_collation_needed() callback to register a fake
           15  +** collating function for any unknown collating sequence.  The fake
           16  +** collating function works like BINARY.
           17  +**
           18  +** This extension can be used to load schemas that contain one or more
           19  +** unknown collating sequences.
           20  +*/
           21  +#include "sqlite3ext.h"
           22  +SQLITE_EXTENSION_INIT1
           23  +#include <string.h>
           24  +
           25  +static int anyCollFunc(
           26  +  void *NotUsed,
           27  +  int nKey1, const void *pKey1,
           28  +  int nKey2, const void *pKey2
           29  +){
           30  +  int rc, n;
           31  +  n = nKey1<nKey2 ? nKey1 : nKey2;
           32  +  rc = memcmp(pKey1, pKey2, n);
           33  +  if( rc==0 ) rc = nKey1 - nKey2;
           34  +  return rc;
           35  +}
           36  +
           37  +static void anyCollNeeded(
           38  +  void *NotUsed,
           39  +  sqlite3 *db,
           40  +  int eTextRep,
           41  +  const char *zCollName
           42  +){
           43  +  sqlite3_create_collation(db, zCollName, eTextRep, 0, anyCollFunc); 
           44  +}
           45  +
           46  +#ifdef _WIN32
           47  +__declspec(dllexport)
           48  +#endif
           49  +int sqlite3_anycollseq_init(
           50  +  sqlite3 *db, 
           51  +  char **pzErrMsg, 
           52  +  const sqlite3_api_routines *pApi
           53  +){
           54  +  int rc = SQLITE_OK;
           55  +  SQLITE_EXTENSION_INIT2(pApi);
           56  +  rc = sqlite3_collation_needed(db, 0, anyCollNeeded);
           57  +  return rc;
           58  +}

Changes to ext/misc/dbdump.c.

   320    320     z = sqlite3_vmprintf(zFormat, ap);
   321    321     va_end(ap);
   322    322     p->xCallback(z, p->pArg);
   323    323     sqlite3_free(z);
   324    324   }
   325    325   
   326    326   /*
   327         -** Output the given string as a quoted string using SQL quoting conventions.
          327  +** Find a string that is not found anywhere in z[].  Return a pointer
          328  +** to that string.
   328    329   **
   329         -** The "\n" and "\r" characters are converted to char(10) and char(13)
   330         -** to prevent them from being transformed by end-of-line translators.
          330  +** Try to use zA and zB first.  If both of those are already found in z[]
          331  +** then make up some string and store it in the buffer zBuf.
          332  +*/
          333  +static const char *unused_string(
          334  +  const char *z,                    /* Result must not appear anywhere in z */
          335  +  const char *zA, const char *zB,   /* Try these first */
          336  +  char *zBuf                        /* Space to store a generated string */
          337  +){
          338  +  unsigned i = 0;
          339  +  if( strstr(z, zA)==0 ) return zA;
          340  +  if( strstr(z, zB)==0 ) return zB;
          341  +  do{
          342  +    sqlite3_snprintf(20,zBuf,"(%s%u)", zA, i++);
          343  +  }while( strstr(z,zBuf)!=0 );
          344  +  return zBuf;
          345  +}
          346  +
          347  +/*
          348  +** Output the given string as a quoted string using SQL quoting conventions.
          349  +** Additionallly , escape the "\n" and "\r" characters so that they do not
          350  +** get corrupted by end-of-line translation facilities in some operating
          351  +** systems.
   331    352   */
   332         -static void output_quoted_string(DState *p, const unsigned char *z){
          353  +static void output_quoted_escaped_string(DState *p, const char *z){
   333    354     int i;
   334    355     char c;
   335         -  int inQuote = 0;
   336         -  int bStarted = 0;
   337         -
   338    356     for(i=0; (c = z[i])!=0 && c!='\'' && c!='\n' && c!='\r'; i++){}
   339    357     if( c==0 ){
   340         -    output_formatted(p, "'%s'", z);
   341         -    return;
   342         -  }
   343         -  while( *z ){
   344         -    for(i=0; (c = z[i])!=0 && c!='\n' && c!='\r' && c!='\''; i++){}
   345         -    if( c=='\'' ) i++;
   346         -    if( i ){
   347         -      if( !inQuote ){
   348         -        if( bStarted ) p->xCallback("||", p->pArg);
          358  +    output_formatted(p,"'%s'",z);
          359  +  }else{
          360  +    const char *zNL = 0;
          361  +    const char *zCR = 0;
          362  +    int nNL = 0;
          363  +    int nCR = 0;
          364  +    char zBuf1[20], zBuf2[20];
          365  +    for(i=0; z[i]; i++){
          366  +      if( z[i]=='\n' ) nNL++;
          367  +      if( z[i]=='\r' ) nCR++;
          368  +    }
          369  +    if( nNL ){
          370  +      p->xCallback("replace(", p->pArg);
          371  +      zNL = unused_string(z, "\\n", "\\012", zBuf1);
          372  +    }
          373  +    if( nCR ){
          374  +      p->xCallback("replace(", p->pArg);
          375  +      zCR = unused_string(z, "\\r", "\\015", zBuf2);
          376  +    }
          377  +    p->xCallback("'", p->pArg);
          378  +    while( *z ){
          379  +      for(i=0; (c = z[i])!=0 && c!='\n' && c!='\r' && c!='\''; i++){}
          380  +      if( c=='\'' ) i++;
          381  +      if( i ){
          382  +        output_formatted(p, "%.*s", i, z);
          383  +        z += i;
          384  +      }
          385  +      if( c=='\'' ){
   349    386           p->xCallback("'", p->pArg);
   350         -        inQuote = 1;
          387  +        continue;
          388  +      }
          389  +      if( c==0 ){
          390  +        break;
          391  +      }
          392  +      z++;
          393  +      if( c=='\n' ){
          394  +        p->xCallback(zNL, p->pArg);
          395  +        continue;
   351    396         }
   352         -      output_formatted(p, "%.*s", i, z);
   353         -      z += i;
   354         -      bStarted = 1;
          397  +      p->xCallback(zCR, p->pArg);
   355    398       }
   356         -    if( c=='\'' ){
   357         -      p->xCallback("'", p->pArg);
   358         -      continue;
          399  +    p->xCallback("'", p->pArg);
          400  +    if( nCR ){
          401  +      output_formatted(p, ",'%s',char(13))", zCR);
   359    402       }
   360         -    if( inQuote ){
   361         -      p->xCallback("'", p->pArg);
   362         -      inQuote = 0;
          403  +    if( nNL ){
          404  +      output_formatted(p, ",'%s',char(10))", zNL);
   363    405       }
   364         -    if( c==0 ){
   365         -      break;
   366         -    }
   367         -    for(i=0; (c = z[i])=='\r' || c=='\n'; i++){
   368         -      if( bStarted ) p->xCallback("||", p->pArg);
   369         -      output_formatted(p, "char(%d)", c);
   370         -      bStarted = 1;
   371         -    }
   372         -    z += i;
   373    406     }
   374         -  if( inQuote ) p->xCallback("'", p->pArg);
   375    407   }
   376    408   
   377    409   /*
   378    410   ** This is an sqlite3_exec callback routine used for dumping the database.
   379    411   ** Each row received by this callback consists of a table name,
   380    412   ** the table type ("index" or "table") and SQL to create the table.
   381    413   ** This routine should print text sufficient to recreate the table.
................................................................................
   491    523                 break;
   492    524               }
   493    525               case SQLITE_NULL: {
   494    526                 p->xCallback("NULL", p->pArg);
   495    527                 break;
   496    528               }
   497    529               case SQLITE_TEXT: {
   498         -              output_quoted_string(p, sqlite3_column_text(pStmt,i));
          530  +              output_quoted_escaped_string(p, 
          531  +                   (const char*)sqlite3_column_text(pStmt,i));
   499    532                 break;
   500    533               }
   501    534               case SQLITE_BLOB: {
   502    535                 int nByte = sqlite3_column_bytes(pStmt,i);
   503    536                 unsigned char *a = (unsigned char*)sqlite3_column_blob(pStmt,i);
   504    537                 int j;
   505    538                 p->xCallback("x'", p->pArg);

Changes to ext/misc/json1.c.

    86     86   #define safe_isspace(x) (jsonIsSpace[(unsigned char)x])
    87     87   
    88     88   #ifndef SQLITE_AMALGAMATION
    89     89     /* Unsigned integer types.  These are already defined in the sqliteInt.h,
    90     90     ** but the definitions need to be repeated for separate compilation. */
    91     91     typedef sqlite3_uint64 u64;
    92     92     typedef unsigned int u32;
           93  +  typedef unsigned short int u16;
    93     94     typedef unsigned char u8;
    94     95   #endif
    95     96   
    96     97   /* Objects */
    97     98   typedef struct JsonString JsonString;
    98     99   typedef struct JsonNode JsonNode;
    99    100   typedef struct JsonParse JsonParse;
................................................................................
   165    166     u32 nNode;         /* Number of slots of aNode[] used */
   166    167     u32 nAlloc;        /* Number of slots of aNode[] allocated */
   167    168     JsonNode *aNode;   /* Array of nodes containing the parse */
   168    169     const char *zJson; /* Original JSON string */
   169    170     u32 *aUp;          /* Index of parent of each node */
   170    171     u8 oom;            /* Set to true if out of memory */
   171    172     u8 nErr;           /* Number of errors seen */
          173  +  u16 iDepth;        /* Nesting depth */
   172    174   };
   173    175   
          176  +/*
          177  +** Maximum nesting depth of JSON for this implementation.
          178  +**
          179  +** This limit is needed to avoid a stack overflow in the recursive
          180  +** descent parser.  A depth of 2000 is far deeper than any sane JSON
          181  +** should go.
          182  +*/
          183  +#define JSON_MAX_DEPTH  2000
          184  +
   174    185   /**************************************************************************
   175    186   ** Utility routines for dealing with JsonString objects
   176    187   **************************************************************************/
   177    188   
   178    189   /* Set the JsonString object to an empty string
   179    190   */
   180    191   static void jsonZero(JsonString *p){
................................................................................
   731    742     while( safe_isspace(z[i]) ){ i++; }
   732    743     if( (c = z[i])=='{' ){
   733    744       /* Parse object */
   734    745       iThis = jsonParseAddNode(pParse, JSON_OBJECT, 0, 0);
   735    746       if( iThis<0 ) return -1;
   736    747       for(j=i+1;;j++){
   737    748         while( safe_isspace(z[j]) ){ j++; }
          749  +      if( ++pParse->iDepth > JSON_MAX_DEPTH ) return -1;
   738    750         x = jsonParseValue(pParse, j);
   739    751         if( x<0 ){
          752  +        pParse->iDepth--;
   740    753           if( x==(-2) && pParse->nNode==(u32)iThis+1 ) return j+1;
   741    754           return -1;
   742    755         }
   743    756         if( pParse->oom ) return -1;
   744    757         pNode = &pParse->aNode[pParse->nNode-1];
   745    758         if( pNode->eType!=JSON_STRING ) return -1;
   746    759         pNode->jnFlags |= JNODE_LABEL;
   747    760         j = x;
   748    761         while( safe_isspace(z[j]) ){ j++; }
   749    762         if( z[j]!=':' ) return -1;
   750    763         j++;
   751    764         x = jsonParseValue(pParse, j);
          765  +      pParse->iDepth--;
   752    766         if( x<0 ) return -1;
   753    767         j = x;
   754    768         while( safe_isspace(z[j]) ){ j++; }
   755    769         c = z[j];
   756    770         if( c==',' ) continue;
   757    771         if( c!='}' ) return -1;
   758    772         break;
................................................................................
   761    775       return j+1;
   762    776     }else if( c=='[' ){
   763    777       /* Parse array */
   764    778       iThis = jsonParseAddNode(pParse, JSON_ARRAY, 0, 0);
   765    779       if( iThis<0 ) return -1;
   766    780       for(j=i+1;;j++){
   767    781         while( safe_isspace(z[j]) ){ j++; }
          782  +      if( ++pParse->iDepth > JSON_MAX_DEPTH ) return -1;
   768    783         x = jsonParseValue(pParse, j);
          784  +      pParse->iDepth--;
   769    785         if( x<0 ){
   770    786           if( x==(-3) && pParse->nNode==(u32)iThis+1 ) return j+1;
   771    787           return -1;
   772    788         }
   773    789         j = x;
   774    790         while( safe_isspace(z[j]) ){ j++; }
   775    791         c = z[j];
................................................................................
   781    797       return j+1;
   782    798     }else if( c=='"' ){
   783    799       /* Parse string */
   784    800       u8 jnFlags = 0;
   785    801       j = i+1;
   786    802       for(;;){
   787    803         c = z[j];
   788         -      if( c==0 ) return -1;
          804  +      if( (c & ~0x1f)==0 ){
          805  +        /* Control characters are not allowed in strings */
          806  +        return -1;
          807  +      }
   789    808         if( c=='\\' ){
   790    809           c = z[++j];
   791    810           if( c=='"' || c=='\\' || c=='/' || c=='b' || c=='f'
   792    811              || c=='n' || c=='r' || c=='t'
   793    812              || (c=='u' && jsonIs4Hex(z+j+1)) ){
   794    813             jnFlags = JNODE_ESCAPE;
   795    814           }else{
................................................................................
   881    900     int i;
   882    901     memset(pParse, 0, sizeof(*pParse));
   883    902     if( zJson==0 ) return 1;
   884    903     pParse->zJson = zJson;
   885    904     i = jsonParseValue(pParse, 0);
   886    905     if( pParse->oom ) i = -1;
   887    906     if( i>0 ){
          907  +    assert( pParse->iDepth==0 );
   888    908       while( safe_isspace(zJson[i]) ) i++;
   889    909       if( zJson[i] ) i = -1;
   890    910     }
   891    911     if( i<=0 ){
   892    912       if( pCtx!=0 ){
   893    913         if( pParse->oom ){
   894    914           sqlite3_result_error_nomem(pCtx);
................................................................................
  1381   1401     jsonParseReset(&x);
  1382   1402   }
  1383   1403   
  1384   1404   /* This is the RFC 7396 MergePatch algorithm.
  1385   1405   */
  1386   1406   static JsonNode *jsonMergePatch(
  1387   1407     JsonParse *pParse,   /* The JSON parser that contains the TARGET */
  1388         -  int iTarget,         /* Node of the TARGET in pParse */
         1408  +  u32 iTarget,         /* Node of the TARGET in pParse */
  1389   1409     JsonNode *pPatch     /* The PATCH */
  1390   1410   ){
  1391   1411     u32 i, j;
  1392   1412     u32 iRoot;
  1393   1413     JsonNode *pTarget;
  1394   1414     if( pPatch->eType!=JSON_OBJECT ){
  1395   1415       return pPatch;

Changes to ext/rbu/rbu.c.

    78     78     char zBuf[200];                 /* Buffer for printf() */
    79     79     char *zErrmsg;                  /* Error message, if any */
    80     80     sqlite3rbu *pRbu;               /* RBU handle */
    81     81     int nStep = 0;                  /* Maximum number of step() calls */
    82     82     int bVacuum = 0;
    83     83     int rc;
    84     84     sqlite3_int64 nProgress = 0;
    85         -  int nArg = argc-2;
           85  +  int nArgc = argc-2;
    86     86   
    87     87     if( argc<3 ) usage(argv[0]);
    88         -  for(i=1; i<nArg; i++){
           88  +  for(i=1; i<nArgc; i++){
    89     89       const char *zArg = argv[i];
    90     90       int nArg = strlen(zArg);
    91     91       if( nArg>1 && nArg<=8 && 0==memcmp(zArg, "-vacuum", nArg) ){
    92     92         bVacuum = 1;
    93     93       }else if( nArg>1 && nArg<=5 && 0==memcmp(zArg, "-step", nArg) && i<nArg-1 ){
    94     94         i++;
    95     95         nStep = atoi(argv[i]);

Changes to src/btree.c.

  8126   8126               ((pCur->curFlags&BTCF_ValidNKey)!=0 && pX->nKey==pCur->info.nKey) );
  8127   8127   
  8128   8128       /* If the cursor is currently on the last row and we are appending a
  8129   8129       ** new row onto the end, set the "loc" to avoid an unnecessary
  8130   8130       ** btreeMoveto() call */
  8131   8131       if( (pCur->curFlags&BTCF_ValidNKey)!=0 && pX->nKey==pCur->info.nKey ){
  8132   8132         loc = 0;
  8133         -    }else if( (pCur->curFlags&BTCF_ValidNKey)!=0 && pX->nKey>0
  8134         -               && pCur->info.nKey==pX->nKey-1 ){
  8135         -      loc = -1;
  8136   8133       }else if( loc==0 ){
  8137   8134         rc = sqlite3BtreeMovetoUnpacked(pCur, 0, pX->nKey, flags!=0, &loc);
  8138   8135         if( rc ) return rc;
  8139   8136       }
  8140   8137     }else if( loc==0 && (flags & BTREE_SAVEPOSITION)==0 ){
  8141   8138       if( pX->nMem ){
  8142   8139         UnpackedRecord r;

Changes to src/btree.h.

   272    272   ** organized and understandable, and it also helps the resulting code to
   273    273   ** run a little faster by using fewer registers for parameter passing.
   274    274   */
   275    275   struct BtreePayload {
   276    276     const void *pKey;       /* Key content for indexes.  NULL for tables */
   277    277     sqlite3_int64 nKey;     /* Size of pKey for indexes.  PRIMARY KEY for tabs */
   278    278     const void *pData;      /* Data for tables.  NULL for indexes */
   279         -  struct Mem *aMem;       /* First of nMem value in the unpacked pKey */
          279  +  sqlite3_value *aMem;    /* First of nMem value in the unpacked pKey */
   280    280     u16 nMem;               /* Number of aMem[] value.  Might be zero */
   281    281     int nData;              /* Size of pData.  0 if none. */
   282    282     int nZero;              /* Extra zero data appended after pData,nData */
   283    283   };
   284    284   
   285    285   int sqlite3BtreeInsert(BtCursor*, const BtreePayload *pPayload,
   286    286                          int flags, int seekResult);

Changes to src/date.c.

   419    419   static void computeYMD(DateTime *p){
   420    420     int Z, A, B, C, D, E, X1;
   421    421     if( p->validYMD ) return;
   422    422     if( !p->validJD ){
   423    423       p->Y = 2000;
   424    424       p->M = 1;
   425    425       p->D = 1;
          426  +  }else if( !validJulianDay(p->iJD) ){
          427  +    datetimeError(p);
          428  +    return;
   426    429     }else{
   427         -    assert( validJulianDay(p->iJD) );
   428    430       Z = (int)((p->iJD + 43200000)/86400000);
   429    431       A = (int)((Z - 1867216.25)/36524.25);
   430    432       A = Z + 1 + A - (A/4);
   431    433       B = A + 1524;
   432    434       C = (int)((B - 122.1)/365.25);
   433    435       D = (36525*(C&32767))/100;
   434    436       E = (int)((B-D)/30.6001);

Changes to src/expr.c.

    54     54     if( op==TK_REGISTER ) op = pExpr->op2;
    55     55   #ifndef SQLITE_OMIT_CAST
    56     56     if( op==TK_CAST ){
    57     57       assert( !ExprHasProperty(pExpr, EP_IntValue) );
    58     58       return sqlite3AffinityType(pExpr->u.zToken, 0);
    59     59     }
    60     60   #endif
    61         -  if( op==TK_AGG_COLUMN || op==TK_COLUMN ){
           61  +  if( (op==TK_AGG_COLUMN || op==TK_COLUMN) && pExpr->pTab ){
    62     62       return sqlite3TableColumnAffinity(pExpr->pTab, pExpr->iColumn);
    63     63     }
    64     64     if( op==TK_SELECT_COLUMN ){
    65     65       assert( pExpr->pLeft->flags&EP_xIsSelect );
    66     66       return sqlite3ExprAffinity(
    67     67           pExpr->pLeft->x.pSelect->pEList->a[pExpr->iColumn].pExpr
    68     68       );
................................................................................
   952    952     ynVar x;
   953    953   
   954    954     if( pExpr==0 ) return;
   955    955     assert( !ExprHasProperty(pExpr, EP_IntValue|EP_Reduced|EP_TokenOnly) );
   956    956     z = pExpr->u.zToken;
   957    957     assert( z!=0 );
   958    958     assert( z[0]!=0 );
   959         -  assert( n==sqlite3Strlen30(z) );
          959  +  assert( n==(u32)sqlite3Strlen30(z) );
   960    960     if( z[1]==0 ){
   961    961       /* Wildcard of the form "?".  Assign the next variable number */
   962    962       assert( z[0]=='?' );
   963    963       x = (ynVar)(++pParse->nVar);
   964    964     }else{
   965    965       int doAdd = 0;
   966    966       if( z[0]=='?' ){
................................................................................
  3188   3188   void sqlite3ExprCodeGetColumnOfTable(
  3189   3189     Vdbe *v,        /* The VDBE under construction */
  3190   3190     Table *pTab,    /* The table containing the value */
  3191   3191     int iTabCur,    /* The table cursor.  Or the PK cursor for WITHOUT ROWID */
  3192   3192     int iCol,       /* Index of the column to extract */
  3193   3193     int regOut      /* Extract the value into this register */
  3194   3194   ){
         3195  +  if( pTab==0 ){
         3196  +    sqlite3VdbeAddOp3(v, OP_Column, iTabCur, iCol, regOut);
         3197  +    return;
         3198  +  }
  3195   3199     if( iCol<0 || iCol==pTab->iPKey ){
  3196   3200       sqlite3VdbeAddOp2(v, OP_Rowid, iTabCur, regOut);
  3197   3201     }else{
  3198   3202       int op = IsVirtual(pTab) ? OP_VColumn : OP_Column;
  3199   3203       int x = iCol;
  3200   3204       if( !HasRowid(pTab) && !IsVirtual(pTab) ){
  3201   3205         x = sqlite3ColumnOfIndex(sqlite3PrimaryKeyIndex(pTab), iCol);
................................................................................
  3880   3884         break;
  3881   3885       }
  3882   3886   
  3883   3887       case TK_VECTOR: {
  3884   3888         sqlite3ErrorMsg(pParse, "row value misused");
  3885   3889         break;
  3886   3890       }
         3891  +
         3892  +    case TK_IF_NULL_ROW: {
         3893  +      int addrINR;
         3894  +      addrINR = sqlite3VdbeAddOp1(v, OP_IfNullRow, pExpr->iTable);
         3895  +      sqlite3ExprCachePush(pParse);
         3896  +      inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target);
         3897  +      sqlite3ExprCachePop(pParse);
         3898  +      sqlite3VdbeJumpHere(v, addrINR);
         3899  +      sqlite3VdbeChangeP3(v, addrINR, inReg);
         3900  +      break;
         3901  +    }
  3887   3902   
  3888   3903       /*
  3889   3904       ** Form A:
  3890   3905       **   CASE x WHEN e1 THEN r1 WHEN e2 THEN r2 ... WHEN eN THEN rN ELSE y END
  3891   3906       **
  3892   3907       ** Form B:
  3893   3908       **   CASE WHEN e1 THEN r1 WHEN e2 THEN r2 ... WHEN eN THEN rN ELSE y END

Changes to src/fkey.c.

  1083   1083   ** to an array of size N, where N is the number of columns in table pTab.
  1084   1084   ** If the i'th column is not modified by the UPDATE, then the corresponding 
  1085   1085   ** entry in the aChange[] array is set to -1. If the column is modified,
  1086   1086   ** the value is 0 or greater. Parameter chngRowid is set to true if the
  1087   1087   ** UPDATE statement modifies the rowid fields of the table.
  1088   1088   **
  1089   1089   ** If any foreign key processing will be required, this function returns
  1090         -** true. If there is no foreign key related processing, this function 
  1091         -** returns false.
         1090  +** non-zero. If there is no foreign key related processing, this function 
         1091  +** returns zero.
         1092  +**
         1093  +** For an UPDATE, this function returns 2 if:
         1094  +**
         1095  +**   * There are any FKs for which pTab is the child and the parent table, or
         1096  +**   * the UPDATE modifies one or more parent keys for which the action is
         1097  +**     not "NO ACTION" (i.e. is CASCADE, SET DEFAULT or SET NULL).
         1098  +**
         1099  +** Or, assuming some other foreign key processing is required, 1.
  1092   1100   */
  1093   1101   int sqlite3FkRequired(
  1094   1102     Parse *pParse,                  /* Parse context */
  1095   1103     Table *pTab,                    /* Table being modified */
  1096   1104     int *aChange,                   /* Non-NULL for UPDATE operations */
  1097   1105     int chngRowid                   /* True for UPDATE that affects rowid */
  1098   1106   ){
         1107  +  int eRet = 0;
  1099   1108     if( pParse->db->flags&SQLITE_ForeignKeys ){
  1100   1109       if( !aChange ){
  1101   1110         /* A DELETE operation. Foreign key processing is required if the 
  1102   1111         ** table in question is either the child or parent table for any 
  1103   1112         ** foreign key constraint.  */
  1104         -      return (sqlite3FkReferences(pTab) || pTab->pFKey);
         1113  +      eRet = (sqlite3FkReferences(pTab) || pTab->pFKey);
  1105   1114       }else{
  1106   1115         /* This is an UPDATE. Foreign key processing is only required if the
  1107   1116         ** operation modifies one or more child or parent key columns. */
  1108   1117         FKey *p;
  1109   1118   
  1110   1119         /* Check if any child key columns are being modified. */
  1111   1120         for(p=pTab->pFKey; p; p=p->pNextFrom){
  1112         -        if( fkChildIsModified(pTab, p, aChange, chngRowid) ) return 1;
         1121  +        if( 0==sqlite3_stricmp(pTab->zName, p->zTo) ) return 2;
         1122  +        if( fkChildIsModified(pTab, p, aChange, chngRowid) ){
         1123  +          eRet = 1;
         1124  +        }
  1113   1125         }
  1114   1126   
  1115   1127         /* Check if any parent key columns are being modified. */
  1116   1128         for(p=sqlite3FkReferences(pTab); p; p=p->pNextTo){
  1117         -        if( fkParentIsModified(pTab, p, aChange, chngRowid) ) return 1;
         1129  +        if( fkParentIsModified(pTab, p, aChange, chngRowid) ){
         1130  +          if( p->aAction[1]!=OE_None ) return 2;
         1131  +          eRet = 1;
         1132  +        }
  1118   1133         }
  1119   1134       }
  1120   1135     }
  1121         -  return 0;
         1136  +  return eRet;
  1122   1137   }
  1123   1138   
  1124   1139   /*
  1125   1140   ** This function is called when an UPDATE or DELETE operation is being 
  1126   1141   ** compiled on table pTab, which is the parent table of foreign-key pFKey.
  1127   1142   ** If the current operation is an UPDATE, then the pChanges parameter is
  1128   1143   ** passed a pointer to the list of columns being modified. If it is a

Changes to src/parse.y.

   188    188       A = 0;
   189    189       sqlite3ErrorMsg(pParse, "unknown table option: %.*s", X.n, X.z);
   190    190     }
   191    191   }
   192    192   columnlist ::= columnlist COMMA columnname carglist.
   193    193   columnlist ::= columnname carglist.
   194    194   columnname(A) ::= nm(A) typetoken(Y). {sqlite3AddColumn(pParse,&A,&Y);}
          195  +
          196  +// The following directive causes tokens ABORT, AFTER, ASC, etc. to
          197  +// fallback to ID if they will not parse as their original value.
          198  +// This obviates the need for the "id" nonterminal.
          199  +//
          200  +%fallback ID
          201  +  ABORT ACTION AFTER ANALYZE ASC ATTACH BEFORE BEGIN BY CASCADE CAST COLUMNKW
          202  +  CONFLICT DATABASE DEFERRED DESC DETACH EACH END EXCLUSIVE EXPLAIN FAIL FOR
          203  +  IGNORE IMMEDIATE INITIALLY INSTEAD LIKE_KW MATCH NO PLAN
          204  +  QUERY KEY OF OFFSET PRAGMA RAISE RECURSIVE RELEASE REPLACE RESTRICT ROW
          205  +  ROLLBACK SAVEPOINT TEMP TRIGGER VACUUM VIEW VIRTUAL WITH WITHOUT
          206  +%ifdef SQLITE_OMIT_COMPOUND_SELECT
          207  +  EXCEPT INTERSECT UNION
          208  +%endif SQLITE_OMIT_COMPOUND_SELECT
          209  +  REINDEX RENAME CTIME_KW IF
          210  +  .
          211  +%wildcard ANY.
   195    212   
   196    213   // Define operator precedence early so that this is the first occurrence
   197    214   // of the operator tokens in the grammer.  Keeping the operators together
   198    215   // causes them to be assigned integer values that are close together,
   199    216   // which keeps parser tables smaller.
   200    217   //
   201    218   // The token values assigned to these symbols is determined by the order
................................................................................
   218    235   %right BITNOT.
   219    236   
   220    237   // An IDENTIFIER can be a generic identifier, or one of several
   221    238   // keywords.  Any non-standard keyword can also be an identifier.
   222    239   //
   223    240   %token_class id  ID|INDEXED.
   224    241   
   225         -// The following directive causes tokens ABORT, AFTER, ASC, etc. to
   226         -// fallback to ID if they will not parse as their original value.
   227         -// This obviates the need for the "id" nonterminal.
   228         -//
   229         -%fallback ID
   230         -  ABORT ACTION AFTER ANALYZE ASC ATTACH BEFORE BEGIN BY CASCADE CAST COLUMNKW
   231         -  CONFLICT DATABASE DEFERRED DESC DETACH EACH END EXCLUSIVE EXPLAIN FAIL FOR
   232         -  IGNORE IMMEDIATE INITIALLY INSTEAD LIKE_KW MATCH NO PLAN
   233         -  QUERY KEY OF OFFSET PRAGMA RAISE RECURSIVE RELEASE REPLACE RESTRICT ROW
   234         -  ROLLBACK SAVEPOINT TEMP TRIGGER VACUUM VIEW VIRTUAL WITH WITHOUT
   235         -%ifdef SQLITE_OMIT_COMPOUND_SELECT
   236         -  EXCEPT INTERSECT UNION
   237         -%endif SQLITE_OMIT_COMPOUND_SELECT
   238         -  REINDEX RENAME CTIME_KW IF
   239         -  .
   240         -%wildcard ANY.
   241         -
   242    242   
   243    243   // And "ids" is an identifer-or-string.
   244    244   //
   245    245   %token_class ids  ID|STRING.
   246    246   
   247    247   // The name of a column or table can be any of the following:
   248    248   //

Changes to src/pragma.c.

  1325   1325           pIdx = 0;
  1326   1326           aiCols = 0;
  1327   1327           if( pParent ){
  1328   1328             x = sqlite3FkLocateIndex(pParse, pParent, pFK, &pIdx, &aiCols);
  1329   1329             assert( x==0 );
  1330   1330           }
  1331   1331           addrOk = sqlite3VdbeMakeLabel(v);
  1332         -        if( pParent && pIdx==0 ){
  1333         -          int iKey = pFK->aCol[0].iFrom;
  1334         -          assert( iKey>=0 && iKey<pTab->nCol );
  1335         -          if( iKey!=pTab->iPKey ){
  1336         -            sqlite3VdbeAddOp3(v, OP_Column, 0, iKey, regRow);
  1337         -            sqlite3ColumnDefault(v, pTab, iKey, regRow);
  1338         -            sqlite3VdbeAddOp2(v, OP_IsNull, regRow, addrOk); VdbeCoverage(v);
  1339         -          }else{
  1340         -            sqlite3VdbeAddOp2(v, OP_Rowid, 0, regRow);
  1341         -          }
  1342         -          sqlite3VdbeAddOp3(v, OP_SeekRowid, i, 0, regRow); VdbeCoverage(v);
         1332  +
         1333  +        /* Generate code to read the child key values into registers
         1334  +        ** regRow..regRow+n. If any of the child key values are NULL, this 
         1335  +        ** row cannot cause an FK violation. Jump directly to addrOk in 
         1336  +        ** this case. */
         1337  +        for(j=0; j<pFK->nCol; j++){
         1338  +          int iCol = aiCols ? aiCols[j] : pFK->aCol[j].iFrom;
         1339  +          sqlite3ExprCodeGetColumnOfTable(v, pTab, 0, iCol, regRow+j);
         1340  +          sqlite3VdbeAddOp2(v, OP_IsNull, regRow+j, addrOk); VdbeCoverage(v);
         1341  +        }
         1342  +
         1343  +        /* Generate code to query the parent index for a matching parent
         1344  +        ** key. If a match is found, jump to addrOk. */
         1345  +        if( pIdx ){
         1346  +          sqlite3VdbeAddOp4(v, OP_MakeRecord, regRow, pFK->nCol, regKey,
         1347  +              sqlite3IndexAffinityStr(db,pIdx), pFK->nCol);
         1348  +          sqlite3VdbeAddOp4Int(v, OP_Found, i, addrOk, regKey, 0);
         1349  +          VdbeCoverage(v);
         1350  +        }else if( pParent ){
         1351  +          int jmp = sqlite3VdbeCurrentAddr(v)+2;
         1352  +          sqlite3VdbeAddOp3(v, OP_SeekRowid, i, jmp, regRow); VdbeCoverage(v);
  1343   1353             sqlite3VdbeGoto(v, addrOk);
  1344         -          sqlite3VdbeJumpHere(v, sqlite3VdbeCurrentAddr(v)-2);
         1354  +          assert( pFK->nCol==1 );
         1355  +        }
         1356  +
         1357  +        /* Generate code to report an FK violation to the caller. */
         1358  +        if( HasRowid(pTab) ){
         1359  +          sqlite3VdbeAddOp2(v, OP_Rowid, 0, regResult+1);
  1345   1360           }else{
  1346         -          for(j=0; j<pFK->nCol; j++){
  1347         -            sqlite3ExprCodeGetColumnOfTable(v, pTab, 0,
  1348         -                            aiCols ? aiCols[j] : pFK->aCol[j].iFrom, regRow+j);
  1349         -            sqlite3VdbeAddOp2(v, OP_IsNull, regRow+j, addrOk); VdbeCoverage(v);
  1350         -          }
  1351         -          if( pParent ){
  1352         -            sqlite3VdbeAddOp4(v, OP_MakeRecord, regRow, pFK->nCol, regKey,
  1353         -                              sqlite3IndexAffinityStr(db,pIdx), pFK->nCol);
  1354         -            sqlite3VdbeAddOp4Int(v, OP_Found, i, addrOk, regKey, 0);
  1355         -            VdbeCoverage(v);
  1356         -          }
         1361  +          sqlite3VdbeAddOp2(v, OP_Null, 0, regResult+1);
  1357   1362           }
  1358         -        sqlite3VdbeAddOp2(v, OP_Rowid, 0, regResult+1);
  1359   1363           sqlite3VdbeMultiLoad(v, regResult+2, "si", pFK->zTo, i-1);
  1360   1364           sqlite3VdbeAddOp2(v, OP_ResultRow, regResult, 4);
  1361   1365           sqlite3VdbeResolveLabel(v, addrOk);
  1362   1366           sqlite3DbFree(db, aiCols);
  1363   1367         }
  1364   1368         sqlite3VdbeAddOp2(v, OP_Next, 0, addrTop+1); VdbeCoverage(v);
  1365   1369         sqlite3VdbeJumpHere(v, addrTop);

Changes to src/select.c.

   108    108     ExprList *pOrderBy,   /* the ORDER BY clause */
   109    109     u32 selFlags,         /* Flag parameters, such as SF_Distinct */
   110    110     Expr *pLimit,         /* LIMIT value.  NULL means not used */
   111    111     Expr *pOffset         /* OFFSET value.  NULL means no offset */
   112    112   ){
   113    113     Select *pNew;
   114    114     Select standin;
   115         -  sqlite3 *db = pParse->db;
   116         -  pNew = sqlite3DbMallocRawNN(db, sizeof(*pNew) );
          115  +  pNew = sqlite3DbMallocRawNN(pParse->db, sizeof(*pNew) );
   117    116     if( pNew==0 ){
   118         -    assert( db->mallocFailed );
          117  +    assert( pParse->db->mallocFailed );
   119    118       pNew = &standin;
   120    119     }
   121    120     if( pEList==0 ){
   122         -    pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db,TK_ASTERISK,0));
          121  +    pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(pParse->db,TK_ASTERISK,0));
   123    122     }
   124    123     pNew->pEList = pEList;
   125    124     pNew->op = TK_SELECT;
   126    125     pNew->selFlags = selFlags;
   127    126     pNew->iLimit = 0;
   128    127     pNew->iOffset = 0;
   129    128   #if SELECTTRACE_ENABLED
   130    129     pNew->zSelName[0] = 0;
   131    130   #endif
   132    131     pNew->addrOpenEphm[0] = -1;
   133    132     pNew->addrOpenEphm[1] = -1;
   134    133     pNew->nSelectRow = 0;
   135         -  if( pSrc==0 ) pSrc = sqlite3DbMallocZero(db, sizeof(*pSrc));
          134  +  if( pSrc==0 ) pSrc = sqlite3DbMallocZero(pParse->db, sizeof(*pSrc));
   136    135     pNew->pSrc = pSrc;
   137    136     pNew->pWhere = pWhere;
   138    137     pNew->pGroupBy = pGroupBy;
   139    138     pNew->pHaving = pHaving;
   140    139     pNew->pOrderBy = pOrderBy;
   141    140     pNew->pPrior = 0;
   142    141     pNew->pNext = 0;
   143    142     pNew->pLimit = pLimit;
   144    143     pNew->pOffset = pOffset;
   145    144     pNew->pWith = 0;
   146         -  assert( pOffset==0 || pLimit!=0 || pParse->nErr>0 || db->mallocFailed!=0 );
   147         -  if( db->mallocFailed ) {
   148         -    clearSelect(db, pNew, pNew!=&standin);
          145  +  assert( pOffset==0 || pLimit!=0 || pParse->nErr>0 || pParse->db->mallocFailed!=0 );
          146  +  if( pParse->db->mallocFailed ) {
          147  +    clearSelect(pParse->db, pNew, pNew!=&standin);
   149    148       pNew = 0;
   150    149     }else{
   151    150       assert( pNew->pSrc!=0 || pParse->nErr>0 );
   152    151     }
   153    152     assert( pNew!=&standin );
   154    153     return pNew;
   155    154   }
................................................................................
  1526   1525   ){
  1527   1526   #ifndef SQLITE_OMIT_DECLTYPE
  1528   1527     Vdbe *v = pParse->pVdbe;
  1529   1528     int i;
  1530   1529     NameContext sNC;
  1531   1530     sNC.pSrcList = pTabList;
  1532   1531     sNC.pParse = pParse;
         1532  +  sNC.pNext = 0;
  1533   1533     for(i=0; i<pEList->nExpr; i++){
  1534   1534       Expr *p = pEList->a[i].pExpr;
  1535   1535       const char *zType;
  1536   1536   #ifdef SQLITE_ENABLE_COLUMN_METADATA
  1537   1537       const char *zOrigDb = 0;
  1538   1538       const char *zOrigTab = 0;
  1539   1539       const char *zOrigCol = 0;
................................................................................
  1549   1549   #else
  1550   1550       zType = columnType(&sNC, p, 0, 0, 0, 0);
  1551   1551   #endif
  1552   1552       sqlite3VdbeSetColName(v, i, COLNAME_DECLTYPE, zType, SQLITE_TRANSIENT);
  1553   1553     }
  1554   1554   #endif /* !defined(SQLITE_OMIT_DECLTYPE) */
  1555   1555   }
         1556  +
         1557  +/*
         1558  +** Return the Table objecct in the SrcList that has cursor iCursor.
         1559  +** Or return NULL if no such Table object exists in the SrcList.
         1560  +*/
         1561  +static Table *tableWithCursor(SrcList *pList, int iCursor){
         1562  +  int j;
         1563  +  for(j=0; j<pList->nSrc; j++){
         1564  +    if( pList->a[j].iCursor==iCursor ) return pList->a[j].pTab;
         1565  +  }
         1566  +  return 0;
         1567  +}
         1568  +
  1556   1569   
  1557   1570   /*
  1558   1571   ** Generate code that will tell the VDBE the names of columns
  1559   1572   ** in the result set.  This information is used to provide the
  1560   1573   ** azCol[] values in the callback.
  1561   1574   */
  1562   1575   static void generateColumnNames(
  1563   1576     Parse *pParse,      /* Parser context */
  1564   1577     SrcList *pTabList,  /* List of tables */
  1565   1578     ExprList *pEList    /* Expressions defining the result set */
  1566   1579   ){
  1567   1580     Vdbe *v = pParse->pVdbe;
  1568         -  int i, j;
         1581  +  int i;
         1582  +  Table *pTab;
  1569   1583     sqlite3 *db = pParse->db;
  1570   1584     int fullNames, shortNames;
  1571   1585   
  1572   1586   #ifndef SQLITE_OMIT_EXPLAIN
  1573   1587     /* If this is an EXPLAIN, skip this step */
  1574   1588     if( pParse->explain ){
  1575   1589       return;
................................................................................
  1586   1600     for(i=0; i<pEList->nExpr; i++){
  1587   1601       Expr *p;
  1588   1602       p = pEList->a[i].pExpr;
  1589   1603       if( NEVER(p==0) ) continue;
  1590   1604       if( pEList->a[i].zName ){
  1591   1605         char *zName = pEList->a[i].zName;
  1592   1606         sqlite3VdbeSetColName(v, i, COLNAME_NAME, zName, SQLITE_TRANSIENT);
  1593         -    }else if( p->op==TK_COLUMN || p->op==TK_AGG_COLUMN ){
  1594         -      Table *pTab;
         1607  +    }else if( (p->op==TK_COLUMN || p->op==TK_AGG_COLUMN)
         1608  +           && (pTab = tableWithCursor(pTabList, p->iTable))!=0
         1609  +    ){
  1595   1610         char *zCol;
  1596   1611         int iCol = p->iColumn;
  1597         -      for(j=0; ALWAYS(j<pTabList->nSrc); j++){
  1598         -        if( pTabList->a[j].iCursor==p->iTable ) break;
  1599         -      }
  1600         -      assert( j<pTabList->nSrc );
  1601         -      pTab = pTabList->a[j].pTab;
  1602   1612         if( iCol<0 ) iCol = pTab->iPKey;
  1603   1613         assert( iCol==-1 || (iCol>=0 && iCol<pTab->nCol) );
  1604   1614         if( iCol<0 ){
  1605   1615           zCol = "rowid";
  1606   1616         }else{
  1607   1617           zCol = pTab->aCol[iCol].zName;
  1608   1618         }
................................................................................
  1676   1686       }else{
  1677   1687         Expr *pColExpr = p;  /* The expression that is the result column name */
  1678   1688         Table *pTab;         /* Table associated with this expression */
  1679   1689         while( pColExpr->op==TK_DOT ){
  1680   1690           pColExpr = pColExpr->pRight;
  1681   1691           assert( pColExpr!=0 );
  1682   1692         }
  1683         -      if( pColExpr->op==TK_COLUMN && ALWAYS(pColExpr->pTab!=0) ){
         1693  +      if( pColExpr->op==TK_COLUMN && pColExpr->pTab!=0 ){
  1684   1694           /* For columns use the column name name */
  1685   1695           int iCol = pColExpr->iColumn;
  1686   1696           pTab = pColExpr->pTab;
  1687   1697           if( iCol<0 ) iCol = pTab->iPKey;
  1688   1698           zName = iCol>=0 ? pTab->aCol[iCol].zName : "rowid";
  1689   1699         }else if( pColExpr->op==TK_ID ){
  1690   1700           assert( !ExprHasProperty(pColExpr, EP_IntValue) );
................................................................................
  3130   3140     **** subqueries ****/
  3131   3141     explainComposite(pParse, p->op, iSub1, iSub2, 0);
  3132   3142     return pParse->nErr!=0;
  3133   3143   }
  3134   3144   #endif
  3135   3145   
  3136   3146   #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW)
         3147  +
         3148  +/* An instance of the SubstContext object describes an substitution edit
         3149  +** to be performed on a parse tree.
         3150  +**
         3151  +** All references to columns in table iTable are to be replaced by corresponding
         3152  +** expressions in pEList.
         3153  +*/
         3154  +typedef struct SubstContext {
         3155  +  Parse *pParse;            /* The parsing context */
         3156  +  int iTable;               /* Replace references to this table */
         3157  +  int iNewTable;            /* New table number */
         3158  +  int isLeftJoin;           /* Add TK_IF_NULL_ROW opcodes on each replacement */
         3159  +  ExprList *pEList;         /* Replacement expressions */
         3160  +} SubstContext;
         3161  +
  3137   3162   /* Forward Declarations */
  3138         -static void substExprList(Parse*, ExprList*, int, ExprList*);
  3139         -static void substSelect(Parse*, Select *, int, ExprList*, int);
         3163  +static void substExprList(SubstContext*, ExprList*);
         3164  +static void substSelect(SubstContext*, Select*, int);
  3140   3165   
  3141   3166   /*
  3142   3167   ** Scan through the expression pExpr.  Replace every reference to
  3143   3168   ** a column in table number iTable with a copy of the iColumn-th
  3144   3169   ** entry in pEList.  (But leave references to the ROWID column 
  3145   3170   ** unchanged.)
  3146   3171   **
  3147   3172   ** This routine is part of the flattening procedure.  A subquery
  3148   3173   ** whose result set is defined by pEList appears as entry in the
  3149   3174   ** FROM clause of a SELECT such that the VDBE cursor assigned to that
  3150         -** FORM clause entry is iTable.  This routine make the necessary 
         3175  +** FORM clause entry is iTable.  This routine makes the necessary 
  3151   3176   ** changes to pExpr so that it refers directly to the source table
  3152   3177   ** of the subquery rather the result set of the subquery.
  3153   3178   */
  3154   3179   static Expr *substExpr(
  3155         -  Parse *pParse,      /* Report errors here */
  3156         -  Expr *pExpr,        /* Expr in which substitution occurs */
  3157         -  int iTable,         /* Table to be substituted */
  3158         -  ExprList *pEList    /* Substitute expressions */
         3180  +  SubstContext *pSubst,  /* Description of the substitution */
         3181  +  Expr *pExpr            /* Expr in which substitution occurs */
  3159   3182   ){
  3160         -  sqlite3 *db = pParse->db;
  3161   3183     if( pExpr==0 ) return 0;
  3162         -  if( pExpr->op==TK_COLUMN && pExpr->iTable==iTable ){
         3184  +  if( ExprHasProperty(pExpr, EP_FromJoin) && pExpr->iRightJoinTable==pSubst->iTable ){
         3185  +    pExpr->iRightJoinTable = pSubst->iNewTable;
         3186  +  }
         3187  +  if( pExpr->op==TK_COLUMN && pExpr->iTable==pSubst->iTable ){
  3163   3188       if( pExpr->iColumn<0 ){
  3164   3189         pExpr->op = TK_NULL;
  3165   3190       }else{
  3166   3191         Expr *pNew;
  3167         -      Expr *pCopy = pEList->a[pExpr->iColumn].pExpr;
  3168         -      assert( pEList!=0 && pExpr->iColumn<pEList->nExpr );
         3192  +      Expr *pCopy = pSubst->pEList->a[pExpr->iColumn].pExpr;
         3193  +      Expr ifNullRow;
         3194  +      assert( pSubst->pEList!=0 && pExpr->iColumn<pSubst->pEList->nExpr );
  3169   3195         assert( pExpr->pLeft==0 && pExpr->pRight==0 );
  3170   3196         if( sqlite3ExprIsVector(pCopy) ){
  3171         -        sqlite3VectorErrorMsg(pParse, pCopy);
         3197  +        sqlite3VectorErrorMsg(pSubst->pParse, pCopy);
  3172   3198         }else{
         3199  +        sqlite3 *db = pSubst->pParse->db;
         3200  +        if( pSubst->isLeftJoin && pCopy->op!=TK_COLUMN ){
         3201  +          memset(&ifNullRow, 0, sizeof(ifNullRow));
         3202  +          ifNullRow.op = TK_IF_NULL_ROW;
         3203  +          ifNullRow.pLeft = pCopy;
         3204  +          ifNullRow.iTable = pSubst->iNewTable;
         3205  +          pCopy = &ifNullRow;
         3206  +        }
  3173   3207           pNew = sqlite3ExprDup(db, pCopy, 0);
  3174   3208           if( pNew && (pExpr->flags & EP_FromJoin) ){
  3175   3209             pNew->iRightJoinTable = pExpr->iRightJoinTable;
  3176   3210             pNew->flags |= EP_FromJoin;
  3177   3211           }
  3178   3212           sqlite3ExprDelete(db, pExpr);
  3179   3213           pExpr = pNew;
  3180   3214         }
  3181   3215       }
  3182   3216     }else{
  3183         -    pExpr->pLeft = substExpr(pParse, pExpr->pLeft, iTable, pEList);
  3184         -    pExpr->pRight = substExpr(pParse, pExpr->pRight, iTable, pEList);
         3217  +    pExpr->pLeft = substExpr(pSubst, pExpr->pLeft);
         3218  +    pExpr->pRight = substExpr(pSubst, pExpr->pRight);
  3185   3219       if( ExprHasProperty(pExpr, EP_xIsSelect) ){
  3186         -      substSelect(pParse, pExpr->x.pSelect, iTable, pEList, 1);
         3220  +      substSelect(pSubst, pExpr->x.pSelect, 1);
  3187   3221       }else{
  3188         -      substExprList(pParse, pExpr->x.pList, iTable, pEList);
         3222  +      substExprList(pSubst, pExpr->x.pList);
  3189   3223       }
  3190   3224     }
  3191   3225     return pExpr;
  3192   3226   }
  3193   3227   static void substExprList(
  3194         -  Parse *pParse,       /* Report errors here */
  3195         -  ExprList *pList,     /* List to scan and in which to make substitutes */
  3196         -  int iTable,          /* Table to be substituted */
  3197         -  ExprList *pEList     /* Substitute values */
         3228  +  SubstContext *pSubst, /* Description of the substitution */
         3229  +  ExprList *pList       /* List to scan and in which to make substitutes */
  3198   3230   ){
  3199   3231     int i;
  3200   3232     if( pList==0 ) return;
  3201   3233     for(i=0; i<pList->nExpr; i++){
  3202         -    pList->a[i].pExpr = substExpr(pParse, pList->a[i].pExpr, iTable, pEList);
         3234  +    pList->a[i].pExpr = substExpr(pSubst, pList->a[i].pExpr);
  3203   3235     }
  3204   3236   }
  3205   3237   static void substSelect(
  3206         -  Parse *pParse,       /* Report errors here */
  3207         -  Select *p,           /* SELECT statement in which to make substitutions */
  3208         -  int iTable,          /* Table to be replaced */
  3209         -  ExprList *pEList,    /* Substitute values */
  3210         -  int doPrior          /* Do substitutes on p->pPrior too */
         3238  +  SubstContext *pSubst, /* Description of the substitution */
         3239  +  Select *p,            /* SELECT statement in which to make substitutions */
         3240  +  int doPrior           /* Do substitutes on p->pPrior too */
  3211   3241   ){
  3212   3242     SrcList *pSrc;
  3213   3243     struct SrcList_item *pItem;
  3214   3244     int i;
  3215   3245     if( !p ) return;
  3216   3246     do{
  3217         -    substExprList(pParse, p->pEList, iTable, pEList);
  3218         -    substExprList(pParse, p->pGroupBy, iTable, pEList);
  3219         -    substExprList(pParse, p->pOrderBy, iTable, pEList);
  3220         -    p->pHaving = substExpr(pParse, p->pHaving, iTable, pEList);
  3221         -    p->pWhere = substExpr(pParse, p->pWhere, iTable, pEList);
         3247  +    substExprList(pSubst, p->pEList);
         3248  +    substExprList(pSubst, p->pGroupBy);
         3249  +    substExprList(pSubst, p->pOrderBy);
         3250  +    p->pHaving = substExpr(pSubst, p->pHaving);
         3251  +    p->pWhere = substExpr(pSubst, p->pWhere);
  3222   3252       pSrc = p->pSrc;
  3223   3253       assert( pSrc!=0 );
  3224   3254       for(i=pSrc->nSrc, pItem=pSrc->a; i>0; i--, pItem++){
  3225         -      substSelect(pParse, pItem->pSelect, iTable, pEList, 1);
         3255  +      substSelect(pSubst, pItem->pSelect, 1);
  3226   3256         if( pItem->fg.isTabFunc ){
  3227         -        substExprList(pParse, pItem->u1.pFuncArg, iTable, pEList);
         3257  +        substExprList(pSubst, pItem->u1.pFuncArg);
  3228   3258         }
  3229   3259       }
  3230   3260     }while( doPrior && (p = p->pPrior)!=0 );
  3231   3261   }
  3232   3262   #endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */
  3233   3263   
  3234   3264   #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW)
................................................................................
  3263   3293   **   (1)  The subquery and the outer query do not both use aggregates.
  3264   3294   **
  3265   3295   **   (2)  The subquery is not an aggregate or (2a) the outer query is not a join
  3266   3296   **        and (2b) the outer query does not use subqueries other than the one
  3267   3297   **        FROM-clause subquery that is a candidate for flattening.  (2b is
  3268   3298   **        due to ticket [2f7170d73bf9abf80] from 2015-02-09.)
  3269   3299   **
  3270         -**   (3)  The subquery is not the right operand of a left outer join
  3271         -**        (Originally ticket #306.  Strengthened by ticket #3300)
         3300  +**   (3)  The subquery is not the right operand of a LEFT JOIN
         3301  +**        or the subquery is not itself a join.
  3272   3302   **
  3273   3303   **   (4)  The subquery is not DISTINCT.
  3274   3304   **
  3275   3305   **  (**)  At one point restrictions (4) and (5) defined a subset of DISTINCT
  3276   3306   **        sub-queries that were excluded from this optimization. Restriction 
  3277   3307   **        (4) has since been expanded to exclude all DISTINCT subqueries.
  3278   3308   **
  3279   3309   **   (6)  The subquery does not use aggregates or the outer query is not
  3280   3310   **        DISTINCT.
  3281   3311   **
  3282   3312   **   (7)  The subquery has a FROM clause.  TODO:  For subqueries without
  3283         -**        A FROM clause, consider adding a FROM close with the special
         3313  +**        A FROM clause, consider adding a FROM clause with the special
  3284   3314   **        table sqlite_once that consists of a single row containing a
  3285   3315   **        single NULL.
  3286   3316   **
  3287   3317   **   (8)  The subquery does not use LIMIT or the outer query is not a join.
  3288   3318   **
  3289   3319   **   (9)  The subquery does not use LIMIT or the outer query does not use
  3290   3320   **        aggregates.
................................................................................
  3382   3412     Select *pParent;    /* Current UNION ALL term of the other query */
  3383   3413     Select *pSub;       /* The inner query or "subquery" */
  3384   3414     Select *pSub1;      /* Pointer to the rightmost select in sub-query */
  3385   3415     SrcList *pSrc;      /* The FROM clause of the outer query */
  3386   3416     SrcList *pSubSrc;   /* The FROM clause of the subquery */
  3387   3417     ExprList *pList;    /* The result set of the outer query */
  3388   3418     int iParent;        /* VDBE cursor number of the pSub result set temp table */
         3419  +  int iNewParent = -1;/* Replacement table for iParent */
         3420  +  int isLeftJoin = 0; /* True if pSub is the right side of a LEFT JOIN */    
  3389   3421     int i;              /* Loop counter */
  3390   3422     Expr *pWhere;                    /* The WHERE clause */
  3391   3423     struct SrcList_item *pSubitem;   /* The subquery */
  3392   3424     sqlite3 *db = pParse->db;
  3393   3425   
  3394   3426     /* Check to see if flattening is permitted.  Return 0 if not.
  3395   3427     */
................................................................................
  3408   3440       if( (p->pWhere && ExprHasProperty(p->pWhere,EP_Subquery))
  3409   3441        || (sqlite3ExprListFlags(p->pEList) & EP_Subquery)!=0
  3410   3442        || (sqlite3ExprListFlags(p->pOrderBy) & EP_Subquery)!=0
  3411   3443       ){
  3412   3444         return 0;                                          /* Restriction (2b)  */
  3413   3445       }
  3414   3446     }
  3415         -    
         3447  +
  3416   3448     pSubSrc = pSub->pSrc;
  3417   3449     assert( pSubSrc );
  3418   3450     /* Prior to version 3.1.2, when LIMIT and OFFSET had to be simple constants,
  3419   3451     ** not arbitrary expressions, we allowed some combining of LIMIT and OFFSET
  3420   3452     ** because they could be computed at compile-time.  But when LIMIT and OFFSET
  3421   3453     ** became arbitrary expressions, we were forced to add restrictions (13)
  3422   3454     ** and (14). */
................................................................................
  3446   3478     if( pSub->selFlags & (SF_Recursive|SF_MinMaxAgg) ){
  3447   3479       return 0; /* Restrictions (22) and (24) */
  3448   3480     }
  3449   3481     if( (p->selFlags & SF_Recursive) && pSub->pPrior ){
  3450   3482       return 0; /* Restriction (23) */
  3451   3483     }
  3452   3484   
  3453         -  /* OBSOLETE COMMENT 1:
  3454         -  ** Restriction 3:  If the subquery is a join, make sure the subquery is 
  3455         -  ** not used as the right operand of an outer join.  Examples of why this
  3456         -  ** is not allowed:
         3485  +  /*
         3486  +  ** If the subquery is the right operand of a LEFT JOIN, then the
         3487  +  ** subquery may not be a join itself.  Example of why this is not allowed:
  3457   3488     **
  3458   3489     **         t1 LEFT OUTER JOIN (t2 JOIN t3)
  3459   3490     **
  3460   3491     ** If we flatten the above, we would get
  3461   3492     **
  3462   3493     **         (t1 LEFT OUTER JOIN t2) JOIN t3
  3463   3494     **
  3464   3495     ** which is not at all the same thing.
  3465   3496     **
  3466         -  ** OBSOLETE COMMENT 2:
  3467         -  ** Restriction 12:  If the subquery is the right operand of a left outer
  3468         -  ** join, make sure the subquery has no WHERE clause.
  3469         -  ** An examples of why this is not allowed:
  3470         -  **
  3471         -  **         t1 LEFT OUTER JOIN (SELECT * FROM t2 WHERE t2.x>0)
  3472         -  **
  3473         -  ** If we flatten the above, we would get
  3474         -  **
  3475         -  **         (t1 LEFT OUTER JOIN t2) WHERE t2.x>0
  3476         -  **
  3477         -  ** But the t2.x>0 test will always fail on a NULL row of t2, which
  3478         -  ** effectively converts the OUTER JOIN into an INNER JOIN.
  3479         -  **
  3480         -  ** THIS OVERRIDES OBSOLETE COMMENTS 1 AND 2 ABOVE:
  3481         -  ** Ticket #3300 shows that flattening the right term of a LEFT JOIN
  3482         -  ** is fraught with danger.  Best to avoid the whole thing.  If the
  3483         -  ** subquery is the right term of a LEFT JOIN, then do not flatten.
         3497  +  ** See also tickets #306, #350, and #3300.
  3484   3498     */
  3485   3499     if( (pSubitem->fg.jointype & JT_OUTER)!=0 ){
  3486         -    return 0;
         3500  +    isLeftJoin = 1;
         3501  +    if( pSubSrc->nSrc>1 ){
         3502  +      return 0; /* Restriction (3) */
         3503  +    }
  3487   3504     }
  3488   3505   
  3489   3506     /* Restriction 17: If the sub-query is a compound SELECT, then it must
  3490   3507     ** use only the UNION ALL operator. And none of the simple select queries
  3491   3508     ** that make up the compound SELECT are allowed to be aggregate or distinct
  3492   3509     ** queries.
  3493   3510     */
................................................................................
  3688   3705       /* Transfer the FROM clause terms from the subquery into the
  3689   3706       ** outer query.
  3690   3707       */
  3691   3708       for(i=0; i<nSubSrc; i++){
  3692   3709         sqlite3IdListDelete(db, pSrc->a[i+iFrom].pUsing);
  3693   3710         assert( pSrc->a[i+iFrom].fg.isTabFunc==0 );
  3694   3711         pSrc->a[i+iFrom] = pSubSrc->a[i];
         3712  +      iNewParent = pSubSrc->a[i].iCursor;
  3695   3713         memset(&pSubSrc->a[i], 0, sizeof(pSubSrc->a[i]));
  3696   3714       }
  3697   3715       pSrc->a[iFrom].fg.jointype = jointype;
  3698   3716     
  3699   3717       /* Now begin substituting subquery result set expressions for 
  3700   3718       ** references to the iParent in the outer query.
  3701   3719       ** 
................................................................................
  3733   3751         }
  3734   3752         assert( pParent->pOrderBy==0 );
  3735   3753         assert( pSub->pPrior==0 );
  3736   3754         pParent->pOrderBy = pOrderBy;
  3737   3755         pSub->pOrderBy = 0;
  3738   3756       }
  3739   3757       pWhere = sqlite3ExprDup(db, pSub->pWhere, 0);
         3758  +    if( isLeftJoin ){
         3759  +      setJoinExpr(pWhere, iNewParent);
         3760  +    }
  3740   3761       if( subqueryIsAgg ){
  3741   3762         assert( pParent->pHaving==0 );
  3742   3763         pParent->pHaving = pParent->pWhere;
  3743   3764         pParent->pWhere = pWhere;
  3744   3765         pParent->pHaving = sqlite3ExprAnd(db, 
  3745   3766             sqlite3ExprDup(db, pSub->pHaving, 0), pParent->pHaving
  3746   3767         );
  3747   3768         assert( pParent->pGroupBy==0 );
  3748   3769         pParent->pGroupBy = sqlite3ExprListDup(db, pSub->pGroupBy, 0);
  3749   3770       }else{
  3750   3771         pParent->pWhere = sqlite3ExprAnd(db, pWhere, pParent->pWhere);
  3751   3772       }
  3752   3773       if( db->mallocFailed==0 ){
  3753         -      substSelect(pParse, pParent, iParent, pSub->pEList, 0);
         3774  +      SubstContext x;
         3775  +      x.pParse = pParse;
         3776  +      x.iTable = iParent;
         3777  +      x.iNewTable = iNewParent;
         3778  +      x.isLeftJoin = isLeftJoin;
         3779  +      x.pEList = pSub->pEList;
         3780  +      substSelect(&x, pParent, 0);
  3754   3781       }
  3755   3782     
  3756   3783       /* The flattened query is distinct if either the inner or the
  3757   3784       ** outer query is distinct. 
  3758   3785       */
  3759   3786       pParent->selFlags |= pSub->selFlags & SF_Distinct;
  3760   3787     
................................................................................
  3849   3876       nChng += pushDownWhereTerms(pParse, pSubq, pWhere->pRight, iCursor);
  3850   3877       pWhere = pWhere->pLeft;
  3851   3878     }
  3852   3879     if( ExprHasProperty(pWhere,EP_FromJoin) ) return 0; /* restriction 5 */
  3853   3880     if( sqlite3ExprIsTableConstant(pWhere, iCursor) ){
  3854   3881       nChng++;
  3855   3882       while( pSubq ){
         3883  +      SubstContext x;
  3856   3884         pNew = sqlite3ExprDup(pParse->db, pWhere, 0);
  3857         -      pNew = substExpr(pParse, pNew, iCursor, pSubq->pEList);
         3885  +      x.pParse = pParse;
         3886  +      x.iTable = iCursor;
         3887  +      x.iNewTable = iCursor;
         3888  +      x.isLeftJoin = 0;
         3889  +      x.pEList = pSubq->pEList;
         3890  +      pNew = substExpr(&x, pNew);
  3858   3891         pSubq->pWhere = sqlite3ExprAnd(pParse->db, pSubq->pWhere, pNew);
  3859   3892         pSubq = pSubq->pPrior;
  3860   3893       }
  3861   3894     }
  3862   3895     return nChng;
  3863   3896   }
  3864   3897   #endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */

Changes to src/shell.c.

   423    423     z = sqlite3_vmprintf(zFormat, ap);
   424    424     va_end(ap);
   425    425     utf8_printf(iotrace, "%s", z);
   426    426     sqlite3_free(z);
   427    427   }
   428    428   #endif
   429    429   
          430  +/*
          431  +** Output string zUtf to stream pOut as w characters.  If w is negative,
          432  +** then right-justify the text.  W is the width in UTF-8 characters, not
          433  +** in bytes.  This is different from the %*.*s specification in printf
          434  +** since with %*.*s the width is measured in bytes, not characters.
          435  +*/
          436  +static void utf8_width_print(FILE *pOut, int w, const char *zUtf){
          437  +  int i;
          438  +  int n;
          439  +  int aw = w<0 ? -w : w;
          440  +  char zBuf[1000];
          441  +  if( aw>sizeof(zBuf)/3 ) aw = sizeof(zBuf)/3;
          442  +  for(i=n=0; zUtf[i]; i++){
          443  +    if( (zUtf[i]&0xc0)!=0x80 ){
          444  +      n++;
          445  +      if( n==aw ){
          446  +        do{ i++; }while( (zUtf[i]&0xc0)==0x80 );
          447  +        break;
          448  +      }
          449  +    }
          450  +  }
          451  +  if( n>=aw ){
          452  +    utf8_printf(pOut, "%.*s", i, zUtf);
          453  +  }else if( w<0 ){
          454  +    utf8_printf(pOut, "%*s%s", aw-n, "", zUtf);
          455  +  }else{
          456  +    utf8_printf(pOut, "%s%*s", zUtf, aw-n, "");
          457  +  }
          458  +}
          459  +
   430    460   
   431    461   /*
   432    462   ** Determines if a string is a number of not.
   433    463   */
   434    464   static int isNumber(const char *z, int *realnum){
   435    465     if( *z=='-' || *z=='+' ) z++;
   436    466     if( !IsDigit(*z) ){
................................................................................
  1874   1904               n = strlen30(azArg && azArg[i] ? azArg[i] : p->nullValue);
  1875   1905               if( w<n ) w = n;
  1876   1906             }
  1877   1907             if( i<ArraySize(p->actualWidth) ){
  1878   1908               p->actualWidth[i] = w;
  1879   1909             }
  1880   1910             if( showHdr ){
  1881         -            if( w<0 ){
  1882         -              utf8_printf(p->out,"%*.*s%s",-w,-w,azCol[i],
  1883         -                      i==nArg-1 ? rowSep : "  ");
  1884         -            }else{
  1885         -              utf8_printf(p->out,"%-*.*s%s",w,w,azCol[i],
  1886         -                      i==nArg-1 ? rowSep : "  ");
  1887         -            }
         1911  +            utf8_width_print(p->out, w, azCol[i]);
         1912  +            utf8_printf(p->out, "%s", i==nArg-1 ? rowSep : "  ");
  1888   1913             }
  1889   1914           }
  1890   1915           if( showHdr ){
  1891   1916             for(i=0; i<nArg; i++){
  1892   1917               int w;
  1893   1918               if( i<ArraySize(p->actualWidth) ){
  1894   1919                  w = p->actualWidth[i];
................................................................................
  1916   1941           }
  1917   1942           if( i==1 && p->aiIndent && p->pStmt ){
  1918   1943             if( p->iIndent<p->nIndent ){
  1919   1944               utf8_printf(p->out, "%*.s", p->aiIndent[p->iIndent], "");
  1920   1945             }
  1921   1946             p->iIndent++;
  1922   1947           }
  1923         -        if( w<0 ){
  1924         -          utf8_printf(p->out,"%*.*s%s",-w,-w,
  1925         -              azArg[i] ? azArg[i] : p->nullValue,
  1926         -              i==nArg-1 ? rowSep : "  ");
  1927         -        }else{
  1928         -          utf8_printf(p->out,"%-*.*s%s",w,w,
  1929         -              azArg[i] ? azArg[i] : p->nullValue,
  1930         -              i==nArg-1 ? rowSep : "  ");
  1931         -        }
         1948  +        utf8_width_print(p->out, w, azArg[i] ? azArg[i] : p->nullValue);
         1949  +        utf8_printf(p->out, "%s", i==nArg-1 ? rowSep : "  ");
  1932   1950         }
  1933   1951         break;
  1934   1952       }
  1935   1953       case MODE_Semi: {   /* .schema and .fullschema output */
  1936   1954         printSchemaLine(p->out, azArg[0], ";\n");
  1937   1955         break;
  1938   1956       }

Changes to src/sqlite.h.in.

  3699   3699   ** ^The sqlite3_value object returned by
  3700   3700   ** [sqlite3_column_value()] is unprotected.
  3701   3701   ** Unprotected sqlite3_value objects may only be used with
  3702   3702   ** [sqlite3_result_value()] and [sqlite3_bind_value()].
  3703   3703   ** The [sqlite3_value_blob | sqlite3_value_type()] family of
  3704   3704   ** interfaces require protected sqlite3_value objects.
  3705   3705   */
  3706         -typedef struct Mem sqlite3_value;
         3706  +typedef struct sqlite3_value sqlite3_value;
  3707   3707   
  3708   3708   /*
  3709   3709   ** CAPI3REF: SQL Function Context Object
  3710   3710   **
  3711   3711   ** The context in which an SQL function executes is stored in an
  3712   3712   ** sqlite3_context object.  ^A pointer to an sqlite3_context object
  3713   3713   ** is always first parameter to [application-defined SQL functions].

Changes to src/sqliteInt.h.

  3320   3320       int n;                                     /* A counter */
  3321   3321       int iCur;                                  /* A cursor number */
  3322   3322       SrcList *pSrcList;                         /* FROM clause */
  3323   3323       struct SrcCount *pSrcCount;                /* Counting column references */
  3324   3324       struct CCurHint *pCCurHint;                /* Used by codeCursorHint() */
  3325   3325       int *aiCol;                                /* array of column indexes */
  3326   3326       struct IdxCover *pIdxCover;                /* Check for index coverage */
         3327  +    struct IdxExprTrans *pIdxTrans;            /* Convert indexed expr to column */
  3327   3328     } u;
  3328   3329   };
  3329   3330   
  3330   3331   /* Forward declarations */
  3331   3332   int sqlite3WalkExpr(Walker*, Expr*);
  3332   3333   int sqlite3WalkExprList(Walker*, ExprList*);
  3333   3334   int sqlite3WalkSelect(Walker*, Select*);

Changes to src/test_delete.c.

    15     15   **   * The journal file.
    16     16   **   * The wal file.
    17     17   **   * The SQLITE_ENABLE_8_3_NAMES version of the db, journal or wal files.
    18     18   **   * Files created by the test_multiplex.c module to extend any of the 
    19     19   **     above.
    20     20   */
    21     21   
    22         -#if SQLITE_OS_WIN
    23         -#  include <io.h>
    24         -#  define F_OK 0
    25         -#else
           22  +#ifndef SQLITE_OS_WIN
    26     23   #  include <unistd.h>
           24  +#  include <errno.h>
    27     25   #endif
    28     26   #include <string.h>
    29         -#include <errno.h>
           27  +#include <assert.h>
    30     28   #include "sqlite3.h"
    31     29   
    32     30   /* The following #defines are copied from test_multiplex.c */
    33     31   #ifndef MX_CHUNK_NUMBER 
    34     32   # define MX_CHUNK_NUMBER 299
    35     33   #endif
    36     34   #ifndef SQLITE_MULTIPLEX_JOURNAL_8_3_OFFSET
................................................................................
    53     51   }
    54     52   
    55     53   /*
    56     54   ** zFile is a filename. Assuming no error occurs, if this file exists, 
    57     55   ** set *pbExists to true and unlink it. Or, if the file does not exist,
    58     56   ** set *pbExists to false before returning.
    59     57   **
    60         -** If an error occurs, the value of errno is returned. Or, if no error
    61         -** occurs, zero is returned.
           58  +** If an error occurs, non-zero is returned. Or, if no error occurs, zero.
    62     59   */
    63         -static int sqlite3DeleteUnlinkIfExists(const char *zFile, int *pbExists){
    64         -  int rc;
           60  +static int sqlite3DeleteUnlinkIfExists(
           61  +  sqlite3_vfs *pVfs,
           62  +  const char *zFile, 
           63  +  int *pbExists
           64  +){
           65  +  int rc = SQLITE_ERROR;
           66  +#if SQLITE_OS_WIN
           67  +  if( pVfs ){
           68  +    if( pbExists ) *pbExists = 1;
           69  +    rc = pVfs->xDelete(pVfs, zFile, 0);
           70  +    if( rc==SQLITE_IOERR_DELETE_NOENT ){
           71  +      if( pbExists ) *pbExists = 0;
           72  +      rc = SQLITE_OK;
           73  +    }
           74  +  }
           75  +#else
           76  +  assert( pVfs==0 );
    65     77     rc = access(zFile, F_OK);
    66     78     if( rc ){
    67     79       if( errno==ENOENT ){ 
    68     80         if( pbExists ) *pbExists = 0;
    69         -      return 0; 
           81  +      rc = SQLITE_OK; 
    70     82       }
    71         -    return errno;
           83  +  }else{
           84  +    if( pbExists ) *pbExists = 1;
           85  +    rc = unlink(zFile);
    72     86     }
    73         -  if( pbExists ) *pbExists = 1;
    74         -  rc = unlink(zFile);
    75         -  if( rc ) return errno;
    76         -  return 0;
           87  +#endif
           88  +  return rc;
    77     89   }
    78     90   
    79     91   /*
    80     92   ** Delete the database file identified by the string argument passed to this
    81     93   ** function. The string must contain a filename, not an SQLite URI.
    82     94   */
    83     95   SQLITE_API int sqlite3_delete_database(
................................................................................
    98    110       { "%s%03d",         0,   0 },
    99    111       { "%s-journal%03d", 0,   0 },
   100    112       { "%s-wal%03d",     0,   0 },
   101    113       { "%s%03d",         0,   1 },
   102    114       { "%s-journal%03d", SQLITE_MULTIPLEX_JOURNAL_8_3_OFFSET, 1 },
   103    115       { "%s-wal%03d",     SQLITE_MULTIPLEX_WAL_8_3_OFFSET, 1 },
   104    116     };
          117  +
          118  +#ifdef SQLITE_OS_WIN
          119  +  sqlite3_vfs *pVfs = sqlite3_vfs_find("win32");
          120  +#else
          121  +  sqlite3_vfs *pVfs = 0;
          122  +#endif
   105    123   
   106    124     /* Allocate a buffer large enough for any of the files that need to be
   107    125     ** deleted.  */
   108    126     nBuf = (int)strlen(zFile) + 100;
   109    127     zBuf = (char*)sqlite3_malloc(nBuf);
   110    128     if( zBuf==0 ) return SQLITE_NOMEM;
   111    129   
   112    130     /* Delete both the regular and 8.3 filenames versions of the database,
   113    131     ** journal, wal and shm files.  */
   114    132     for(i=0; rc==0 && i<sizeof(azFmt)/sizeof(azFmt[0]); i++){
   115    133       sqlite3_snprintf(nBuf, zBuf, azFmt[i], zFile);
   116         -    rc = sqlite3DeleteUnlinkIfExists(zBuf, 0);
          134  +    rc = sqlite3DeleteUnlinkIfExists(pVfs, zBuf, 0);
   117    135       if( rc==0 && i!=0 ){
   118    136         sqlite3Delete83Name(zBuf);
   119         -      rc = sqlite3DeleteUnlinkIfExists(zBuf, 0);
          137  +      rc = sqlite3DeleteUnlinkIfExists(pVfs, zBuf, 0);
   120    138       }
   121    139     }
   122    140   
   123    141     /* Delete any multiplexor files */
   124    142     for(i=0; rc==0 && i<sizeof(aMFile)/sizeof(aMFile[0]); i++){
   125    143       struct MFile *p = &aMFile[i];
   126    144       int iChunk;
   127    145       for(iChunk=1; iChunk<=MX_CHUNK_NUMBER; iChunk++){
   128    146         int bExists;
   129    147         sqlite3_snprintf(nBuf, zBuf, p->zFmt, zFile, iChunk+p->iOffset);
   130    148         if( p->b83 ) sqlite3Delete83Name(zBuf);
   131         -      rc = sqlite3DeleteUnlinkIfExists(zBuf, &bExists);
          149  +      rc = sqlite3DeleteUnlinkIfExists(pVfs, zBuf, &bExists);
   132    150         if( bExists==0 || rc!=0 ) break;
   133    151       }
   134    152     }
   135    153   
   136    154     sqlite3_free(zBuf);
   137    155     return (rc ? SQLITE_ERROR : SQLITE_OK);
   138    156   }

Changes to src/treeview.c.

   234    234   
   235    235   /*
   236    236   ** Generate a human-readable explanation of an expression tree.
   237    237   */
   238    238   void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){
   239    239     const char *zBinOp = 0;   /* Binary operator */
   240    240     const char *zUniOp = 0;   /* Unary operator */
   241         -  char zFlgs[30];
          241  +  char zFlgs[60];
   242    242     pView = sqlite3TreeViewPush(pView, moreToFollow);
   243    243     if( pExpr==0 ){
   244    244       sqlite3TreeViewLine(pView, "nil");
   245    245       sqlite3TreeViewPop(pView);
   246    246       return;
   247    247     }
   248    248     if( pExpr->flags ){
   249         -    sqlite3_snprintf(sizeof(zFlgs),zFlgs,"  flags=0x%x",pExpr->flags);
          249  +    if( ExprHasProperty(pExpr, EP_FromJoin) ){
          250  +      sqlite3_snprintf(sizeof(zFlgs),zFlgs,"  flags=0x%x iRJT=%d",
          251  +                       pExpr->flags, pExpr->iRightJoinTable);
          252  +    }else{
          253  +      sqlite3_snprintf(sizeof(zFlgs),zFlgs,"  flags=0x%x",pExpr->flags);
          254  +    }
   250    255     }else{
   251    256       zFlgs[0] = 0;
   252    257     }
   253    258     switch( pExpr->op ){
   254    259       case TK_AGG_COLUMN: {
   255    260         sqlite3TreeViewLine(pView, "AGG{%d:%d}%s",
   256    261               pExpr->iTable, pExpr->iColumn, zFlgs);
................................................................................
   460    465         sqlite3TreeViewBareExprList(pView, pExpr->x.pList, "VECTOR");
   461    466         break;
   462    467       }
   463    468       case TK_SELECT_COLUMN: {
   464    469         sqlite3TreeViewLine(pView, "SELECT-COLUMN %d", pExpr->iColumn);
   465    470         sqlite3TreeViewSelect(pView, pExpr->pLeft->x.pSelect, 0);
   466    471         break;
          472  +    }
          473  +    case TK_IF_NULL_ROW: {
          474  +      sqlite3TreeViewLine(pView, "IF-NULL-ROW %d", pExpr->iTable);
          475  +      sqlite3TreeViewExpr(pView, pExpr->pLeft, 0);
          476  +      break;
   467    477       }
   468    478       default: {
   469    479         sqlite3TreeViewLine(pView, "op=%d", pExpr->op);
   470    480         break;
   471    481       }
   472    482     }
   473    483     if( zBinOp ){

Changes to src/update.c.

   281    281     ** being updated.  Fill in aRegIdx[] with a register number that will hold
   282    282     ** the key for accessing each index.
   283    283     **
   284    284     ** FIXME:  Be smarter about omitting indexes that use expressions.
   285    285     */
   286    286     for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){
   287    287       int reg;
   288         -    if( chngKey || hasFK || pIdx->pPartIdxWhere || pIdx==pPk ){
          288  +    if( chngKey || hasFK>1 || pIdx->pPartIdxWhere || pIdx==pPk ){
   289    289         reg = ++pParse->nMem;
   290    290         pParse->nMem += pIdx->nColumn;
   291    291       }else{
   292    292         reg = 0;
   293    293         for(i=0; i<pIdx->nKeyCol; i++){
   294    294           i16 iIdxCol = pIdx->aiColumn[i];
   295    295           if( iIdxCol<0 || aXRef[iIdxCol]>=0 ){
................................................................................
   636    636       ** pre-update hook. If the caller invokes preupdate_new(), the returned
   637    637       ** value is copied from memory cell (regNewRowid+1+iCol), where iCol
   638    638       ** is the column index supplied by the user.
   639    639       */
   640    640       assert( regNew==regNewRowid+1 );
   641    641   #ifdef SQLITE_ENABLE_PREUPDATE_HOOK
   642    642       sqlite3VdbeAddOp3(v, OP_Delete, iDataCur,
   643         -        OPFLAG_ISUPDATE | ((hasFK || chngKey) ? 0 : OPFLAG_ISNOOP),
          643  +        OPFLAG_ISUPDATE | ((hasFK>1 || chngKey) ? 0 : OPFLAG_ISNOOP),
   644    644           regNewRowid
   645    645       );
   646    646       if( eOnePass==ONEPASS_MULTI ){
   647    647         assert( hasFK==0 && chngKey==0 );
   648    648         sqlite3VdbeChangeP5(v, OPFLAG_SAVEPOSITION);
   649    649       }
   650    650       if( !pParse->nested ){
   651    651         sqlite3VdbeAppendP4(v, pTab, P4_TABLE);
   652    652       }
   653    653   #else
   654         -    if( hasFK || chngKey ){
          654  +    if( hasFK>1 || chngKey ){
   655    655         sqlite3VdbeAddOp2(v, OP_Delete, iDataCur, 0);
   656    656       }
   657    657   #endif
   658    658       if( bReplace || chngKey ){
   659    659         sqlite3VdbeJumpHere(v, addr1);
   660    660       }
   661    661   

Changes to src/vdbe.c.

   482    482     }
   483    483     if( p->flags & MEM_Subtype ) printf(" subtype=0x%02x", p->eSubtype);
   484    484   }
   485    485   static void registerTrace(int iReg, Mem *p){
   486    486     printf("REG[%d] = ", iReg);
   487    487     memTracePrint(p);
   488    488     printf("\n");
          489  +  sqlite3VdbeCheckMemInvariants(p);
   489    490   }
   490    491   #endif
   491    492   
   492    493   #ifdef SQLITE_DEBUG
   493    494   #  define REGISTER_TRACE(R,M) if(db->flags&SQLITE_VdbeTrace)registerTrace(R,M)
   494    495   #else
   495    496   #  define REGISTER_TRACE(R,M)
................................................................................
  1148   1149   ** instruction, but do not free any string or blob memory associated with
  1149   1150   ** the register, so that if the value was a string or blob that was
  1150   1151   ** previously copied using OP_SCopy, the copies will continue to be valid.
  1151   1152   */
  1152   1153   case OP_SoftNull: {
  1153   1154     assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) );
  1154   1155     pOut = &aMem[pOp->p1];
  1155         -  pOut->flags = (pOut->flags|MEM_Null)&~MEM_Undefined;
         1156  +  pOut->flags = (pOut->flags&~(MEM_Undefined|MEM_AffMask))|MEM_Null;
  1156   1157     break;
  1157   1158   }
  1158   1159   
  1159   1160   /* Opcode: Blob P1 P2 * P4 *
  1160   1161   ** Synopsis: r[P2]=P4 (len=P1)
  1161   1162   **
  1162   1163   ** P4 points to a blob of data P1 bytes long.  Store this
................................................................................
  1491   1492   
  1492   1493     pIn1 = &aMem[pOp->p1];
  1493   1494     type1 = numericType(pIn1);
  1494   1495     pIn2 = &aMem[pOp->p2];
  1495   1496     type2 = numericType(pIn2);
  1496   1497     pOut = &aMem[pOp->p3];
  1497   1498     flags = pIn1->flags | pIn2->flags;
  1498         -  if( (flags & MEM_Null)!=0 ) goto arithmetic_result_is_null;
  1499   1499     if( (type1 & type2 & MEM_Int)!=0 ){
  1500   1500       iA = pIn1->u.i;
  1501   1501       iB = pIn2->u.i;
  1502   1502       bIntint = 1;
  1503   1503       switch( pOp->opcode ){
  1504   1504         case OP_Add:       if( sqlite3AddInt64(&iB,iA) ) goto fp_math;  break;
  1505   1505         case OP_Subtract:  if( sqlite3SubInt64(&iB,iA) ) goto fp_math;  break;
................................................................................
  1515   1515           if( iA==-1 ) iA = 1;
  1516   1516           iB %= iA;
  1517   1517           break;
  1518   1518         }
  1519   1519       }
  1520   1520       pOut->u.i = iB;
  1521   1521       MemSetTypeFlag(pOut, MEM_Int);
         1522  +  }else if( (flags & MEM_Null)!=0 ){
         1523  +    goto arithmetic_result_is_null;
  1522   1524     }else{
  1523   1525       bIntint = 0;
  1524   1526   fp_math:
  1525   1527       rA = sqlite3VdbeRealValue(pIn1);
  1526   1528       rB = sqlite3VdbeRealValue(pIn2);
  1527   1529       switch( pOp->opcode ){
  1528   1530         case OP_Add:         rB += rA;       break;
................................................................................
  2425   2427     pIn1 = &aMem[pOp->p1];
  2426   2428     VdbeBranchTaken( (pIn1->flags & MEM_Null)==0, 2);
  2427   2429     if( (pIn1->flags & MEM_Null)==0 ){
  2428   2430       goto jump_to_p2;
  2429   2431     }
  2430   2432     break;
  2431   2433   }
         2434  +
         2435  +/* Opcode: IfNullRow P1 P2 P3 * *
         2436  +** Synopsis: if P1.nullRow then r[P3]=NULL, goto P2
         2437  +**
         2438  +** Check the cursor P1 to see if it is currently pointing at a NULL row.
         2439  +** If it is, then set register P3 to NULL and jump immediately to P2.
         2440  +** If P1 is not on a NULL row, then fall through without making any
         2441  +** changes.
         2442  +*/
         2443  +case OP_IfNullRow: {         /* jump */
         2444  +  assert( pOp->p1>=0 && pOp->p1<p->nCursor );
         2445  +  if( p->apCsr[pOp->p1]->nullRow ){
         2446  +    sqlite3VdbeMemSetNull(aMem + pOp->p3);
         2447  +    goto jump_to_p2;
         2448  +  }
         2449  +  break;
         2450  +}
  2432   2451   
  2433   2452   /* Opcode: Column P1 P2 P3 P4 P5
  2434   2453   ** Synopsis: r[P3]=PX
  2435   2454   **
  2436   2455   ** Interpret the data that cursor P1 points to as a structure built using
  2437   2456   ** the MakeRecord instruction.  (See the MakeRecord opcode for additional
  2438   2457   ** information about the format of the data.)  Extract the P2-th column

Changes to src/vdbe.h.

    26     26   */
    27     27   typedef struct Vdbe Vdbe;
    28     28   
    29     29   /*
    30     30   ** The names of the following types declared in vdbeInt.h are required
    31     31   ** for the VdbeOp definition.
    32     32   */
    33         -typedef struct Mem Mem;
           33  +typedef struct sqlite3_value Mem;
    34     34   typedef struct SubProgram SubProgram;
    35     35   
    36     36   /*
    37     37   ** A single instruction of the virtual machine has an opcode
    38     38   ** and as many as three operands.  The instruction is recorded
    39     39   ** as an instance of the following structure:
    40     40   */

Changes to src/vdbeInt.h.

   181    181   #define VdbeFrameMem(p) ((Mem *)&((u8 *)p)[ROUND8(sizeof(VdbeFrame))])
   182    182   
   183    183   /*
   184    184   ** Internally, the vdbe manipulates nearly all SQL values as Mem
   185    185   ** structures. Each Mem struct may cache multiple representations (string,
   186    186   ** integer etc.) of the same value.
   187    187   */
   188         -struct Mem {
          188  +struct sqlite3_value {
   189    189     union MemValue {
   190    190       double r;           /* Real value used when MEM_Real is set in flags */
   191    191       i64 i;              /* Integer value used when MEM_Int is set in flags */
   192    192       int nZero;          /* Used when bit MEM_Zero is set in flags */
   193    193       FuncDef *pDef;      /* Used only when flags==MEM_Agg */
   194    194       RowSet *pRowSet;    /* Used only when flags==MEM_RowSet */
   195    195       VdbeFrame *pFrame;  /* Used when flags==MEM_Frame */

Changes to src/vdbeaux.c.

   875    875   ** Free the space allocated for aOp and any p4 values allocated for the
   876    876   ** opcodes contained within. If aOp is not NULL it is assumed to contain 
   877    877   ** nOp entries. 
   878    878   */
   879    879   static void vdbeFreeOpArray(sqlite3 *db, Op *aOp, int nOp){
   880    880     if( aOp ){
   881    881       Op *pOp;
   882         -    for(pOp=aOp; pOp<&aOp[nOp]; pOp++){
          882  +    for(pOp=&aOp[nOp-1]; pOp>=aOp; pOp--){
   883    883         if( pOp->p4type ) freeP4(db, pOp->p4type, pOp->p4.p);
   884    884   #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS
   885    885         sqlite3DbFree(db, pOp->zComment);
   886    886   #endif     
   887    887       }
   888    888       sqlite3DbFreeNN(db, aOp);
   889    889     }

Changes to src/vdbemem.c.

    36     36     ** Mem.z = Mem.zMalloc without having to check Mem.flags&MEM_Dyn.
    37     37     ** That saves a few cycles in inner loops. */
    38     38     assert( (p->flags & MEM_Dyn)==0 || p->szMalloc==0 );
    39     39   
    40     40     /* Cannot be both MEM_Int and MEM_Real at the same time */
    41     41     assert( (p->flags & (MEM_Int|MEM_Real))!=(MEM_Int|MEM_Real) );
    42     42   
           43  +  /* Cannot be both MEM_Null and some other type */
           44  +  assert( (p->flags & MEM_Null)==0 ||
           45  +          (p->flags & (MEM_Int|MEM_Real|MEM_Str|MEM_Blob))==0 );
           46  +
    43     47     /* The szMalloc field holds the correct memory allocation size */
    44     48     assert( p->szMalloc==0
    45     49          || p->szMalloc==sqlite3DbMallocSize(p->db,p->zMalloc) );
    46     50   
    47     51     /* If p holds a string or blob, the Mem.z must point to exactly
    48     52     ** one of the following:
    49     53     **
................................................................................
   121    125     /* If the bPreserve flag is set to true, then the memory cell must already
   122    126     ** contain a valid string or blob value.  */
   123    127     assert( bPreserve==0 || pMem->flags&(MEM_Blob|MEM_Str) );
   124    128     testcase( bPreserve && pMem->z==0 );
   125    129   
   126    130     assert( pMem->szMalloc==0
   127    131          || pMem->szMalloc==sqlite3DbMallocSize(pMem->db, pMem->zMalloc) );
   128         -  if( pMem->szMalloc<n ){
   129         -    if( n<32 ) n = 32;
   130         -    if( bPreserve && pMem->szMalloc>0 && pMem->z==pMem->zMalloc ){
   131         -      pMem->z = pMem->zMalloc = sqlite3DbReallocOrFree(pMem->db, pMem->z, n);
   132         -      bPreserve = 0;
   133         -    }else{
   134         -      if( pMem->szMalloc>0 ) sqlite3DbFreeNN(pMem->db, pMem->zMalloc);
   135         -      pMem->zMalloc = sqlite3DbMallocRaw(pMem->db, n);
   136         -    }
   137         -    if( pMem->zMalloc==0 ){
   138         -      sqlite3VdbeMemSetNull(pMem);
   139         -      pMem->z = 0;
   140         -      pMem->szMalloc = 0;
   141         -      return SQLITE_NOMEM_BKPT;
   142         -    }else{
   143         -      pMem->szMalloc = sqlite3DbMallocSize(pMem->db, pMem->zMalloc);
   144         -    }
          132  +  if( n<32 ) n = 32;
          133  +  if( bPreserve && pMem->szMalloc>0 && pMem->z==pMem->zMalloc ){
          134  +    pMem->z = pMem->zMalloc = sqlite3DbReallocOrFree(pMem->db, pMem->z, n);
          135  +    bPreserve = 0;
          136  +  }else{
          137  +    if( pMem->szMalloc>0 ) sqlite3DbFreeNN(pMem->db, pMem->zMalloc);
          138  +    pMem->zMalloc = sqlite3DbMallocRaw(pMem->db, n);
          139  +  }
          140  +  if( pMem->zMalloc==0 ){
          141  +    sqlite3VdbeMemSetNull(pMem);
          142  +    pMem->z = 0;
          143  +    pMem->szMalloc = 0;
          144  +    return SQLITE_NOMEM_BKPT;
          145  +  }else{
          146  +    pMem->szMalloc = sqlite3DbMallocSize(pMem->db, pMem->zMalloc);
   145    147     }
   146    148   
   147         -  if( bPreserve && pMem->z && pMem->z!=pMem->zMalloc ){
          149  +  if( bPreserve && pMem->z && ALWAYS(pMem->z!=pMem->zMalloc) ){
   148    150       memcpy(pMem->zMalloc, pMem->z, pMem->n);
   149    151     }
   150    152     if( (pMem->flags&MEM_Dyn)!=0 ){
   151    153       assert( pMem->xDel!=0 && pMem->xDel!=SQLITE_DYNAMIC );
   152    154       pMem->xDel((void *)(pMem->z));
   153    155     }
   154    156   

Changes to src/vtab.c.

  1049   1049     unsigned char *z;
  1050   1050   
  1051   1051   
  1052   1052     /* Check to see the left operand is a column in a virtual table */
  1053   1053     if( NEVER(pExpr==0) ) return pDef;
  1054   1054     if( pExpr->op!=TK_COLUMN ) return pDef;
  1055   1055     pTab = pExpr->pTab;
  1056         -  if( NEVER(pTab==0) ) return pDef;
         1056  +  if( pTab==0 ) return pDef;
  1057   1057     if( !IsVirtual(pTab) ) return pDef;
  1058   1058     pVtab = sqlite3GetVTable(db, pTab)->pVtab;
  1059   1059     assert( pVtab!=0 );
  1060   1060     assert( pVtab->pModule!=0 );
  1061   1061     pMod = (sqlite3_module *)pVtab->pModule;
  1062   1062     if( pMod->xFindFunction==0 ) return pDef;
  1063   1063    

Changes to src/where.c.

  3433   3433     }
  3434   3434   
  3435   3435     whereLoopClear(db, pNew);
  3436   3436     return rc;
  3437   3437   }
  3438   3438   
  3439   3439   /*
  3440         -** Examine a WherePath (with the addition of the extra WhereLoop of the 5th
         3440  +** Examine a WherePath (with the addition of the extra WhereLoop of the 6th
  3441   3441   ** parameters) to see if it outputs rows in the requested ORDER BY
  3442   3442   ** (or GROUP BY) without requiring a separate sort operation.  Return N:
  3443   3443   ** 
  3444   3444   **   N>0:   N terms of the ORDER BY clause are satisfied
  3445   3445   **   N==0:  No terms of the ORDER BY clause are satisfied
  3446   3446   **   N<0:   Unknown yet how many terms of ORDER BY might be satisfied.   
  3447   3447   **
................................................................................
  3528   3528         if( wctrlFlags & WHERE_ORDERBY_LIMIT ) continue;
  3529   3529       }else{
  3530   3530         pLoop = pLast;
  3531   3531       }
  3532   3532       if( pLoop->wsFlags & WHERE_VIRTUALTABLE ){
  3533   3533         if( pLoop->u.vtab.isOrdered ) obSat = obDone;
  3534   3534         break;
         3535  +    }else{
         3536  +      pLoop->u.btree.nIdxCol = 0;
  3535   3537       }
  3536   3538       iCur = pWInfo->pTabList->a[pLoop->iTab].iCursor;
  3537   3539   
  3538   3540       /* Mark off any ORDER BY term X that is a column in the table of
  3539   3541       ** the current loop for which there is term in the WHERE
  3540   3542       ** clause of the form X IS NULL or X=? that reference only outer
  3541   3543       ** loops.
................................................................................
  3673   3675               }
  3674   3676             }
  3675   3677             if( iColumn>=0 ){
  3676   3678               pColl = sqlite3ExprCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr);
  3677   3679               if( !pColl ) pColl = db->pDfltColl;
  3678   3680               if( sqlite3StrICmp(pColl->zName, pIndex->azColl[j])!=0 ) continue;
  3679   3681             }
         3682  +          pLoop->u.btree.nIdxCol = j+1;
  3680   3683             isMatch = 1;
  3681   3684             break;
  3682   3685           }
  3683   3686           if( isMatch && (wctrlFlags & WHERE_GROUPBY)==0 ){
  3684   3687             /* Make sure the sort order is compatible in an ORDER BY clause.
  3685   3688             ** Sort order is irrelevant for a GROUP BY clause. */
  3686   3689             if( revSet ){
................................................................................
  4443   4446       sqlite3DbFree(db, pWInfo);
  4444   4447       pWInfo = 0;
  4445   4448       goto whereBeginError;
  4446   4449     }
  4447   4450     pWInfo->pParse = pParse;
  4448   4451     pWInfo->pTabList = pTabList;
  4449   4452     pWInfo->pOrderBy = pOrderBy;
         4453  +  pWInfo->pWhere = pWhere;
  4450   4454     pWInfo->pResultSet = pResultSet;
  4451   4455     pWInfo->aiCurOnePass[0] = pWInfo->aiCurOnePass[1] = -1;
  4452   4456     pWInfo->nLevel = nTabList;
  4453   4457     pWInfo->iBreak = pWInfo->iContinue = sqlite3VdbeMakeLabel(v);
  4454   4458     pWInfo->wctrlFlags = wctrlFlags;
  4455   4459     pWInfo->iLimit = iAuxArg;
  4456   4460     pWInfo->savedNQueryLoop = pParse->nQueryLoop;
................................................................................
  4753   4757         assert( iIndexCur>=0 );
  4754   4758         if( op ){
  4755   4759           sqlite3VdbeAddOp3(v, op, iIndexCur, pIx->tnum, iDb);
  4756   4760           sqlite3VdbeSetP4KeyInfo(pParse, pIx);
  4757   4761           if( (pLoop->wsFlags & WHERE_CONSTRAINT)!=0
  4758   4762            && (pLoop->wsFlags & (WHERE_COLUMN_RANGE|WHERE_SKIPSCAN))==0
  4759   4763            && (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)==0
         4764  +         && pWInfo->eDistinct!=WHERE_DISTINCT_ORDERED
  4760   4765           ){
  4761   4766             sqlite3VdbeChangeP5(v, OPFLAG_SEEKEQ); /* Hint to COMDB2 */
  4762   4767           }
  4763   4768           VdbeComment((v, "%s", pIx->zName));
  4764   4769   #ifdef SQLITE_ENABLE_COLUMN_USED_MASK
  4765   4770           {
  4766   4771             u64 colUsed = 0;
................................................................................
  4841   4846     */
  4842   4847     VdbeModuleComment((v, "End WHERE-core"));
  4843   4848     sqlite3ExprCacheClear(pParse);
  4844   4849     for(i=pWInfo->nLevel-1; i>=0; i--){
  4845   4850       int addr;
  4846   4851       pLevel = &pWInfo->a[i];
  4847   4852       pLoop = pLevel->pWLoop;
  4848         -    sqlite3VdbeResolveLabel(v, pLevel->addrCont);
  4849   4853       if( pLevel->op!=OP_Noop ){
         4854  +#ifndef SQLITE_DISABLE_SKIPAHEAD_DISTINCT
         4855  +      int addrSeek = 0;
         4856  +      Index *pIdx;
         4857  +      int n;
         4858  +      if( pWInfo->eDistinct==WHERE_DISTINCT_ORDERED
         4859  +       && (pLoop->wsFlags & WHERE_INDEXED)!=0
         4860  +       && (pIdx = pLoop->u.btree.pIndex)->hasStat1
         4861  +       && (n = pLoop->u.btree.nIdxCol)>0
         4862  +       && pIdx->aiRowLogEst[n]>=36
         4863  +      ){
         4864  +        int r1 = pParse->nMem+1;
         4865  +        int j, op;
         4866  +        for(j=0; j<n; j++){
         4867  +          sqlite3VdbeAddOp3(v, OP_Column, pLevel->iIdxCur, j, r1+j);
         4868  +        }
         4869  +        pParse->nMem += n+1;
         4870  +        op = pLevel->op==OP_Prev ? OP_SeekLT : OP_SeekGT;
         4871  +        addrSeek = sqlite3VdbeAddOp4Int(v, op, pLevel->iIdxCur, 0, r1, n);
         4872  +        VdbeCoverageIf(v, op==OP_SeekLT);
         4873  +        VdbeCoverageIf(v, op==OP_SeekGT);
         4874  +        sqlite3VdbeAddOp2(v, OP_Goto, 1, pLevel->p2);
         4875  +      }
         4876  +#endif /* SQLITE_DISABLE_SKIPAHEAD_DISTINCT */
         4877  +      /* The common case: Advance to the next row */
         4878  +      sqlite3VdbeResolveLabel(v, pLevel->addrCont);
  4850   4879         sqlite3VdbeAddOp3(v, pLevel->op, pLevel->p1, pLevel->p2, pLevel->p3);
  4851   4880         sqlite3VdbeChangeP5(v, pLevel->p5);
  4852   4881         VdbeCoverage(v);
  4853   4882         VdbeCoverageIf(v, pLevel->op==OP_Next);
  4854   4883         VdbeCoverageIf(v, pLevel->op==OP_Prev);
  4855   4884         VdbeCoverageIf(v, pLevel->op==OP_VNext);
         4885  +#ifndef SQLITE_DISABLE_SKIPAHEAD_DISTINCT
         4886  +      if( addrSeek ) sqlite3VdbeJumpHere(v, addrSeek);
         4887  +#endif
         4888  +    }else{
         4889  +      sqlite3VdbeResolveLabel(v, pLevel->addrCont);
  4856   4890       }
  4857   4891       if( pLoop->wsFlags & WHERE_IN_ABLE && pLevel->u.in.nIn>0 ){
  4858   4892         struct InLoop *pIn;
  4859   4893         int j;
  4860   4894         sqlite3VdbeResolveLabel(v, pLevel->addrNxt);
  4861   4895         for(j=pLevel->u.in.nIn, pIn=&pLevel->u.in.aInLoop[j-1]; j>0; j--, pIn--){
  4862   4896           sqlite3VdbeJumpHere(v, pIn->addrInTop+1);
................................................................................
  4971   5005               pOp->p1 = pLevel->iIdxCur;
  4972   5006             }
  4973   5007             assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 || x>=0 
  4974   5008                 || pWInfo->eOnePass );
  4975   5009           }else if( pOp->opcode==OP_Rowid ){
  4976   5010             pOp->p1 = pLevel->iIdxCur;
  4977   5011             pOp->opcode = OP_IdxRowid;
         5012  +        }else if( pOp->opcode==OP_IfNullRow ){
         5013  +          pOp->p1 = pLevel->iIdxCur;
  4978   5014           }
  4979   5015         }
  4980   5016       }
  4981   5017     }
  4982   5018   
  4983   5019     /* Final cleanup
  4984   5020     */
  4985   5021     pParse->nQueryLoop = pWInfo->savedNQueryLoop;
  4986   5022     whereInfoFree(db, pWInfo);
  4987   5023     return;
  4988   5024   }

Changes to src/whereInt.h.

   120    120     LogEst rRun;          /* Cost of running each loop */
   121    121     LogEst nOut;          /* Estimated number of output rows */
   122    122     union {
   123    123       struct {               /* Information for internal btree tables */
   124    124         u16 nEq;               /* Number of equality constraints */
   125    125         u16 nBtm;              /* Size of BTM vector */
   126    126         u16 nTop;              /* Size of TOP vector */
          127  +      u16 nIdxCol;           /* Index column used for ORDER BY */
   127    128         Index *pIndex;         /* Index used, or NULL */
   128    129       } btree;
   129    130       struct {               /* Information for virtual tables */
   130    131         int idxNum;            /* Index number */
   131    132         u8 needFree;           /* True if sqlite3_free(idxStr) is needed */
   132    133         i8 isOrdered;          /* True if satisfies ORDER BY */
   133    134         u16 omitMask;          /* Terms that may be omitted */
................................................................................
   413    414   ** planner.
   414    415   */
   415    416   struct WhereInfo {
   416    417     Parse *pParse;            /* Parsing and code generating context */
   417    418     SrcList *pTabList;        /* List of tables in the join */
   418    419     ExprList *pOrderBy;       /* The ORDER BY clause or NULL */
   419    420     ExprList *pResultSet;     /* Result set of the query */
          421  +  Expr *pWhere;             /* The complete WHERE clause */
   420    422     LogEst iLimit;            /* LIMIT if wctrlFlags has WHERE_USE_LIMIT */
   421    423     int aiCurOnePass[2];      /* OP_OpenWrite cursors for the ONEPASS opt */
   422    424     int iContinue;            /* Jump here to continue with next record */
   423    425     int iBreak;               /* Jump here to break out of the loop */
   424    426     int savedNQueryLoop;      /* pParse->nQueryLoop outside the WHERE loop */
   425    427     u16 wctrlFlags;           /* Flags originally passed to sqlite3WhereBegin() */
   426    428     u8 nLevel;                /* Number of nested loop */

Changes to src/wherecode.c.

  1034   1034         }
  1035   1035       }
  1036   1036     }else{
  1037   1037       assert( nReg==1 );
  1038   1038       sqlite3ExprCode(pParse, p, iReg);
  1039   1039     }
  1040   1040   }
         1041  +
         1042  +/* An instance of the IdxExprTrans object carries information about a
         1043  +** mapping from an expression on table columns into a column in an index
         1044  +** down through the Walker.
         1045  +*/
         1046  +typedef struct IdxExprTrans {
         1047  +  Expr *pIdxExpr;    /* The index expression */
         1048  +  int iTabCur;       /* The cursor of the corresponding table */
         1049  +  int iIdxCur;       /* The cursor for the index */
         1050  +  int iIdxCol;       /* The column for the index */
         1051  +} IdxExprTrans;
         1052  +
         1053  +/* The walker node callback used to transform matching expressions into
         1054  +** a reference to an index column for an index on an expression.
         1055  +**
         1056  +** If pExpr matches, then transform it into a reference to the index column
         1057  +** that contains the value of pExpr.
         1058  +*/
         1059  +static int whereIndexExprTransNode(Walker *p, Expr *pExpr){
         1060  +  IdxExprTrans *pX = p->u.pIdxTrans;
         1061  +  if( sqlite3ExprCompare(pExpr, pX->pIdxExpr, pX->iTabCur)==0 ){
         1062  +    pExpr->op = TK_COLUMN;
         1063  +    pExpr->iTable = pX->iIdxCur;
         1064  +    pExpr->iColumn = pX->iIdxCol;
         1065  +    pExpr->pTab = 0;
         1066  +    return WRC_Prune;
         1067  +  }else{
         1068  +    return WRC_Continue;
         1069  +  }
         1070  +}
         1071  +
         1072  +/*
         1073  +** For an indexes on expression X, locate every instance of expression X in pExpr
         1074  +** and change that subexpression into a reference to the appropriate column of
         1075  +** the index.
         1076  +*/
         1077  +static void whereIndexExprTrans(
         1078  +  Index *pIdx,      /* The Index */
         1079  +  int iTabCur,      /* Cursor of the table that is being indexed */
         1080  +  int iIdxCur,      /* Cursor of the index itself */
         1081  +  WhereInfo *pWInfo /* Transform expressions in this WHERE clause */
         1082  +){
         1083  +  int iIdxCol;               /* Column number of the index */
         1084  +  ExprList *aColExpr;        /* Expressions that are indexed */
         1085  +  Walker w;
         1086  +  IdxExprTrans x;
         1087  +  aColExpr = pIdx->aColExpr;
         1088  +  if( aColExpr==0 ) return;  /* Not an index on expressions */
         1089  +  memset(&w, 0, sizeof(w));
         1090  +  w.xExprCallback = whereIndexExprTransNode;
         1091  +  w.u.pIdxTrans = &x;
         1092  +  x.iTabCur = iTabCur;
         1093  +  x.iIdxCur = iIdxCur;
         1094  +  for(iIdxCol=0; iIdxCol<aColExpr->nExpr; iIdxCol++){
         1095  +    if( pIdx->aiColumn[iIdxCol]!=XN_EXPR ) continue;
         1096  +    assert( aColExpr->a[iIdxCol].pExpr!=0 );
         1097  +    x.iIdxCol = iIdxCol;
         1098  +    x.pIdxExpr = aColExpr->a[iIdxCol].pExpr;
         1099  +    sqlite3WalkExpr(&w, pWInfo->pWhere);
         1100  +    sqlite3WalkExprList(&w, pWInfo->pOrderBy);
         1101  +    sqlite3WalkExprList(&w, pWInfo->pResultSet);
         1102  +  }
         1103  +}
  1041   1104   
  1042   1105   /*
  1043   1106   ** Generate code for the start of the iLevel-th loop in the WHERE clause
  1044   1107   ** implementation described by pWInfo.
  1045   1108   */
  1046   1109   Bitmask sqlite3WhereCodeOneLoopStart(
  1047   1110     WhereInfo *pWInfo,   /* Complete information about the WHERE clause */
................................................................................
  1615   1678         for(j=0; j<pPk->nKeyCol; j++){
  1616   1679           k = sqlite3ColumnOfIndex(pIdx, pPk->aiColumn[j]);
  1617   1680           sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, k, iRowidReg+j);
  1618   1681         }
  1619   1682         sqlite3VdbeAddOp4Int(v, OP_NotFound, iCur, addrCont,
  1620   1683                              iRowidReg, pPk->nKeyCol); VdbeCoverage(v);
  1621   1684       }
         1685  +
         1686  +    /* If pIdx is an index on one or more expressions, then look through
         1687  +    ** all the expressions in pWInfo and try to transform matching expressions
         1688  +    ** into reference to index columns.
         1689  +    */
         1690  +    whereIndexExprTrans(pIdx, iCur, iIdxCur, pWInfo);
         1691  +
  1622   1692   
  1623   1693       /* Record the instruction used to terminate the loop. */
  1624   1694       if( pLoop->wsFlags & WHERE_ONEROW ){
  1625   1695         pLevel->op = OP_Noop;
  1626   1696       }else if( bRev ){
  1627   1697         pLevel->op = OP_Prev;
  1628   1698       }else{

Changes to src/whereexpr.c.

   826    826     return mask;
   827    827   }
   828    828   
   829    829   /*
   830    830   ** Expression pExpr is one operand of a comparison operator that might
   831    831   ** be useful for indexing.  This routine checks to see if pExpr appears
   832    832   ** in any index.  Return TRUE (1) if pExpr is an indexed term and return
   833         -** FALSE (0) if not.  If TRUE is returned, also set *piCur to the cursor
   834         -** number of the table that is indexed and *piColumn to the column number
          833  +** FALSE (0) if not.  If TRUE is returned, also set aiCurCol[0] to the cursor
          834  +** number of the table that is indexed and aiCurCol[1] to the column number
   835    835   ** of the column that is indexed, or XN_EXPR (-2) if an expression is being
   836    836   ** indexed.
   837    837   **
   838    838   ** If pExpr is a TK_COLUMN column reference, then this routine always returns
   839    839   ** true even if that particular column is not indexed, because the column
   840    840   ** might be added to an automatic index later.
   841    841   */
   842         -static int exprMightBeIndexed(
          842  +static SQLITE_NOINLINE int exprMightBeIndexed2(
   843    843     SrcList *pFrom,        /* The FROM clause */
   844         -  int op,                /* The specific comparison operator */
   845    844     Bitmask mPrereq,       /* Bitmask of FROM clause terms referenced by pExpr */
   846         -  Expr *pExpr,           /* An operand of a comparison operator */
   847         -  int *piCur,            /* Write the referenced table cursor number here */
   848         -  int *piColumn          /* Write the referenced table column number here */
          845  +  int *aiCurCol,         /* Write the referenced table cursor and column here */
          846  +  Expr *pExpr            /* An operand of a comparison operator */
   849    847   ){
   850    848     Index *pIdx;
   851    849     int i;
   852    850     int iCur;
   853         -
          851  +  for(i=0; mPrereq>1; i++, mPrereq>>=1){}
          852  +  iCur = pFrom->a[i].iCursor;
          853  +  for(pIdx=pFrom->a[i].pTab->pIndex; pIdx; pIdx=pIdx->pNext){
          854  +    if( pIdx->aColExpr==0 ) continue;
          855  +    for(i=0; i<pIdx->nKeyCol; i++){
          856  +      if( pIdx->aiColumn[i]!=XN_EXPR ) continue;
          857  +      if( sqlite3ExprCompareSkip(pExpr, pIdx->aColExpr->a[i].pExpr, iCur)==0 ){
          858  +        aiCurCol[0] = iCur;
          859  +        aiCurCol[1] = XN_EXPR;
          860  +        return 1;
          861  +      }
          862  +    }
          863  +  }
          864  +  return 0;
          865  +}
          866  +static int exprMightBeIndexed(
          867  +  SrcList *pFrom,        /* The FROM clause */
          868  +  Bitmask mPrereq,       /* Bitmask of FROM clause terms referenced by pExpr */
          869  +  int *aiCurCol,         /* Write the referenced table cursor & column here */
          870  +  Expr *pExpr,           /* An operand of a comparison operator */
          871  +  int op                 /* The specific comparison operator */
          872  +){
   854    873     /* If this expression is a vector to the left or right of a 
   855    874     ** inequality constraint (>, <, >= or <=), perform the processing 
   856    875     ** on the first element of the vector.  */
   857    876     assert( TK_GT+1==TK_LE && TK_GT+2==TK_LT && TK_GT+3==TK_GE );
   858    877     assert( TK_IS<TK_GE && TK_ISNULL<TK_GE && TK_IN<TK_GE );
   859    878     assert( op<=TK_GE );
   860    879     if( pExpr->op==TK_VECTOR && (op>=TK_GT && ALWAYS(op<=TK_GE)) ){
   861    880       pExpr = pExpr->x.pList->a[0].pExpr;
   862    881     }
   863    882   
   864    883     if( pExpr->op==TK_COLUMN ){
   865         -    *piCur = pExpr->iTable;
   866         -    *piColumn = pExpr->iColumn;
          884  +    aiCurCol[0] = pExpr->iTable;
          885  +    aiCurCol[1] = pExpr->iColumn;
   867    886       return 1;
   868    887     }
   869    888     if( mPrereq==0 ) return 0;                 /* No table references */
   870    889     if( (mPrereq&(mPrereq-1))!=0 ) return 0;   /* Refs more than one table */
   871         -  for(i=0; mPrereq>1; i++, mPrereq>>=1){}
   872         -  iCur = pFrom->a[i].iCursor;
   873         -  for(pIdx=pFrom->a[i].pTab->pIndex; pIdx; pIdx=pIdx->pNext){
   874         -    if( pIdx->aColExpr==0 ) continue;
   875         -    for(i=0; i<pIdx->nKeyCol; i++){
   876         -      if( pIdx->aiColumn[i]!=XN_EXPR ) continue;
   877         -      if( sqlite3ExprCompareSkip(pExpr, pIdx->aColExpr->a[i].pExpr, iCur)==0 ){
   878         -        *piCur = iCur;
   879         -        *piColumn = XN_EXPR;
   880         -        return 1;
   881         -      }
   882         -    }
   883         -  }
   884         -  return 0;
          890  +  return exprMightBeIndexed2(pFrom,mPrereq,aiCurCol,pExpr);
   885    891   }
   886    892   
   887    893   /*
   888    894   ** The input to this routine is an WhereTerm structure with only the
   889    895   ** "pExpr" field filled in.  The job of this routine is to analyze the
   890    896   ** subexpression and populate all the other fields of the WhereTerm
   891    897   ** structure.
................................................................................
   957    963       }
   958    964     }
   959    965     pTerm->prereqAll = prereqAll;
   960    966     pTerm->leftCursor = -1;
   961    967     pTerm->iParent = -1;
   962    968     pTerm->eOperator = 0;
   963    969     if( allowedOp(op) ){
   964         -    int iCur, iColumn;
          970  +    int aiCurCol[2];
   965    971       Expr *pLeft = sqlite3ExprSkipCollate(pExpr->pLeft);
   966    972       Expr *pRight = sqlite3ExprSkipCollate(pExpr->pRight);
   967    973       u16 opMask = (pTerm->prereqRight & prereqLeft)==0 ? WO_ALL : WO_EQUIV;
   968    974   
   969    975       if( pTerm->iField>0 ){
   970    976         assert( op==TK_IN );
   971    977         assert( pLeft->op==TK_VECTOR );
   972    978         pLeft = pLeft->x.pList->a[pTerm->iField-1].pExpr;
   973    979       }
   974    980   
   975         -    if( exprMightBeIndexed(pSrc, op, prereqLeft, pLeft, &iCur, &iColumn) ){
   976         -      pTerm->leftCursor = iCur;
   977         -      pTerm->u.leftColumn = iColumn;
          981  +    if( exprMightBeIndexed(pSrc, prereqLeft, aiCurCol, pLeft, op) ){
          982  +      pTerm->leftCursor = aiCurCol[0];
          983  +      pTerm->u.leftColumn = aiCurCol[1];
   978    984         pTerm->eOperator = operatorMask(op) & opMask;
   979    985       }
   980    986       if( op==TK_IS ) pTerm->wtFlags |= TERM_IS;
   981    987       if( pRight 
   982         -     && exprMightBeIndexed(pSrc, op, pTerm->prereqRight, pRight, &iCur,&iColumn)
          988  +     && exprMightBeIndexed(pSrc, pTerm->prereqRight, aiCurCol, pRight, op)
   983    989       ){
   984    990         WhereTerm *pNew;
   985    991         Expr *pDup;
   986    992         u16 eExtraOp = 0;        /* Extra bits for pNew->eOperator */
   987    993         assert( pTerm->iField==0 );
   988    994         if( pTerm->leftCursor>=0 ){
   989    995           int idxNew;
................................................................................
  1005   1011             eExtraOp = WO_EQUIV;
  1006   1012           }
  1007   1013         }else{
  1008   1014           pDup = pExpr;
  1009   1015           pNew = pTerm;
  1010   1016         }
  1011   1017         exprCommute(pParse, pDup);
  1012         -      pNew->leftCursor = iCur;
  1013         -      pNew->u.leftColumn = iColumn;
         1018  +      pNew->leftCursor = aiCurCol[0];
         1019  +      pNew->u.leftColumn = aiCurCol[1];
  1014   1020         testcase( (prereqLeft | extraRight) != prereqLeft );
  1015   1021         pNew->prereqRight = prereqLeft | extraRight;
  1016   1022         pNew->prereqAll = prereqAll;
  1017   1023         pNew->eOperator = (operatorMask(pDup->op) + eExtraOp) & opMask;
  1018   1024       }
  1019   1025     }
  1020   1026   

Added test/distinct2.test.

            1  +# 2016-04-15
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this script is DISTINCT queries using the skip-ahead 
           13  +# optimization.
           14  +#
           15  +
           16  +set testdir [file dirname $argv0]
           17  +source $testdir/tester.tcl
           18  +
           19  +set testprefix distinct2
           20  +
           21  +do_execsql_test 100 {
           22  +  CREATE TABLE t1(x INTEGER PRIMARY KEY);
           23  +  INSERT INTO t1 VALUES(0),(1),(2);
           24  +  CREATE TABLE t2 AS
           25  +     SELECT DISTINCT a.x AS aa, b.x AS bb
           26  +      FROM t1 a, t1 b;
           27  +  SELECT *, '|' FROM t2 ORDER BY aa, bb;
           28  +} {0 0 | 0 1 | 0 2 | 1 0 | 1 1 | 1 2 | 2 0 | 2 1 | 2 2 |}
           29  +do_execsql_test 110 {
           30  +  DROP TABLE t2;
           31  +  CREATE TABLE t2 AS
           32  +     SELECT DISTINCT a.x AS aa, b.x AS bb
           33  +       FROM t1 a, t1 b
           34  +      WHERE a.x IN t1 AND b.x IN t1;
           35  +  SELECT *, '|' FROM t2 ORDER BY aa, bb;
           36  +} {0 0 | 0 1 | 0 2 | 1 0 | 1 1 | 1 2 | 2 0 | 2 1 | 2 2 |}
           37  +do_execsql_test 120 {
           38  +  CREATE TABLE t102 (i0 TEXT UNIQUE NOT NULL);
           39  +  INSERT INTO t102 VALUES ('0'),('1'),('2');
           40  +  DROP TABLE t2;
           41  +  CREATE TABLE t2 AS
           42  +    SELECT DISTINCT * 
           43  +    FROM t102 AS t0 
           44  +    JOIN t102 AS t4 ON (t2.i0 IN t102)
           45  +    NATURAL JOIN t102 AS t3
           46  +    JOIN t102 AS t1 ON (t0.i0 IN t102)
           47  +    JOIN t102 AS t2 ON (t2.i0=+t0.i0 OR (t0.i0<>500 AND t2.i0=t1.i0));
           48  +  SELECT *, '|' FROM t2 ORDER BY 1, 2, 3, 4, 5;
           49  +} {0 0 0 0 | 0 0 1 0 | 0 0 1 1 | 0 0 2 0 | 0 0 2 2 | 0 1 0 0 | 0 1 1 0 | 0 1 1 1 | 0 1 2 0 | 0 1 2 2 | 0 2 0 0 | 0 2 1 0 | 0 2 1 1 | 0 2 2 0 | 0 2 2 2 | 1 0 0 0 | 1 0 0 1 | 1 0 1 1 | 1 0 2 1 | 1 0 2 2 | 1 1 0 0 | 1 1 0 1 | 1 1 1 1 | 1 1 2 1 | 1 1 2 2 | 1 2 0 0 | 1 2 0 1 | 1 2 1 1 | 1 2 2 1 | 1 2 2 2 | 2 0 0 0 | 2 0 0 2 | 2 0 1 1 | 2 0 1 2 | 2 0 2 2 | 2 1 0 0 | 2 1 0 2 | 2 1 1 1 | 2 1 1 2 | 2 1 2 2 | 2 2 0 0 | 2 2 0 2 | 2 2 1 1 | 2 2 1 2 | 2 2 2 2 |}
           50  +
           51  +do_execsql_test 400 {
           52  +  CREATE TABLE t4(a,b,c,d,e,f,g,h,i,j);
           53  +  INSERT INTO t4 VALUES(0,1,2,3,4,5,6,7,8,9);
           54  +  INSERT INTO t4 SELECT * FROM t4;
           55  +  INSERT INTO t4 SELECT * FROM t4;
           56  +  CREATE INDEX t4x ON t4(c,d,e);
           57  +  SELECT DISTINCT a,b,c FROM t4 WHERE a=0 AND b=1;
           58  +} {0 1 2}
           59  +do_execsql_test 410 {
           60  +  SELECT DISTINCT a,b,c,d FROM t4 WHERE a=0 AND b=1;
           61  +} {0 1 2 3}
           62  +do_execsql_test 411 {
           63  +  SELECT DISTINCT d,a,b,c FROM t4 WHERE a=0 AND b=1;
           64  +} {3 0 1 2}
           65  +do_execsql_test 420 {
           66  +  SELECT DISTINCT a,b,c,d,e FROM t4 WHERE a=0 AND b=1;
           67  +} {0 1 2 3 4}
           68  +do_execsql_test 430 {
           69  +  SELECT DISTINCT a,b,c,d,e,f FROM t4 WHERE a=0 AND b=1;
           70  +} {0 1 2 3 4 5}
           71  +
           72  +do_execsql_test 500 {
           73  +  CREATE TABLE t5(a INT, b INT);
           74  +  CREATE UNIQUE INDEX t5x ON t5(a+b);
           75  +  INSERT INTO t5(a,b) VALUES(0,0),(1,0),(1,1),(0,3);
           76  +  CREATE TEMP TABLE out AS SELECT DISTINCT a+b FROM t5;
           77  +  SELECT * FROM out ORDER BY 1;
           78  +} {0 1 2 3}
           79  +
           80  +do_execsql_test 600 {
           81  +  CREATE TABLE t6a(x INTEGER PRIMARY KEY);
           82  +  INSERT INTO t6a VALUES(1);
           83  +  CREATE TABLE t6b(y INTEGER PRIMARY KEY);
           84  +  INSERT INTO t6b VALUES(2),(3);
           85  +  SELECT DISTINCT x, x FROM t6a, t6b;
           86  +} {1 1}
           87  +
           88  +do_execsql_test 700 {
           89  +  CREATE TABLE t7(a, b, c);
           90  +  WITH s(i) AS (
           91  +    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE (i+1)<200
           92  +  )
           93  +  INSERT INTO t7 SELECT i/100, i/50, i FROM s;
           94  +}
           95  +do_execsql_test 710 {
           96  +  SELECT DISTINCT a, b FROM t7;
           97  +} {
           98  +  0 0    0 1
           99  +  1 2    1 3
          100  +}
          101  +do_execsql_test 720 {
          102  +  SELECT DISTINCT a, b+1 FROM t7;
          103  +} {
          104  +  0 1    0 2
          105  +  1 3    1 4
          106  +}
          107  +do_execsql_test 730 {
          108  +  CREATE INDEX i7 ON t7(a, b+1);
          109  +  ANALYZE;
          110  +  SELECT DISTINCT a, b+1 FROM t7;
          111  +} {
          112  +  0 1    0 2
          113  +  1 3    1 4
          114  +}
          115  +
          116  +do_execsql_test 800 {
          117  +  CREATE TABLE t8(a, b, c);
          118  +  WITH s(i) AS (
          119  +    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE (i+1)<100
          120  +  )
          121  +  INSERT INTO t8 SELECT i/40, i/20, i/40 FROM s;
          122  +}
          123  +
          124  +do_execsql_test 820 {
          125  +  SELECT DISTINCT a, b, c FROM t8;
          126  +} {
          127  +  0 0 0    0 1 0
          128  +  1 2 1    1 3 1
          129  +  2 4 2
          130  +}
          131  +
          132  +do_execsql_test 820 {
          133  +  SELECT DISTINCT a, b, c FROM t8 WHERE b=3;
          134  +} {1 3 1}
          135  +
          136  +do_execsql_test 830 {
          137  +  CREATE INDEX i8 ON t8(a, c);
          138  +  ANALYZE;
          139  +  SELECT DISTINCT a, b, c FROM t8 WHERE b=3;
          140  +} {1 3 1}
          141  +
          142  +do_execsql_test 900 {
          143  +  CREATE TABLE t9(v);
          144  +  INSERT INTO t9 VALUES 
          145  +    ('abcd'), ('Abcd'), ('aBcd'), ('ABcd'), ('abCd'), ('AbCd'), ('aBCd'), 
          146  +    ('ABCd'), ('abcD'), ('AbcD'), ('aBcD'), ('ABcD'), ('abCD'), ('AbCD'), 
          147  +    ('aBCD'), ('ABCD'),
          148  +    ('wxyz'), ('Wxyz'), ('wXyz'), ('WXyz'), ('wxYz'), ('WxYz'), ('wXYz'), 
          149  +    ('WXYz'), ('wxyZ'), ('WxyZ'), ('wXyZ'), ('WXyZ'), ('wxYZ'), ('WxYZ'), 
          150  +    ('wXYZ'), ('WXYZ');
          151  +}
          152  +
          153  +do_execsql_test 910 {
          154  +  SELECT DISTINCT v COLLATE NOCASE, v FROM t9 ORDER BY +v;
          155  +} {
          156  +  ABCD ABCD ABCd ABCd ABcD ABcD ABcd ABcd AbCD
          157  +  AbCD AbCd AbCd AbcD AbcD Abcd Abcd
          158  +  WXYZ WXYZ WXYz WXYz WXyZ WXyZ WXyz WXyz WxYZ
          159  +  WxYZ WxYz WxYz WxyZ WxyZ Wxyz Wxyz
          160  +  aBCD aBCD aBCd aBCd aBcD aBcD aBcd aBcd abCD
          161  +  abCD abCd abCd abcD abcD abcd abcd
          162  +  wXYZ wXYZ wXYz wXYz wXyZ wXyZ wXyz wXyz wxYZ
          163  +  wxYZ wxYz wxYz wxyZ wxyZ wxyz wxyz
          164  +}
          165  +
          166  +do_execsql_test 920 {
          167  +  CREATE INDEX i9 ON t9(v COLLATE NOCASE, v);
          168  +  ANALYZE;
          169  +
          170  +  SELECT DISTINCT v COLLATE NOCASE, v FROM t9 ORDER BY +v;
          171  +} {
          172  +  ABCD ABCD ABCd ABCd ABcD ABcD ABcd ABcd AbCD
          173  +  AbCD AbCd AbCd AbcD AbcD Abcd Abcd
          174  +  WXYZ WXYZ WXYz WXYz WXyZ WXyZ WXyz WXyz WxYZ
          175  +  WxYZ WxYz WxYz WxyZ WxyZ Wxyz Wxyz
          176  +  aBCD aBCD aBCd aBCd aBcD aBcD aBcd aBcd abCD
          177  +  abCD abCd abCd abcD abcD abcd abcd
          178  +  wXYZ wXYZ wXYz wXYz wXyZ wXyZ wXyz wXyz wxYZ
          179  +  wxYZ wxYz wxYz wxyZ wxyZ wxyz wxyz
          180  +}
          181  +
          182  +
          183  +finish_test

Changes to test/fkey5.test.

   384    384     PRAGMA foreign_key_check(k2);
   385    385   } {}
   386    386   do_execsql_test 9.4 {
   387    387     INSERT INTO k2 VALUES('six', 'seven');
   388    388     PRAGMA foreign_key_check(k2);
   389    389   } {k2 3 s1 0}
   390    390   
          391  +#-------------------------------------------------------------------------
          392  +# Test using a WITHOUT ROWID table as the child table with an INTEGER 
          393  +# PRIMARY KEY as the parent key.
          394  +#
          395  +reset_db
          396  +do_execsql_test 10.1 {
          397  +  CREATE TABLE p30 (id INTEGER PRIMARY KEY);
          398  +  CREATE TABLE IF NOT EXISTS c30 (
          399  +      line INTEGER, 
          400  +      master REFERENCES p30(id), 
          401  +      PRIMARY KEY(master)
          402  +  ) WITHOUT ROWID;
          403  +
          404  +  INSERT INTO p30 (id) VALUES (1);
          405  +  INSERT INTO c30 (master, line)  VALUES (1, 999);
          406  +}
          407  +do_execsql_test 10.2 {
          408  +  PRAGMA foreign_key_check;
          409  +}
          410  +do_execsql_test 10.3 {
          411  +  INSERT INTO c30 VALUES(45, 45);
          412  +  PRAGMA foreign_key_check;
          413  +} {c30 {} p30 0}
          414  +
          415  +#-------------------------------------------------------------------------
          416  +# Test "foreign key mismatch" errors.
          417  +#
          418  +reset_db
          419  +do_execsql_test 11.0 {
          420  +  CREATE TABLE tt(y);
          421  +  CREATE TABLE c11(x REFERENCES tt(y));
          422  +}
          423  +do_catchsql_test 11.1 {
          424  +  PRAGMA foreign_key_check;
          425  +} {1 {foreign key mismatch - "c11" referencing "tt"}}
   391    426   
   392    427   finish_test

Changes to test/fts3aa.test.

   239    239   do_execsql_test 8.4 {
   240    240     SELECT docid FROM t0 WHERE t0 MATCH 'abc';
   241    241   } {6 5}
   242    242   do_execsql_test 8.5 {
   243    243     SELECT docid FROM t0 WHERE t0 MATCH '"abc abc"';
   244    244   } {}
   245    245   
          246  +do_execsql_test 9.1 {
          247  +  CREATE VIRTUAL TABLE t9 USING fts4(a, "", '---');
          248  +}
          249  +do_execsql_test 9.2 {
          250  +  CREATE VIRTUAL TABLE t10 USING fts3(<, b, c);
          251  +}
   246    252   
   247    253   finish_test

Added test/fts3corrupt3.test.

            1  +# 2010 October 27
            2  +#
            3  +#    May you do good and not evil.
            4  +#    May you find forgiveness for yourself and forgive others.
            5  +#    May you share freely, never taking more than you give.
            6  +#
            7  +#***********************************************************************
            8  +# Test that the FTS3 extension does not crash when it encounters a
            9  +# corrupt data structure on disk.
           10  +#
           11  +
           12  +
           13  +set testdir [file dirname $argv0]
           14  +source $testdir/tester.tcl
           15  +
           16  +# If SQLITE_ENABLE_FTS3 is not defined, omit this file.
           17  +ifcapable !fts3 { finish_test ; return }
           18  +
           19  +set ::testprefix fts3corrupt3
           20  +
           21  +#-------------------------------------------------------------------------
           22  +# Test that fts3 does not choke on an oversized varint.
           23  +#
           24  +do_execsql_test 1.0 {
           25  +  PRAGMA page_size = 512;
           26  +  CREATE VIRTUAL TABLE t1 USING fts3;
           27  +  BEGIN;
           28  +    INSERT INTO t1 VALUES('one');
           29  +    INSERT INTO t1 VALUES('one');
           30  +    INSERT INTO t1 VALUES('one');
           31  +  COMMIT;
           32  +}
           33  +do_execsql_test 1.1 {
           34  +  SELECT quote(root) from t1_segdir;
           35  +} {X'00036F6E6509010200010200010200'}
           36  +do_execsql_test 1.2 {
           37  +  UPDATE t1_segdir SET root = X'00036F6E650EFFFFFFFFFFFFFFFFFFFFFFFF0200';
           38  +}
           39  +do_catchsql_test 1.3 {
           40  +  SELECT rowid FROM t1 WHERE t1 MATCH 'one'
           41  +} {0 -1}
           42  +
           43  +#-------------------------------------------------------------------------
           44  +# Interior node with the prefix or suffix count of an entry set to a
           45  +# negative value.
           46  +#
           47  +set doc1 [string repeat "x " 600]
           48  +set doc2 [string repeat "y " 600]
           49  +set doc3 [string repeat "z " 600]
           50  +
           51  +do_execsql_test 2.0 {
           52  +  CREATE VIRTUAL TABLE t2 USING fts3;
           53  +  BEGIN;
           54  +    INSERT INTO t2 VALUES($doc1);
           55  +    INSERT INTO t2 VALUES($doc2);
           56  +    INSERT INTO t2 VALUES($doc3);
           57  +  COMMIT;
           58  +}
           59  +do_execsql_test 2.1 {
           60  +  SELECT quote(root) from t2_segdir;
           61  +} {X'0101017900017A'}
           62  +
           63  +
           64  +
           65  +finish_test

Changes to test/fts3fault.test.

    14     14   source $testdir/tester.tcl
    15     15   
    16     16   set ::testprefix fts3fault
    17     17   
    18     18   # If SQLITE_ENABLE_FTS3 is not defined, omit this file.
    19     19   ifcapable !fts3 { finish_test ; return }
    20     20   
    21         -if 0 {
    22         -
    23     21   # Test error handling in the sqlite3Fts3Init() function. This is the 
    24     22   # function that registers the FTS3 module and various support functions
    25     23   # with SQLite.
    26     24   #
    27     25   do_faultsim_test 1 -body { 
    28     26     sqlite3 db test.db 
    29     27     expr 0
................................................................................
   156    154   } -test {
   157    155     faultsim_test_result {1 {unrecognized parameter: matchnfo=fts3}} \
   158    156                          {1 {vtable constructor failed: t1}} \
   159    157                          {1 {SQL logic error or missing database}}
   160    158   }
   161    159   
   162    160   
   163         -}
   164         -
   165    161   proc mit {blob} {
   166    162     set scan(littleEndian) i*
   167    163     set scan(bigEndian) I*
   168    164     binary scan $blob $scan($::tcl_platform(byteOrder)) r
   169    165     return $r
   170    166   }
   171    167   
................................................................................
   229    225   do_faultsim_test 9.1 -prep {
   230    226     faultsim_restore_and_reopen
   231    227   } -body {
   232    228     execsql { SELECT offsets(t9) FROM t9 WHERE t9 MATCH 'to*' }
   233    229   } -test {
   234    230     faultsim_test_result {0 {{0 0 20 39 0 0 64 2}}}
   235    231   }
          232  +
          233  +do_faultsim_test 10.1 -prep {
          234  +  faultsim_delete_and_reopen
          235  +} -body {
          236  +  execsql { CREATE VIRTUAL TABLE t1 USING fts4(a, b, languageid=d) }
          237  +} -test {
          238  +  faultsim_test_result {0 {}}
          239  +}
   236    240   
   237    241   finish_test

Changes to test/fts3fault2.test.

   169    169     faultsim_restore_and_reopen
   170    170     db eval {SELECT * FROM sqlite_master}
   171    171   } -body {
   172    172     execsql { SELECT docid FROM t6 WHERE t6 MATCH '"a* b"' }
   173    173   } -test {
   174    174     faultsim_test_result {0 -1}
   175    175   }
          176  +
          177  +#-------------------------------------------------------------------------
          178  +# Inject faults into a query for an N-byte prefix that uses a prefix=N+1
          179  +# index.
          180  +reset_db
          181  +do_execsql_test 7.0 {
          182  +  CREATE VIRTUAL TABLE t7 USING fts4(x,prefix=2);
          183  +  INSERT INTO t7 VALUES('the quick brown fox');
          184  +  INSERT INTO t7 VALUES('jumped over the');
          185  +  INSERT INTO t7 VALUES('lazy dog');
          186  +}
          187  +do_faultsim_test 7.1 -faults oom* -body {
          188  +  execsql { SELECT docid FROM t7 WHERE t7 MATCH 't*' }
          189  +} -test {
          190  +  faultsim_test_result {0 {1 2}}
          191  +}
          192  +
          193  +#-------------------------------------------------------------------------
          194  +# Inject faults into a opening an existing fts3 table that has been 
          195  +# upgraded to add an %_stat table.
          196  +#
          197  +reset_db
          198  +do_execsql_test 8.0 {
          199  +  CREATE VIRTUAL TABLE t8 USING fts3;
          200  +  INSERT INTO t8 VALUES('the quick brown fox');
          201  +  INSERT INTO t8 VALUES('jumped over the');
          202  +  INSERT INTO t8 VALUES('lazy dog');
          203  +  INSERT INTO t8(t8) VALUES('automerge=8');
          204  +  SELECT name FROM sqlite_master WHERE name LIKE 't8%';
          205  +} {
          206  +  t8 t8_content t8_segments t8_segdir t8_stat
          207  +}
          208  +faultsim_save_and_close
          209  +
          210  +do_faultsim_test 8.1 -faults oom* -prep {
          211  +  faultsim_restore_and_reopen
          212  +} -body {
          213  +  execsql { INSERT INTO t8 VALUES('one two three') }
          214  +} -test {
          215  +  faultsim_test_result {0 {}}
          216  +}
          217  +
          218  +do_faultsim_test 8.2 -faults oom* -prep {
          219  +  faultsim_restore_and_reopen
          220  +} -body {
          221  +  execsql { ALTER TABLE t8 RENAME TO t8ii }
          222  +} -test {
          223  +  faultsim_test_result {0 {}}
          224  +}
          225  +
          226  +#-------------------------------------------------------------------------
          227  +reset_db
          228  +set chunkconfig [fts3_configure_incr_load 1 1]
          229  +do_execsql_test 9.0 {
          230  +  PRAGMA page_size = 512;
          231  +  CREATE VIRTUAL TABLE t9 USING fts3;
          232  +  WITH s(i) AS (
          233  +    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<50
          234  +  )
          235  +  INSERT INTO t9 SELECT 'one two three' FROM s;
          236  +}
          237  +
          238  +do_faultsim_test 8.2 -faults io* -body {
          239  +  execsql { SELECT count(*) FROM t9 WHERE t9 MATCH '"one two three"' }
          240  +} -test {
          241  +  faultsim_test_result {0 50}
          242  +}
          243  +
          244  +eval fts3_configure_incr_load $chunkconfig
          245  +
   176    246   
   177    247   finish_test
          248  +
          249  +

Added test/fts3misc.test.

            1  +# 2017 March 22
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this script is testing the FTS3 module.
           13  +#
           14  +
           15  +set testdir [file dirname $argv0]
           16  +source $testdir/tester.tcl
           17  +set testprefix fts3misc
           18  +
           19  +# If SQLITE_ENABLE_FTS3 is defined, omit this file.
           20  +ifcapable !fts3 {
           21  +  finish_test
           22  +  return
           23  +}
           24  +
           25  +#-------------------------------------------------------------------------
           26  +# A self-join.
           27  +#
           28  +do_execsql_test 1.0 {
           29  +  CREATE VIRTUAL TABLE t1 USING fts3(a, b);
           30  +  INSERT INTO t1 VALUES('one', 'i');
           31  +  INSERT INTO t1 VALUES('one', 'ii');
           32  +  INSERT INTO t1 VALUES('two', 'i');
           33  +  INSERT INTO t1 VALUES('two', 'ii');
           34  +}
           35  +
           36  +do_execsql_test 1.1 {
           37  +  SELECT a.a, b.b FROM t1 a, t1 b WHERE a.t1 MATCH 'two' AND b.t1 MATCH 'i'
           38  +} {two i two i two i two i}
           39  +
           40  +#-------------------------------------------------------------------------
           41  +# FTS tables with 128 or more columns.
           42  +#
           43  +proc v1 {v} {
           44  +  set vector [list a b c d e f g h]
           45  +  set res [list]
           46  +  for {set i 0} {$i<8} {incr i} {
           47  +    if {$v & (1 << $i)} { lappend res [lindex $vector $i] }
           48  +  }
           49  +  set res
           50  +}
           51  +proc v2 {v} {
           52  +  set vector [list d e f g h i j k]
           53  +  set res [list]
           54  +  for {set i 0} {$i<8} {incr i} {
           55  +    if {$v & (1 << $i)} { lappend res [lindex $vector $i] }
           56  +  }
           57  +  set res
           58  +}
           59  +db func v1 v1
           60  +db func v2 v2
           61  +
           62  +do_test 2.0 {
           63  +  set cols [list]
           64  +  for {set i 0} {$i<200} {incr i} {
           65  +    lappend cols "c$i"
           66  +  }
           67  +  execsql "CREATE VIRTUAL TABLE t2 USING fts3([join $cols ,])"
           68  +  execsql {
           69  +    WITH data(i) AS (
           70  +      SELECT 1 UNION ALL SELECT i+1 FROM data WHERE i<200
           71  +    )
           72  +    INSERT INTO t2(c198, c199) SELECT v1(i), v2(i) FROM data;
           73  +  }
           74  +} {}
           75  +do_execsql_test 2.1 {
           76  +  SELECT rowid FROM t2 WHERE t2 MATCH '"a b c"'
           77  +} {
           78  +  7 15 23 31 39 47 55 63 71 79 87 95 103 111 
           79  +  119 127 135 143 151 159 167 175 183 191 199
           80  +}
           81  +do_execsql_test 2.2 {
           82  +  SELECT rowid FROM t2 WHERE t2 MATCH '"g h i"'
           83  +} {
           84  +  56 57 58 59 60 61 62 63 120 121 122 123 124 
           85  +  125 126 127 184 185 186 187 188 189 190 191
           86  +}
           87  +do_execsql_test 2.3 {
           88  +  SELECT rowid FROM t2 WHERE t2 MATCH '"i h"'
           89  +} {
           90  +}
           91  +do_execsql_test 2.4 {
           92  +  SELECT rowid FROM t2 WHERE t2 MATCH '"f e"'
           93  +} {
           94  +}
           95  +do_execsql_test 2.5 {
           96  +  SELECT rowid FROM t2 WHERE t2 MATCH '"e f"'
           97  +} {
           98  +  6 7 14 15 22 23 30 31 38 39 46 47 48 49 50 51 52 53 54 55 56 
           99  +  57 58 59 60 61 62 63 70 71 78 79 86 87 94 95 102 103 110 
          100  +  111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127
          101  +  134 135 142 143 150 151 158 159 166 167 174 175 176 177 178 179 180 
          102  +  181 182 183 184 185 186 187 188 189 190 191 198 199
          103  +}
          104  +
          105  +#-------------------------------------------------------------------------
          106  +# Range constraints on the docid using non-integer values.
          107  +#
          108  +do_execsql_test 2.6 {
          109  +  SELECT rowid FROM t2 WHERE t2 MATCH 'e' AND rowid BETWEEN NULL AND 45;
          110  +} {}
          111  +do_execsql_test 2.7 {
          112  +  SELECT rowid FROM t2 WHERE t2 MATCH 'e' AND rowid BETWEEN 11.5 AND 48.2;
          113  +} {
          114  +  14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 
          115  +  29 30 31 34 35 38 39 42 43 46 47 48
          116  +}
          117  +do_execsql_test 2.8 {
          118  +  SELECT rowid FROM t2 WHERE t2 MATCH 'e' AND rowid BETWEEN '11.5' AND '48.2';
          119  +} {
          120  +  14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 
          121  +  29 30 31 34 35 38 39 42 43 46 47 48
          122  +}
          123  +
          124  +#-------------------------------------------------------------------------
          125  +# Phrase query tests. 
          126  +#
          127  +do_execsql_test 3.1.1 {
          128  +  CREATE VIRTUAL TABLE t3 USING fts3;
          129  +  INSERT INTO t3 VALUES('a b c');
          130  +  INSERT INTO t3 VALUES('d e f');
          131  +  INSERT INTO t3 VALUES('a b d');
          132  +  INSERT INTO t3 VALUES('1 2 3 4 5 6 7 8 9 10 11');
          133  +}
          134  +do_execsql_test 3.1.2 {
          135  +  SELECT * FROM t3 WHERE t3 MATCH '"a b x y"' ORDER BY docid DESC
          136  +}
          137  +do_execsql_test 3.1.3 {
          138  +  SELECT * FROM t3 WHERE t3 MATCH '"a b c" OR "a b x y"' ORDER BY docid DESC
          139  +} {{a b c}}
          140  +do_execsql_test 3.1.4 {
          141  +  SELECT * FROM t3 WHERE t3 MATCH '"a* b* x* a*"'
          142  +}
          143  +do_execsql_test 3.1.5 {
          144  +  SELECT rowid FROM t3 WHERE t3 MATCH '"2 3 4 5 6 7 8 9"'
          145  +} {4}
          146  +
          147  +#-------------------------------------------------------------------------
          148  +#
          149  +reset_db
          150  +do_execsql_test 4.0 {
          151  +  PRAGMA page_size = 512;
          152  +  CREATE VIRTUAL TABLE t4 USING fts4;
          153  +  WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<8000 )
          154  +  INSERT INTO t4 SELECT 'a b c a b c a b c' FROM s;
          155  +}
          156  +do_execsql_test 4.1 {
          157  +  SELECT count(*) FROM t4 WHERE t4 MATCH '"a b c" OR "c a b"'
          158  +} {8000}
          159  +do_execsql_test 4.2 {
          160  +  SELECT quote(value) from t4_stat where id=0
          161  +} {X'C03EC0B204C0A608'}
          162  +do_execsql_test 4.3 {
          163  +  UPDATE t4_stat SET value = X'C03EC0B204C0A60800' WHERE id=0;
          164  +}
          165  +do_catchsql_test 4.4 {
          166  +  SELECT count(*) FROM t4 WHERE t4 MATCH '"a b c" OR "c a b"'
          167  +} {1 {database disk image is malformed}}
          168  +do_execsql_test 4.5 {
          169  +  UPDATE t4_stat SET value = X'00C03EC0B204C0A608' WHERE id=0;
          170  +}
          171  +do_catchsql_test 4.6 {
          172  +  SELECT count(*) FROM t4 WHERE t4 MATCH '"a b c" OR "c a b"'
          173  +} {1 {database disk image is malformed}}
          174  +
          175  +#-------------------------------------------------------------------------
          176  +#
          177  +reset_db
          178  +do_execsql_test 5.0 {
          179  +  CREATE VIRTUAL TABLE t5 USING fts4;
          180  +  INSERT INTO t5 VALUES('a x x x x b x x x x c');
          181  +  INSERT INTO t5 VALUES('a x x x x b x x x x c');
          182  +  INSERT INTO t5 VALUES('a x x x x b x x x x c');
          183  +}
          184  +do_execsql_test 5.1 {
          185  +  SELECT rowid FROM t5 WHERE t5 MATCH 'a NEAR/4 b NEAR/4 c'
          186  +} {1 2 3}
          187  +do_execsql_test 5.2 {
          188  +  SELECT rowid FROM t5 WHERE t5 MATCH 'a NEAR/3 b NEAR/4 c'
          189  +} {}
          190  +do_execsql_test 5.3 {
          191  +  SELECT rowid FROM t5 WHERE t5 MATCH 'a NEAR/4 b NEAR/3 c'
          192  +} {}
          193  +do_execsql_test 5.4 {
          194  +  SELECT rowid FROM t5 WHERE t5 MATCH 'y NEAR/4 b NEAR/4 c'
          195  +} {}
          196  +do_execsql_test 5.5 {
          197  +  SELECT rowid FROM t5 WHERE t5 MATCH 'x OR a NEAR/3 b NEAR/3 c'
          198  +} {1 2 3}
          199  +do_execsql_test 5.5 {
          200  +  SELECT rowid FROM t5 WHERE t5 MATCH 'x OR y NEAR/3 b NEAR/3 c'
          201  +} {1 2 3}
          202  +
          203  +#-------------------------------------------------------------------------
          204  +#
          205  +reset_db
          206  +do_execsql_test 6.0 {
          207  +  CREATE VIRTUAL TABLE t6 USING fts4;
          208  +
          209  +  BEGIN;
          210  +  WITH s(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<50000)
          211  +    INSERT INTO t6 SELECT 'x x x x x x x x x x x' FROM s;
          212  +
          213  +  INSERT INTO t6 VALUES('x x x x x x x x x x x A');
          214  +  INSERT INTO t6 VALUES('x x x x x x x x x x x B');
          215  +  INSERT INTO t6 VALUES('x x x x x x x x x x x A');
          216  +  INSERT INTO t6 VALUES('x x x x x x x x x x x B');
          217  +
          218  +  WITH s(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<50000)
          219  +    INSERT INTO t6 SELECT 'x x x x x x x x x x x' FROM s;
          220  +  COMMIT;
          221  +}
          222  +breakpoint
          223  +do_execsql_test 6.1 {
          224  +  SELECT rowid FROM t6 WHERE t6 MATCH 'b OR "x a"'
          225  +} {50001 50002 50003 50004}
          226  +
          227  +
          228  +finish_test

Changes to test/fts4langid.test.

    10     10   #*************************************************************************
    11     11   # This file implements regression tests for SQLite library.  The
    12     12   # focus of this script is testing the languageid=xxx FTS4 option.
    13     13   #
    14     14   
    15     15   set testdir [file dirname $argv0]
    16     16   source $testdir/tester.tcl
    17         -set ::testprefix fts4content
    18     17   
    19     18   # If SQLITE_ENABLE_FTS3 is defined, omit this file.
    20     19   ifcapable !fts3 {
    21     20     finish_test
    22     21     return
    23     22   }
    24     23   
................................................................................
   336    335   do_test_query1 3.3.2 {"zero one two"} { rowid_list "zero one two" }
   337    336   do_test_query1 3.3.3 {zero one two} {
   338    337     and_merge_lists [rowid_list zero] [rowid_list one] [rowid_list two]
   339    338   }
   340    339   do_test_query1 3.3.4 {"zero one" OR "one two"} {
   341    340     or_merge_lists [rowid_list "zero one"] [rowid_list "one two"]
   342    341   }
          342  +
          343  +do_execsql_test 3.4 {
          344  +  CREATE TABLE t8c(a, b);
          345  +  CREATE VIRTUAL TABLE t8 USING fts4(content=t8c, languageid=langid);
          346  +  INSERT INTO t8(docid, a, b) VALUES(-1, 'one two three', 'x y z');
          347  +  SELECT docid FROM t8 WHERE t8 MATCH 'one x' AND langid=0
          348  +} {-1}
   343    349   
   344    350   #-------------------------------------------------------------------------
   345    351   # Test cases 4.*
   346    352   #
   347    353   proc build_multilingual_db_2 {db} {
   348    354     $db eval {
   349    355       CREATE VIRTUAL TABLE t4 USING fts4(

Changes to test/in5.test.

   232    232   do_execsql_test 8.3 {
   233    233     INSERT INTO n1 VALUES(1, NULL), (2, NULL), (3, NULL);
   234    234     SELECT count(*) FROM n1 WHERE a IN (1, 2, 3)
   235    235   } 3
   236    236   do_execsql_test 8.4 {
   237    237     SELECT count(*) FROM n1 WHERE a IN (SELECT +a FROM n1)
   238    238   } 3
          239  +
          240  +#-------------------------------------------------------------------------
          241  +# Test that ticket 61fe97454c is fixed.
          242  +#
          243  +do_execsql_test 9.0 {
          244  +  CREATE TABLE t9(a INTEGER PRIMARY KEY);
          245  +  INSERT INTO t9 VALUES (44), (45);
          246  +}
          247  +do_execsql_test 9.1 {
          248  +  SELECT * FROM t9 WHERE a IN (44, 45, 44, 45)
          249  +} {44 45}
          250  +
   239    251   
   240    252   finish_test

Added test/indexexpr2.test.

            1  +# 2017 April 11
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +# This file implements regression tests for SQLite library.
           12  +#
           13  +
           14  +set testdir [file dirname $argv0]
           15  +source $testdir/tester.tcl
           16  +set testprefix indexexpr2
           17  +
           18  +do_execsql_test 1 {
           19  +  CREATE TABLE t1(a, b);
           20  +  INSERT INTO t1 VALUES(1, 'one');
           21  +  INSERT INTO t1 VALUES(2, 'two');
           22  +  INSERT INTO t1 VALUES(3, 'three');
           23  +
           24  +  CREATE INDEX i1 ON t1(b || 'x');
           25  +}
           26  +
           27  +do_execsql_test 1.1 {
           28  +  SELECT 'TWOX' == (b || 'x') FROM t1 WHERE (b || 'x')>'onex'
           29  +} {0 0}
           30  +
           31  +do_execsql_test 1.2 {
           32  +  SELECT 'TWOX' == (b || 'x') COLLATE nocase  FROM t1 WHERE (b || 'x')>'onex'
           33  +} {0 1}
           34  +
           35  +do_execsql_test 2.0 {
           36  +  CREATE INDEX i2 ON t1(a+1);
           37  +}
           38  +
           39  +do_execsql_test 2.1 {
           40  +  SELECT a+1, quote(a+1) FROM t1 ORDER BY 1;
           41  +} {2 2 3 3 4 4}
           42  +
           43  +finish_test
           44  +

Changes to test/json101.test.

   351    351     INSERT INTO t8(a) VALUES('abc' || char(1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35) || 'xyz');
   352    352     UPDATE t8 SET b=json_array(a);
   353    353     SELECT b FROM t8;
   354    354   } {{["abc\u0001\u0002\u0003\u0004\u0005\u0006\u0007\b\t\n\u000b\f\r\u000e\u000f\u0010\u0011\u0012\u0013\u0014\u0015\u0016\u0017\u0018\u0019\u001a\u001b\u001c\u001d\u001e\u001f !\"#xyz"]}}
   355    355   do_execsql_test json-8.2 {
   356    356     SELECT a=json_extract(b,'$[0]') FROM t8;
   357    357   } {1}
          358  +
          359  +# 2017-04-12.  Regression reported on the mailing list by Rolf Ade
          360  +#
          361  +do_execsql_test json-8.3 {
          362  +  SELECT json_valid(char(0x22,0xe4,0x22));
          363  +} {1}
          364  +do_execsql_test json-8.4 {
          365  +  SELECT unicode(json_extract(char(0x22,228,0x22),'$'));
          366  +} {228}
   358    367   
   359    368   # The json_quote() function transforms an SQL value into a JSON value.
   360    369   # String values are quoted and interior quotes are escaped.  NULL values
   361    370   # are rendered as the unquoted string "null".
   362    371   #
   363    372   do_execsql_test json-9.1 {
   364    373     SELECT json_quote('abc"xyz');
................................................................................
   684    693   do_execsql_test json-10.94 {
   685    694     SELECT json_valid('" \} "');
   686    695   } {0}
   687    696   do_execsql_test json-10.95 {
   688    697     SELECT json_valid('" \~ "');
   689    698   } {0}
   690    699   
          700  +#--------------------------------------------------------------------------
          701  +# 2017-04-11.  https://www.sqlite.org/src/info/981329adeef51011
          702  +# Stack overflow on deeply nested JSON.
          703  +#
          704  +# The following tests confirm that deeply nested JSON is considered invalid.
          705  +#
          706  +do_execsql_test json-11.0 {
          707  +  /* Shallow enough to be parsed */
          708  +  SELECT json_valid(printf('%.2000c0%.2000c','[',']'));
          709  +} {1}
          710  +do_execsql_test json-11.1 {
          711  +  /* Too deep by one */
          712  +  SELECT json_valid(printf('%.2001c0%.2001c','[',']'));
          713  +} {0}
          714  +do_execsql_test json-11.2 {
          715  +  /* Shallow enough to be parsed { */
          716  +  SELECT json_valid(replace(printf('%.2000c0%.2000c','[','}'),'[','{"a":'));
          717  +  /* } */
          718  +} {1}
          719  +do_execsql_test json-11.3 {
          720  +  /* Too deep by one { */
          721  +  SELECT json_valid(replace(printf('%.2001c0%.2001c','[','}'),'[','{"a":'));
          722  +  /* } */
          723  +} {0}
   691    724   
   692    725   finish_test

Changes to test/json102.test.

   314    314   do_execsql_test json102-1406 { SELECT json_valid('{"x":-0.1}') } 1
   315    315   do_execsql_test json102-1407 { SELECT json_valid('{"x":0.0000}') } 1
   316    316   do_execsql_test json102-1408 { SELECT json_valid('{"x":-0.0000}') } 1
   317    317   do_execsql_test json102-1409 { SELECT json_valid('{"x":01.5}') } 0
   318    318   do_execsql_test json102-1410 { SELECT json_valid('{"x":-01.5}') } 0
   319    319   do_execsql_test json102-1411 { SELECT json_valid('{"x":00}') } 0
   320    320   do_execsql_test json102-1412 { SELECT json_valid('{"x":-00}') } 0
          321  +
          322  +#------------------------------------------------------------------------
          323  +# 2017-04-10 ticket 6c9b5514077fed34551f98e64c09a10dc2fc8e16
          324  +# JSON extension accepts strings containing control characters.
          325  +#
          326  +# The JSON spec requires that all control characters be escaped.
          327  +#
          328  +do_execsql_test json102-1500 {
          329  +  WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<0x20)
          330  +  SELECT x FROM c WHERE json_valid(printf('{"a":"x%sz"}', char(x))) ORDER BY x;
          331  +} {32}
          332  +
          333  +# All control characters are escaped
          334  +#
          335  +do_execsql_test json102-1501 {
          336  +  WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<0x1f)
          337  +  SELECT sum(json_valid(json_quote('a'||char(x)||'z'))) FROM c ORDER BY x;
          338  +} {31}
   321    339   
   322    340   finish_test

Changes to test/permutations.test.

   261    261     fts3offsets.test fts3prefix2.test fts3prefix.test fts3query.test
   262    262     fts3shared.test fts3snippet.test fts3sort.test fts3tok1.test
   263    263     fts3tok_err.test fts3varint.test fts4aa.test fts4check.test
   264    264     fts4content.test fts4docid.test fts4growth2.test fts4growth.test
   265    265     fts4incr.test fts4langid.test fts4lastrowid.test fts4merge2.test
   266    266     fts4merge4.test fts4merge.test fts4noti.test fts4onepass.test 
   267    267     fts4opt.test fts4unicode.test
          268  +  fts3corrupt3.test
          269  +  fts3misc.test
   268    270   }
   269    271   
   270    272   test_suite "fts5" -prefix "" -description {
   271    273     All FTS5 tests.
   272    274   } -files [glob -nocomplain $::testdir/../ext/fts5/test/*.test]
   273    275   
   274    276   test_suite "fts5-light" -prefix "" -description {

Changes to tool/addopcodes.tcl.

    35     35     AGG_FUNCTION
    36     36     AGG_COLUMN
    37     37     UMINUS
    38     38     UPLUS
    39     39     REGISTER
    40     40     VECTOR
    41     41     SELECT_COLUMN
           42  +  IF_NULL_ROW
    42     43     ASTERISK
    43     44     SPAN
    44     45     SPACE
    45     46     ILLEGAL
    46     47   }
    47     48   if {[lrange $extras end-1 end]!="SPACE ILLEGAL"} {
    48     49     error "SPACE and ILLEGAL must be the last two token codes and they\

Changes to tool/fuzzershell.c.

   246    246   static void StrAppend(Str *p, const char *z){
   247    247     sqlite3_uint64 n = strlen(z);
   248    248     if( p->n + n >= p->nAlloc ){
   249    249       char *zNew;
   250    250       sqlite3_uint64 nNew;
   251    251       if( p->oomErr ) return;
   252    252       nNew = p->nAlloc*2 + 100 + n;
   253         -    zNew = sqlite3_realloc(p->z, nNew);
          253  +    zNew = sqlite3_realloc(p->z, (int)nNew);
   254    254       if( zNew==0 ){
   255    255         sqlite3_free(p->z);
   256    256         memset(p, 0, sizeof(*p));
   257    257         p->oomErr = 1;
   258    258         return;
   259    259       }
   260    260       p->z = zNew;
   261    261       p->nAlloc = nNew;
   262    262     }
   263         -  memcpy(p->z + p->n, z, n);
          263  +  memcpy(p->z + p->n, z, (size_t)n);
   264    264     p->n += n;
   265    265     p->z[p->n] = 0;
   266    266   }
   267    267   
   268    268   /* Return the current string content */
   269    269   static char *StrStr(Str *p){
   270    270    return p->z;
................................................................................
  1112   1112   
  1113   1113           /* If using an input database file and that database contains a table
  1114   1114           ** named "autoexec" with a column "sql", then replace the input SQL
  1115   1115           ** with the concatenated text of the autoexec table.  In this way,
  1116   1116           ** if the database file is the input being fuzzed, the SQL text is
  1117   1117           ** fuzzed at the same time. */
  1118   1118           if( sqlite3_table_column_metadata(db,0,"autoexec","sql",0,0,0,0,0)==0 ){
  1119         -          sqlite3_stmt *pStmt;
  1120         -          rc = sqlite3_prepare_v2(db, "SELECT sql FROM autoexec", -1, &pStmt, 0);
         1119  +          sqlite3_stmt *pStmt2;
         1120  +          rc = sqlite3_prepare_v2(db,"SELECT sql FROM autoexec",-1,&pStmt2,0);
  1121   1121             if( rc==SQLITE_OK ){
  1122         -            while( sqlite3_step(pStmt)==SQLITE_ROW ){
  1123         -              StrAppend(&sql, (const char*)sqlite3_column_text(pStmt, 0));
         1122  +            while( sqlite3_step(pStmt2)==SQLITE_ROW ){
         1123  +              StrAppend(&sql, (const char*)sqlite3_column_text(pStmt2, 0));
  1124   1124                 StrAppend(&sql, "\n");
  1125   1125               }
  1126   1126             }
  1127         -          sqlite3_finalize(pStmt);
         1127  +          sqlite3_finalize(pStmt2);
  1128   1128             zSql = StrStr(&sql);
  1129   1129           }
  1130   1130   
  1131   1131           g.bOomEnable = 1;
  1132   1132           if( verboseFlag ){
  1133   1133             zErrMsg = 0;
  1134   1134             rc = sqlite3_exec(db, zSql, execCallback, 0, &zErrMsg);

Changes to tool/lemon.c.

   164    164   struct lemon;
   165    165   struct action;
   166    166   
   167    167   static struct action *Action_new(void);
   168    168   static struct action *Action_sort(struct action *);
   169    169   
   170    170   /********** From the file "build.h" ************************************/
   171         -void FindRulePrecedences();
   172         -void FindFirstSets();
   173         -void FindStates();
   174         -void FindLinks();
   175         -void FindFollowSets();
   176         -void FindActions();
          171  +void FindRulePrecedences(struct lemon*);
          172  +void FindFirstSets(struct lemon*);
          173  +void FindStates(struct lemon*);
          174  +void FindLinks(struct lemon*);
          175  +void FindFollowSets(struct lemon*);
          176  +void FindActions(struct lemon*);
   177    177   
   178    178   /********* From the file "configlist.h" *********************************/
   179    179   void Configlist_init(void);
   180    180   struct config *Configlist_add(struct rule *, int);
   181    181   struct config *Configlist_addbasis(struct rule *, int);
   182    182   void Configlist_closure(struct lemon *);
   183    183   void Configlist_sort(void);
................................................................................
   453    453   /* Routines to manage the state table */
   454    454   
   455    455   int Configcmp(const char *, const char *);
   456    456   struct state *State_new(void);
   457    457   void State_init(void);
   458    458   int State_insert(struct state *, struct config *);
   459    459   struct state *State_find(struct config *);
   460         -struct state **State_arrayof(/*  */);
          460  +struct state **State_arrayof(void);
   461    461   
   462    462   /* Routines used for efficiency in Configlist_add */
   463    463   
   464    464   void Configtable_init(void);
   465    465   int Configtable_insert(struct config *);
   466    466   struct config *Configtable_find(struct config *);
   467    467   void Configtable_clear(int(*)(struct config *));
................................................................................
   557    557   ** value is then added to this initial offset to get an index X into the
   558    558   ** yy_action array. If the aAction[X].lookahead equals the value of the
   559    559   ** of the lookahead input, then the value of the action_number output is
   560    560   ** aAction[X].action.  If the lookaheads do not match then the
   561    561   ** default action for the state_number is returned.
   562    562   **
   563    563   ** All actions associated with a single state_number are first entered
   564         -** into aLookahead[] using multiple calls to acttab_action().  Then the 
   565         -** actions for that single state_number are placed into the aAction[] 
          564  +** into aLookahead[] using multiple calls to acttab_action().  Then the
          565  +** actions for that single state_number are placed into the aAction[]
   566    566   ** array with a single call to acttab_insert().  The acttab_insert() call
   567    567   ** also resets the aLookahead[] array in preparation for the next
   568    568   ** state number.
   569    569   */
   570    570   struct lookahead_action {
   571    571     int lookahead;             /* Value of the lookahead token */
   572    572     int action;                /* Action to take on the given lookahead */
................................................................................
   608    608       fprintf(stderr,"Unable to allocate memory for a new acttab.");
   609    609       exit(1);
   610    610     }
   611    611     memset(p, 0, sizeof(*p));
   612    612     return p;
   613    613   }
   614    614   
   615         -/* Add a new action to the current transaction set.  
          615  +/* Add a new action to the current transaction set.
   616    616   **
   617    617   ** This routine is called once for each lookahead for a particular
   618    618   ** state.
   619    619   */
   620    620   void acttab_action(acttab *p, int lookahead, int action){
   621    621     if( p->nLookahead>=p->nLookaheadAlloc ){
   622    622       p->nLookaheadAlloc += 25;
................................................................................
   670    670       }
   671    671       for(i=oldAlloc; i<p->nActionAlloc; i++){
   672    672         p->aAction[i].lookahead = -1;
   673    673         p->aAction[i].action = -1;
   674    674       }
   675    675     }
   676    676   
   677         -  /* Scan the existing action table looking for an offset that is a 
          677  +  /* Scan the existing action table looking for an offset that is a
   678    678     ** duplicate of the current transaction set.  Fall out of the loop
   679    679     ** if and when the duplicate is found.
   680    680     **
   681    681     ** i is the index in p->aAction[] where p->mnLookahead is inserted.
   682    682     */
   683    683     for(i=p->nAction-1; i>=0; i--){
   684    684       if( p->aAction[i].lookahead==p->mnLookahead ){
................................................................................
   748    748   /********************** From the file "build.c" *****************************/
   749    749   /*
   750    750   ** Routines to construction the finite state machine for the LEMON
   751    751   ** parser generator.
   752    752   */
   753    753   
   754    754   /* Find a precedence symbol of every rule in the grammar.
   755         -** 
          755  +**
   756    756   ** Those rules which have a precedence symbol coded in the input
   757    757   ** grammar using the "[symbol]" construct will already have the
   758    758   ** rp->precsym field filled.  Other rules take as their precedence
   759    759   ** symbol the first RHS symbol with a defined precedence.  If there
   760    760   ** are not RHS symbols with a defined precedence, the precedence
   761    761   ** symbol field is left blank.
   762    762   */
................................................................................
  1068   1068     int change;
  1069   1069   
  1070   1070     for(i=0; i<lemp->nstate; i++){
  1071   1071       for(cfp=lemp->sorted[i]->cfp; cfp; cfp=cfp->next){
  1072   1072         cfp->status = INCOMPLETE;
  1073   1073       }
  1074   1074     }
  1075         -  
         1075  +
  1076   1076     do{
  1077   1077       progress = 0;
  1078   1078       for(i=0; i<lemp->nstate; i++){
  1079   1079         for(cfp=lemp->sorted[i]->cfp; cfp; cfp=cfp->next){
  1080   1080           if( cfp->status==COMPLETE ) continue;
  1081   1081           for(plp=cfp->fplp; plp; plp=plp->next){
  1082   1082             change = SetUnion(plp->cfp->fws,cfp->fws);
................................................................................
  1099   1099   {
  1100   1100     int i,j;
  1101   1101     struct config *cfp;
  1102   1102     struct state *stp;
  1103   1103     struct symbol *sp;
  1104   1104     struct rule *rp;
  1105   1105   
  1106         -  /* Add all of the reduce actions 
         1106  +  /* Add all of the reduce actions
  1107   1107     ** A reduce action is added for each element of the followset of
  1108   1108     ** a configuration which has its dot at the extreme right.
  1109   1109     */
  1110   1110     for(i=0; i<lemp->nstate; i++){   /* Loop over all states */
  1111   1111       stp = lemp->sorted[i];
  1112   1112       for(cfp=stp->cfp; cfp; cfp=cfp->next){  /* Loop over all configurations */
  1113   1113         if( cfp->rp->nrhs==cfp->dot ){        /* Is dot at extreme right? */
................................................................................
  1216   1216         errcnt++;
  1217   1217       }else if( spx->prec>spy->prec ){
  1218   1218         apy->type = RD_RESOLVED;
  1219   1219       }else if( spx->prec<spy->prec ){
  1220   1220         apx->type = RD_RESOLVED;
  1221   1221       }
  1222   1222     }else{
  1223         -    assert( 
         1223  +    assert(
  1224   1224         apx->type==SH_RESOLVED ||
  1225   1225         apx->type==RD_RESOLVED ||
  1226   1226         apx->type==SSCONFLICT ||
  1227   1227         apx->type==SRCONFLICT ||
  1228   1228         apx->type==RRCONFLICT ||
  1229   1229         apy->type==SH_RESOLVED ||
  1230   1230         apy->type==RD_RESOLVED ||
................................................................................
  1247   1247   static struct config *freelist = 0;      /* List of free configurations */
  1248   1248   static struct config *current = 0;       /* Top of list of configurations */
  1249   1249   static struct config **currentend = 0;   /* Last on list of configs */
  1250   1250   static struct config *basis = 0;         /* Top of list of basis configs */
  1251   1251   static struct config **basisend = 0;     /* End of list of basis configs */
  1252   1252   
  1253   1253   /* Return a pointer to a new configuration */
  1254         -PRIVATE struct config *newconfig(){
         1254  +PRIVATE struct config *newconfig(void){
  1255   1255     struct config *newcfg;
  1256   1256     if( freelist==0 ){
  1257   1257       int i;
  1258   1258       int amt = 3;
  1259   1259       freelist = (struct config *)calloc( amt, sizeof(struct config) );
  1260   1260       if( freelist==0 ){
  1261   1261         fprintf(stderr,"Unable to allocate memory for a new configuration.");
................................................................................
  1273   1273   PRIVATE void deleteconfig(struct config *old)
  1274   1274   {
  1275   1275     old->next = freelist;
  1276   1276     freelist = old;
  1277   1277   }
  1278   1278   
  1279   1279   /* Initialized the configuration list builder */
  1280         -void Configlist_init(){
         1280  +void Configlist_init(void){
  1281   1281     current = 0;
  1282   1282     currentend = &current;
  1283   1283     basis = 0;
  1284   1284     basisend = &basis;
  1285   1285     Configtable_init();
  1286   1286     return;
  1287   1287   }
  1288   1288   
  1289   1289   /* Initialized the configuration list builder */
  1290         -void Configlist_reset(){
         1290  +void Configlist_reset(void){
  1291   1291     current = 0;
  1292   1292     currentend = &current;
  1293   1293     basis = 0;
  1294   1294     basisend = &basis;
  1295   1295     Configtable_clear(0);
  1296   1296     return;
  1297   1297   }
................................................................................
  1393   1393         }
  1394   1394       }
  1395   1395     }
  1396   1396     return;
  1397   1397   }
  1398   1398   
  1399   1399   /* Sort the configuration list */
  1400         -void Configlist_sort(){
         1400  +void Configlist_sort(void){
  1401   1401     current = (struct config*)msort((char*)current,(char**)&(current->next),
  1402   1402                                     Configcmp);
  1403   1403     currentend = 0;
  1404   1404     return;
  1405   1405   }
  1406   1406   
  1407   1407   /* Sort the basis configuration list */
  1408         -void Configlist_sortbasis(){
         1408  +void Configlist_sortbasis(void){
  1409   1409     basis = (struct config*)msort((char*)current,(char**)&(current->bp),
  1410   1410                                   Configcmp);
  1411   1411     basisend = 0;
  1412   1412     return;
  1413   1413   }
  1414   1414   
  1415   1415   /* Return a pointer to the head of the configuration list and
  1416   1416   ** reset the list */
  1417         -struct config *Configlist_return(){
         1417  +struct config *Configlist_return(void){
  1418   1418     struct config *old;
  1419   1419     old = current;
  1420   1420     current = 0;
  1421   1421     currentend = 0;
  1422   1422     return old;
  1423   1423   }
  1424   1424   
  1425   1425   /* Return a pointer to the head of the configuration list and
  1426   1426   ** reset the list */
  1427         -struct config *Configlist_basis(){
         1427  +struct config *Configlist_basis(void){
  1428   1428     struct config *old;
  1429   1429     old = basis;
  1430   1430     basis = 0;
  1431   1431     basisend = 0;
  1432   1432     return old;
  1433   1433   }
  1434   1434   
................................................................................
  1462   1462   /*
  1463   1463   ** Main program file for the LEMON parser generator.
  1464   1464   */
  1465   1465   
  1466   1466   /* Report an out-of-memory condition and abort.  This function
  1467   1467   ** is used mostly by the "MemoryCheck" macro in struct.h
  1468   1468   */
  1469         -void memory_error(){
         1469  +void memory_error(void){
  1470   1470     fprintf(stderr,"Out of memory.  Aborting...\n");
  1471   1471     exit(1);
  1472   1472   }
  1473   1473   
  1474   1474   static int nDefine = 0;      /* Number of -D options on the command line */
  1475   1475   static char **azDefine = 0;  /* Name of the -D macros */
  1476   1476   
................................................................................
  1602   1602     int exitcode;
  1603   1603     struct lemon lem;
  1604   1604     struct rule *rp;
  1605   1605   
  1606   1606     OptInit(argv,options,stderr);
  1607   1607     if( version ){
  1608   1608        printf("Lemon version 1.0\n");
  1609         -     exit(0); 
         1609  +     exit(0);
  1610   1610     }
  1611   1611     if( OptNArgs()!=1 ){
  1612   1612       fprintf(stderr,"Exactly one filename argument is required.\n");
  1613   1613       exit(1);
  1614   1614     }
  1615   1615     memset(&lem, 0, sizeof(lem));
  1616   1616     lem.errorcnt = 0;
................................................................................
  2047   2047       fprintf(err,"Valid command line options for \"%s\" are:\n",*a);
  2048   2048       OptPrint();
  2049   2049       exit(1);
  2050   2050     }
  2051   2051     return 0;
  2052   2052   }
  2053   2053   
  2054         -int OptNArgs(){
         2054  +int OptNArgs(void){
  2055   2055     int cnt = 0;
  2056   2056     int dashdash = 0;
  2057   2057     int i;
  2058   2058     if( argv!=0 && argv[0]!=0 ){
  2059   2059       for(i=1; argv[i]; i++){
  2060   2060         if( dashdash || !ISOPT(argv[i]) ) cnt++;
  2061   2061         if( strcmp(argv[i],"--")==0 ) dashdash = 1;
................................................................................
  2074   2074   void OptErr(int n)
  2075   2075   {
  2076   2076     int i;
  2077   2077     i = argindex(n);
  2078   2078     if( i>=0 ) errline(i,0,errstream);
  2079   2079   }
  2080   2080   
  2081         -void OptPrint(){
         2081  +void OptPrint(void){
  2082   2082     int i;
  2083   2083     int max, len;
  2084   2084     max = 0;
  2085   2085     for(i=0; op[i].label; i++){
  2086   2086       len = lemonStrlen(op[i].label) + 1;
  2087   2087       switch( op[i].type ){
  2088   2088         case OPT_FLAG:
................................................................................
  2303   2303           psp->errorcnt++;
  2304   2304           psp->state = RESYNC_AFTER_RULE_ERROR;
  2305   2305         }
  2306   2306         break;
  2307   2307       case IN_RHS:
  2308   2308         if( x[0]=='.' ){
  2309   2309           struct rule *rp;
  2310         -        rp = (struct rule *)calloc( sizeof(struct rule) + 
         2310  +        rp = (struct rule *)calloc( sizeof(struct rule) +
  2311   2311                sizeof(struct symbol*)*psp->nrhs + sizeof(char*)*psp->nrhs, 1);
  2312   2312           if( rp==0 ){
  2313   2313             ErrorMsg(psp->filename,psp->tokenlineno,
  2314   2314               "Can't allocate enough memory for this rule.");
  2315   2315             psp->errorcnt++;
  2316   2316             psp->prevrule = 0;
  2317   2317           }else{
................................................................................
  2892   2892   /*
  2893   2893   ** Routines processing configuration follow-set propagation links
  2894   2894   ** in the LEMON parser generator.
  2895   2895   */
  2896   2896   static struct plink *plink_freelist = 0;
  2897   2897   
  2898   2898   /* Allocate a new plink */
  2899         -struct plink *Plink_new(){
         2899  +struct plink *Plink_new(void){
  2900   2900     struct plink *newlink;
  2901   2901   
  2902   2902     if( plink_freelist==0 ){
  2903   2903       int i;
  2904   2904       int amt = 100;
  2905   2905       plink_freelist = (struct plink *)calloc( amt, sizeof(struct plink) );
  2906   2906       if( plink_freelist==0 ){
................................................................................
  2993   2993       fprintf(stderr,"Can't open file \"%s\".\n",lemp->outname);
  2994   2994       lemp->errorcnt++;
  2995   2995       return 0;
  2996   2996     }
  2997   2997     return fp;
  2998   2998   }
  2999   2999   
  3000         -/* Duplicate the input file without comments and without actions 
         3000  +/* Duplicate the input file without comments and without actions
  3001   3001   ** on rules */
  3002   3002   void Reprint(struct lemon *lemp)
  3003   3003   {
  3004   3004     struct rule *rp;
  3005   3005     struct symbol *sp;
  3006   3006     int i, j, maxlen, len, ncolumns, skip;
  3007   3007     printf("// Reprint of input file \"%s\".\n// Symbols:\n",lemp->filename);
................................................................................
  3144   3144         break;
  3145   3145       case SRCONFLICT:
  3146   3146       case RRCONFLICT:
  3147   3147         fprintf(fp,"%*s reduce       %-7d ** Parsing conflict **",
  3148   3148           indent,ap->sp->name,ap->x.rp->iRule);
  3149   3149         break;
  3150   3150       case SSCONFLICT:
  3151         -      fprintf(fp,"%*s shift        %-7d ** Parsing conflict **", 
         3151  +      fprintf(fp,"%*s shift        %-7d ** Parsing conflict **",
  3152   3152           indent,ap->sp->name,ap->x.stp->statenum);
  3153   3153         break;
  3154   3154       case SH_RESOLVED:
  3155   3155         if( showPrecedenceConflict ){
  3156   3156           fprintf(fp,"%*s shift        %-7d -- dropped by precedence",
  3157   3157                   indent,ap->sp->name,ap->x.stp->statenum);
  3158   3158         }else{
................................................................................
  3417   3417       str++;
  3418   3418     }
  3419   3419     if( str[-1]!='\n' ){
  3420   3420       putc('\n',out);
  3421   3421       (*lineno)++;
  3422   3422     }
  3423   3423     if (!lemp->nolinenosflag) {
  3424         -    (*lineno)++; tplt_linedir(out,*lineno,lemp->outname); 
         3424  +    (*lineno)++; tplt_linedir(out,*lineno,lemp->outname);
  3425   3425     }
  3426   3426     return;
  3427   3427   }
  3428   3428   
  3429   3429   /*
  3430   3430   ** The following routine emits code for the destructor for the
  3431   3431   ** symbol sp
................................................................................
  3462   3462        cp++;
  3463   3463        continue;
  3464   3464      }
  3465   3465      if( *cp=='\n' ) (*lineno)++;
  3466   3466      fputc(*cp,out);
  3467   3467    }
  3468   3468    fprintf(out,"\n"); (*lineno)++;
  3469         - if (!lemp->nolinenosflag) { 
  3470         -   (*lineno)++; tplt_linedir(out,*lineno,lemp->outname); 
         3469  + if (!lemp->nolinenosflag) {
         3470  +   (*lineno)++; tplt_linedir(out,*lineno,lemp->outname);
  3471   3471    }
  3472   3472    fprintf(out,"}\n"); (*lineno)++;
  3473   3473    return;
  3474   3474   }
  3475   3475   
  3476   3476   /*
  3477   3477   ** Return TRUE (non-zero) if the given symbol has a destructor.
................................................................................
  3586   3586         rp->codePrefix = Strsafe(append_str(0,0,0,0));
  3587   3587         rp->noCode = 0;
  3588   3588       }
  3589   3589     }else if( rp->lhsalias==0 ){
  3590   3590       /* There is no LHS value symbol. */
  3591   3591       lhsdirect = 1;
  3592   3592     }else if( strcmp(rp->lhsalias,rp->rhsalias[0])==0 ){
  3593         -    /* The LHS symbol and the left-most RHS symbol are the same, so 
         3593  +    /* The LHS symbol and the left-most RHS symbol are the same, so
  3594   3594       ** direct writing is allowed */
  3595   3595       lhsdirect = 1;
  3596   3596       lhsused = 1;
  3597   3597       used[0] = 1;
  3598   3598       if( rp->lhs->dtnum!=rp->rhs[0]->dtnum ){
  3599   3599         ErrorMsg(lemp->filename,rp->ruleline,
  3600   3600           "%s(%s) and %s(%s) share the same label but have "
  3601   3601           "different datatypes.",
  3602   3602           rp->lhs->name, rp->lhsalias, rp->rhs[0]->name, rp->rhsalias[0]);
  3603   3603         lemp->errorcnt++;
  3604         -    }    
         3604  +    }
  3605   3605     }else{
  3606   3606       lemon_sprintf(zOvwrt, "/*%s-overwrites-%s*/",
  3607   3607                     rp->lhsalias, rp->rhsalias[0]);
  3608   3608       zSkip = strstr(rp->code, zOvwrt);
  3609   3609       if( zSkip!=0 ){
  3610   3610         /* The code contains a special comment that indicates that it is safe
  3611   3611         ** for the LHS label to overwrite left-most RHS label. */
................................................................................
  3736   3736       rp->codeSuffix = Strsafe(cp);
  3737   3737       rp->noCode = 0;
  3738   3738     }
  3739   3739   
  3740   3740     return rc;
  3741   3741   }
  3742   3742   
  3743         -/* 
         3743  +/*
  3744   3744   ** Generate code which executes when the rule "rp" is reduced.  Write
  3745   3745   ** the code to "out".  Make sure lineno stays up-to-date.
  3746   3746   */
  3747   3747   PRIVATE void emit_code(
  3748   3748     FILE *out,
  3749   3749     struct rule *rp,
  3750   3750     struct lemon *lemp,
................................................................................
  4233   4233     /* Output the yy_shift_ofst[] table */
  4234   4234     n = lemp->nxstate;
  4235   4235     while( n>0 && lemp->sorted[n-1]->iTknOfst==NO_OFFSET ) n--;
  4236   4236     fprintf(out, "#define YY_SHIFT_USE_DFLT (%d)\n", lemp->nactiontab); lineno++;
  4237   4237     fprintf(out, "#define YY_SHIFT_COUNT    (%d)\n", n-1); lineno++;
  4238   4238     fprintf(out, "#define YY_SHIFT_MIN      (%d)\n", mnTknOfst); lineno++;
  4239   4239     fprintf(out, "#define YY_SHIFT_MAX      (%d)\n", mxTknOfst); lineno++;
  4240         -  fprintf(out, "static const %s yy_shift_ofst[] = {\n", 
         4240  +  fprintf(out, "static const %s yy_shift_ofst[] = {\n",
  4241   4241          minimum_size_type(mnTknOfst, lemp->nterminal+lemp->nactiontab, &sz));
  4242   4242          lineno++;
  4243   4243     lemp->tablesize += n*sz;
  4244   4244     for(i=j=0; i<n; i++){
  4245   4245       int ofst;
  4246   4246       stp = lemp->sorted[i];
  4247   4247       ofst = stp->iTknOfst;
................................................................................
  4260   4260     /* Output the yy_reduce_ofst[] table */
  4261   4261     fprintf(out, "#define YY_REDUCE_USE_DFLT (%d)\n", mnNtOfst-1); lineno++;
  4262   4262     n = lemp->nxstate;
  4263   4263     while( n>0 && lemp->sorted[n-1]->iNtOfst==NO_OFFSET ) n--;
  4264   4264     fprintf(out, "#define YY_REDUCE_COUNT (%d)\n", n-1); lineno++;
  4265   4265     fprintf(out, "#define YY_REDUCE_MIN   (%d)\n", mnNtOfst); lineno++;
  4266   4266     fprintf(out, "#define YY_REDUCE_MAX   (%d)\n", mxNtOfst); lineno++;
  4267         -  fprintf(out, "static const %s yy_reduce_ofst[] = {\n", 
         4267  +  fprintf(out, "static const %s yy_reduce_ofst[] = {\n",
  4268   4268             minimum_size_type(mnNtOfst-1, mxNtOfst, &sz)); lineno++;
  4269   4269     lemp->tablesize += n*sz;
  4270   4270     for(i=j=0; i<n; i++){
  4271   4271       int ofst;
  4272   4272       stp = lemp->sorted[i];
  4273   4273       ofst = stp->iNtOfst;
  4274   4274       if( ofst==NO_OFFSET ) ofst = mnNtOfst - 1;
................................................................................
  4339   4339       fprintf(out," /* %3d */ \"", i);
  4340   4340       writeRuleText(out, rp);
  4341   4341       fprintf(out,"\",\n"); lineno++;
  4342   4342     }
  4343   4343     tplt_xfer(lemp->name,in,out,&lineno);
  4344   4344   
  4345   4345     /* Generate code which executes every time a symbol is popped from
  4346         -  ** the stack while processing errors or while destroying the parser. 
         4346  +  ** the stack while processing errors or while destroying the parser.
  4347   4347     ** (In other words, generate the %destructor actions)
  4348   4348     */
  4349   4349     if( lemp->tokendest ){
  4350   4350       int once = 1;
  4351   4351       for(i=0; i<lemp->nsymbol; i++){
  4352   4352         struct symbol *sp = lemp->symbols[i];
  4353   4353         if( sp==0 || sp->type!=TERMINAL ) continue;
................................................................................
  4405   4405     }
  4406   4406     tplt_xfer(lemp->name,in,out,&lineno);
  4407   4407   
  4408   4408     /* Generate code which executes whenever the parser stack overflows */
  4409   4409     tplt_print(out,lemp,lemp->overflow,&lineno);
  4410   4410     tplt_xfer(lemp->name,in,out,&lineno);
  4411   4411   
  4412         -  /* Generate the table of rule information 
         4412  +  /* Generate the table of rule information
  4413   4413     **
  4414   4414     ** Note: This code depends on the fact that rules are number
  4415   4415     ** sequentually beginning with 0.
  4416   4416     */
  4417   4417     for(rp=lemp->rule; rp; rp=rp->next){
  4418   4418       fprintf(out,"  { %d, %d },\n",rp->lhs->index,rp->nrhs); lineno++;
  4419   4419     }
................................................................................
  4516   4516       }
  4517   4517     }
  4518   4518     out = file_open(lemp,".h","wb");
  4519   4519     if( out ){
  4520   4520       for(i=1; i<lemp->nterminal; i++){
  4521   4521         fprintf(out,"#define %s%-30s %3d\n",prefix,lemp->symbols[i]->name,i);
  4522   4522       }
  4523         -    fclose(out);  
         4523  +    fclose(out);
  4524   4524     }
  4525   4525     return;
  4526   4526   }
  4527   4527   
  4528   4528   /* Reduce the size of the action tables, if possible, by making use
  4529   4529   ** of defaults.
  4530   4530   **
................................................................................
  4563   4563           if( rp2==rp ) n++;
  4564   4564         }
  4565   4565         if( n>nbest ){
  4566   4566           nbest = n;
  4567   4567           rbest = rp;
  4568   4568         }
  4569   4569       }
  4570         - 
         4570  +
  4571   4571       /* Do not make a default if the number of rules to default
  4572   4572       ** is not at least 1 or if the wildcard token is a possible
  4573   4573       ** lookahead.
  4574   4574       */
  4575   4575       if( nbest<1 || usesWildcard ) continue;
  4576   4576   
  4577   4577   
................................................................................
  4720   4720   /* Set the set size */
  4721   4721   void SetSize(int n)
  4722   4722   {
  4723   4723     size = n+1;
  4724   4724   }
  4725   4725   
  4726   4726   /* Allocate a new set */
  4727         -char *SetNew(){
         4727  +char *SetNew(void){
  4728   4728     char *s;
  4729   4729     s = (char*)calloc( size, 1);
  4730   4730     if( s==0 ){
  4731   4731       extern void memory_error();
  4732   4732       memory_error();
  4733   4733     }
  4734   4734     return s;
................................................................................
  4826   4826     struct s_x1node **from;  /* Previous link */
  4827   4827   } x1node;
  4828   4828   
  4829   4829   /* There is only one instance of the array, which is the following */
  4830   4830   static struct s_x1 *x1a;
  4831   4831   
  4832   4832   /* Allocate a new associative array */
  4833         -void Strsafe_init(){
         4833  +void Strsafe_init(void){
  4834   4834     if( x1a ) return;
  4835   4835     x1a = (struct s_x1*)malloc( sizeof(struct s_x1) );
  4836   4836     if( x1a ){
  4837   4837       x1a->size = 1024;
  4838   4838       x1a->count = 0;
  4839   4839       x1a->tbl = (x1node*)calloc(1024, sizeof(x1node) + sizeof(x1node*));
  4840   4840       if( x1a->tbl==0 ){
................................................................................
  4993   4993     struct s_x2node **from;  /* Previous link */
  4994   4994   } x2node;
  4995   4995   
  4996   4996   /* There is only one instance of the array, which is the following */
  4997   4997   static struct s_x2 *x2a;
  4998   4998   
  4999   4999   /* Allocate a new associative array */
  5000         -void Symbol_init(){
         5000  +void Symbol_init(void){
  5001   5001     if( x2a ) return;
  5002   5002     x2a = (struct s_x2*)malloc( sizeof(struct s_x2) );
  5003   5003     if( x2a ){
  5004   5004       x2a->size = 128;
  5005   5005       x2a->count = 0;
  5006   5006       x2a->tbl = (x2node*)calloc(128, sizeof(x2node) + sizeof(x2node*));
  5007   5007       if( x2a->tbl==0 ){
................................................................................
  5190   5190     struct s_x3node **from;  /* Previous link */
  5191   5191   } x3node;
  5192   5192   
  5193   5193   /* There is only one instance of the array, which is the following */
  5194   5194   static struct s_x3 *x3a;
  5195   5195   
  5196   5196   /* Allocate a new associative array */
  5197         -void State_init(){
         5197  +void State_init(void){
  5198   5198     if( x3a ) return;
  5199   5199     x3a = (struct s_x3*)malloc( sizeof(struct s_x3) );
  5200   5200     if( x3a ){
  5201   5201       x3a->size = 128;
  5202   5202       x3a->count = 0;
  5203   5203       x3a->tbl = (x3node*)calloc(128, sizeof(x3node) + sizeof(x3node*));
  5204   5204       if( x3a->tbl==0 ){
................................................................................
  5284   5284     }
  5285   5285     return np ? np->data : 0;
  5286   5286   }
  5287   5287   
  5288   5288   /* Return an array of pointers to all data in the table.
  5289   5289   ** The array is obtained from malloc.  Return NULL if memory allocation
  5290   5290   ** problems, or if the array is empty. */
  5291         -struct state **State_arrayof()
         5291  +struct state **State_arrayof(void)
  5292   5292   {
  5293   5293     struct state **array;
  5294   5294     int i,arrSize;
  5295   5295     if( x3a==0 ) return 0;
  5296   5296     arrSize = x3a->count;
  5297   5297     array = (struct state **)calloc(arrSize, sizeof(struct state *));
  5298   5298     if( array ){
................................................................................
  5330   5330     struct s_x4node **from;  /* Previous link */
  5331   5331   } x4node;
  5332   5332   
  5333   5333   /* There is only one instance of the array, which is the following */
  5334   5334   static struct s_x4 *x4a;
  5335   5335   
  5336   5336   /* Allocate a new associative array */
  5337         -void Configtable_init(){
         5337  +void Configtable_init(void){
  5338   5338     if( x4a ) return;
  5339   5339     x4a = (struct s_x4*)malloc( sizeof(struct s_x4) );
  5340   5340     if( x4a ){
  5341   5341       x4a->size = 64;
  5342   5342       x4a->count = 0;
  5343   5343       x4a->tbl = (x4node*)calloc(64, sizeof(x4node) + sizeof(x4node*));
  5344   5344       if( x4a->tbl==0 ){

Changes to tool/showwal.c.

     8      8   #include <fcntl.h>
     9      9   
    10     10   #define ISDIGIT(X)  isdigit((unsigned char)(X))
    11     11   #define ISPRINT(X)  isprint((unsigned char)(X))
    12     12   
    13     13   #if !defined(_MSC_VER)
    14     14   #include <unistd.h>
           15  +#include <sys/types.h>
    15     16   #else
    16     17   #include <io.h>
    17     18   #endif
    18     19   
    19     20   #include <stdlib.h>
    20     21   #include <string.h>
    21     22   
................................................................................
   575    576             nByte = pagesize;
   576    577           }
   577    578           ofst = 32 + hdrSize + (iStart-1)*(pagesize+24) + 24;
   578    579           a = getContent(ofst, nByte);
   579    580           decode_btree_page(a, iStart, hdrSize, zLeft+1);
   580    581           free(a);
   581    582           continue;
          583  +#if !defined(_MSC_VER)
          584  +      }else if( zLeft && strcmp(zLeft,"truncate")==0 ){
          585  +        /* Frame number followed by "truncate" truncates the WAL file
          586  +        ** after that frame */
          587  +        off_t newSize = 32 + iStart*(pagesize+24);
          588  +        truncate(argv[1], newSize);
          589  +        continue;
          590  +#endif
   582    591         }else{
   583    592           iEnd = iStart;
   584    593         }
   585    594         if( iStart<1 || iEnd<iStart || iEnd>mxFrame ){
   586    595           fprintf(stderr,
   587    596             "Page argument should be LOWER?..UPPER?.  Range 1 to %d\n",
   588    597             mxFrame);