/ Check-in [a1a85b84]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Merge the latest trunk changes into the winrt branch.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | winrt
Files: files | file ages | folders
SHA1: a1a85b846aac07979e732a9f36d178bd567e103a
User & Date: drh 2012-03-30 14:01:40
Context
2012-04-17
21:00
When compiling for WinRT, always use the 'appcontainer' linker option. check-in: 300bcfe3 user: mistachkin tags: winrt
2012-03-30
14:01
Merge the latest trunk changes into the winrt branch. check-in: a1a85b84 user: drh tags: winrt
13:34
Fix an FTS4 test script problem for windows. check-in: 36aa6665 user: drh tags: trunk
12:27
Simplify the winRead and winWrite VFS functions to reduce the number of system calls. check-in: 10ce8467 user: mistachkin tags: winrt
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to VERSION.

     1         -3.7.11
            1  +3.7.12

Changes to ext/fts3/fts3.c.

    66     66   **
    67     67   ** FTS3 used to optionally store character offsets using a compile-time
    68     68   ** option.  But that functionality is no longer supported.
    69     69   **
    70     70   ** A doclist is stored like this:
    71     71   **
    72     72   ** array {
    73         -**   varint docid;
           73  +**   varint docid;          (delta from previous doclist)
    74     74   **   array {                (position list for column 0)
    75     75   **     varint position;     (2 more than the delta from previous position)
    76     76   **   }
    77     77   **   array {
    78     78   **     varint POS_COLUMN;   (marks start of position list for new column)
    79     79   **     varint column;       (index of new column)
    80     80   **     array {
................................................................................
    97     97   **   value:     123 5 9 1 1 14 35 0 234 72 0
    98     98   **
    99     99   ** The 123 value is the first docid.  For column zero in this document
   100    100   ** there are two matches at positions 3 and 10 (5-2 and 9-2+3).  The 1
   101    101   ** at D signals the start of a new column; the 1 at E indicates that the
   102    102   ** new column is column number 1.  There are two positions at 12 and 45
   103    103   ** (14-2 and 35-2+12).  The 0 at H indicate the end-of-document.  The
   104         -** 234 at I is the next docid.  It has one position 72 (72-2) and then
   105         -** terminates with the 0 at K.
          104  +** 234 at I is the delta to next docid (357).  It has one position 70
          105  +** (72-2) and then terminates with the 0 at K.
   106    106   **
   107    107   ** A "position-list" is the list of positions for multiple columns for
   108    108   ** a single docid.  A "column-list" is the set of positions for a single
   109    109   ** column.  Hence, a position-list consists of one or more column-lists,
   110    110   ** a document record consists of a docid followed by a position-list and
   111    111   ** a doclist consists of one or more document records.
   112    112   **
................................................................................
   565    565       }
   566    566   
   567    567       sqlite3_free(zSql);
   568    568       sqlite3_free(zCols);
   569    569       *pRc = rc;
   570    570     }
   571    571   }
          572  +
          573  +/*
          574  +** Create the %_stat table if it does not already exist.
          575  +*/
          576  +void sqlite3Fts3CreateStatTable(int *pRc, Fts3Table *p){
          577  +  fts3DbExec(pRc, p->db, 
          578  +      "CREATE TABLE IF NOT EXISTS %Q.'%q_stat'"
          579  +          "(id INTEGER PRIMARY KEY, value BLOB);",
          580  +      p->zDb, p->zName
          581  +  );
          582  +  if( (*pRc)==SQLITE_OK ) p->bHasStat = 1;
          583  +}
   572    584   
   573    585   /*
   574    586   ** Create the backing store tables (%_content, %_segments and %_segdir)
   575    587   ** required by the FTS3 table passed as the only argument. This is done
   576    588   ** as part of the vtab xCreate() method.
   577    589   **
   578    590   ** If the p->bHasDocsize boolean is true (indicating that this is an
................................................................................
   626    638     );
   627    639     if( p->bHasDocsize ){
   628    640       fts3DbExec(&rc, db, 
   629    641           "CREATE TABLE %Q.'%q_docsize'(docid INTEGER PRIMARY KEY, size BLOB);",
   630    642           p->zDb, p->zName
   631    643       );
   632    644     }
          645  +  assert( p->bHasStat==p->bFts4 );
   633    646     if( p->bHasStat ){
   634         -    fts3DbExec(&rc, db, 
   635         -        "CREATE TABLE %Q.'%q_stat'(id INTEGER PRIMARY KEY, value BLOB);",
   636         -        p->zDb, p->zName
   637         -    );
          647  +    sqlite3Fts3CreateStatTable(&rc, p);
   638    648     }
   639    649     return rc;
   640    650   }
   641    651   
   642    652   /*
   643    653   ** Store the current database page-size in bytes in p->nPgsz.
   644    654   **
................................................................................
  1271   1281     p->nColumn = nCol;
  1272   1282     p->nPendingData = 0;
  1273   1283     p->azColumn = (char **)&p[1];
  1274   1284     p->pTokenizer = pTokenizer;
  1275   1285     p->nMaxPendingData = FTS3_MAX_PENDING_DATA;
  1276   1286     p->bHasDocsize = (isFts4 && bNoDocsize==0);
  1277   1287     p->bHasStat = isFts4;
         1288  +  p->bFts4 = isFts4;
  1278   1289     p->bDescIdx = bDescIdx;
         1290  +  p->bAutoincrmerge = 0xff;   /* 0xff means setting unknown */
  1279   1291     p->zContentTbl = zContent;
  1280   1292     p->zLanguageid = zLanguageid;
  1281   1293     zContent = 0;
  1282   1294     zLanguageid = 0;
  1283   1295     TESTONLY( p->inTransaction = -1 );
  1284   1296     TESTONLY( p->mxSavepoint = -1 );
  1285   1297   
................................................................................
  1323   1335   
  1324   1336     /* If this is an xCreate call, create the underlying tables in the 
  1325   1337     ** database. TODO: For xConnect(), it could verify that said tables exist.
  1326   1338     */
  1327   1339     if( isCreate ){
  1328   1340       rc = fts3CreateTables(p);
  1329   1341     }
         1342  +
         1343  +  /* Check to see if a legacy fts3 table has been "upgraded" by the
         1344  +  ** addition of a %_stat table so that it can use incremental merge.
         1345  +  */
         1346  +  if( !isFts4 && !isCreate ){
         1347  +    int rc2 = SQLITE_OK;
         1348  +    fts3DbExec(&rc2, db, "SELECT 1 FROM %Q.'%q_stat' WHERE id=2",
         1349  +               p->zDb, p->zName);
         1350  +    if( rc2==SQLITE_OK ) p->bHasStat = 1;
         1351  +  }
  1330   1352   
  1331   1353     /* Figure out the page-size for the database. This is required in order to
  1332   1354     ** estimate the cost of loading large doclists from the database.  */
  1333   1355     fts3DatabasePageSize(&rc, p);
  1334   1356     p->nNodeSize = p->nPgsz-35;
  1335   1357   
  1336   1358     /* Declare the table schema to SQLite. */
................................................................................
  2667   2689   
  2668   2690   /*
  2669   2691   ** Set up a cursor object for iterating through a full-text index or a 
  2670   2692   ** single level therein.
  2671   2693   */
  2672   2694   int sqlite3Fts3SegReaderCursor(
  2673   2695     Fts3Table *p,                   /* FTS3 table handle */
  2674         -  int iLangid,
         2696  +  int iLangid,                    /* Language-id to search */
  2675   2697     int iIndex,                     /* Index to search (from 0 to p->nIndex-1) */
  2676   2698     int iLevel,                     /* Level of segments to scan */
  2677   2699     const char *zTerm,              /* Term to query for */
  2678   2700     int nTerm,                      /* Size of zTerm in bytes */
  2679   2701     int isPrefix,                   /* True for a prefix search */
  2680   2702     int isScan,                     /* True to scan from zTerm to EOF */
  2681   2703     Fts3MultiSegReader *pCsr       /* Cursor object to populate */
................................................................................
  2685   2707         ||  iLevel==FTS3_SEGCURSOR_PENDING 
  2686   2708         ||  iLevel>=0
  2687   2709     );
  2688   2710     assert( iLevel<FTS3_SEGDIR_MAXLEVEL );
  2689   2711     assert( FTS3_SEGCURSOR_ALL<0 && FTS3_SEGCURSOR_PENDING<0 );
  2690   2712     assert( isPrefix==0 || isScan==0 );
  2691   2713   
  2692         -  /* "isScan" is only set to true by the ft4aux module, an ordinary
  2693         -  ** full-text tables. */
  2694         -  assert( isScan==0 || p->aIndex==0 );
  2695         -
  2696   2714     memset(pCsr, 0, sizeof(Fts3MultiSegReader));
  2697         -
  2698   2715     return fts3SegReaderCursor(
  2699   2716         p, iLangid, iIndex, iLevel, zTerm, nTerm, isPrefix, isScan, pCsr
  2700   2717     );
  2701   2718   }
  2702   2719   
  2703   2720   /*
  2704   2721   ** In addition to its current configuration, have the Fts3MultiSegReader
................................................................................
  2955   2972         return SQLITE_NOMEM;
  2956   2973       }
  2957   2974   
  2958   2975       pCsr->iLangid = 0;
  2959   2976       if( nVal==2 ) pCsr->iLangid = sqlite3_value_int(apVal[1]);
  2960   2977   
  2961   2978       rc = sqlite3Fts3ExprParse(p->pTokenizer, pCsr->iLangid,
  2962         -        p->azColumn, p->bHasStat, p->nColumn, iCol, zQuery, -1, &pCsr->pExpr
         2979  +        p->azColumn, p->bFts4, p->nColumn, iCol, zQuery, -1, &pCsr->pExpr
  2963   2980       );
  2964   2981       if( rc!=SQLITE_OK ){
  2965   2982         if( rc==SQLITE_ERROR ){
  2966   2983           static const char *zErr = "malformed MATCH expression: [%s]";
  2967   2984           p->base.zErrMsg = sqlite3_mprintf(zErr, zQuery);
  2968   2985         }
  2969   2986         return rc;
................................................................................
  3098   3115   }
  3099   3116   
  3100   3117   /*
  3101   3118   ** Implementation of xSync() method. Flush the contents of the pending-terms
  3102   3119   ** hash-table to the database.
  3103   3120   */
  3104   3121   static int fts3SyncMethod(sqlite3_vtab *pVtab){
  3105         -  int rc = sqlite3Fts3PendingTermsFlush((Fts3Table *)pVtab);
  3106         -  sqlite3Fts3SegmentsClose((Fts3Table *)pVtab);
         3122  +
         3123  +  /* Following an incremental-merge operation, assuming that the input
         3124  +  ** segments are not completely consumed (the usual case), they are updated
         3125  +  ** in place to remove the entries that have already been merged. This
         3126  +  ** involves updating the leaf block that contains the smallest unmerged
         3127  +  ** entry and each block (if any) between the leaf and the root node. So
         3128  +  ** if the height of the input segment b-trees is N, and input segments
         3129  +  ** are merged eight at a time, updating the input segments at the end
         3130  +  ** of an incremental-merge requires writing (8*(1+N)) blocks. N is usually
         3131  +  ** small - often between 0 and 2. So the overhead of the incremental
         3132  +  ** merge is somewhere between 8 and 24 blocks. To avoid this overhead
         3133  +  ** dwarfing the actual productive work accomplished, the incremental merge
         3134  +  ** is only attempted if it will write at least 64 leaf blocks. Hence
         3135  +  ** nMinMerge.
         3136  +  **
         3137  +  ** Of course, updating the input segments also involves deleting a bunch
         3138  +  ** of blocks from the segments table. But this is not considered overhead
         3139  +  ** as it would also be required by a crisis-merge that used the same input 
         3140  +  ** segments.
         3141  +  */
         3142  +  const u32 nMinMerge = 64;       /* Minimum amount of incr-merge work to do */
         3143  +
         3144  +  Fts3Table *p = (Fts3Table*)pVtab;
         3145  +  int rc = sqlite3Fts3PendingTermsFlush(p);
         3146  +
         3147  +  if( rc==SQLITE_OK && p->bAutoincrmerge==1 && p->nLeafAdd>(nMinMerge/16) ){
         3148  +    int mxLevel = 0;              /* Maximum relative level value in db */
         3149  +    int A;                        /* Incr-merge parameter A */
         3150  +
         3151  +    rc = sqlite3Fts3MaxLevel(p, &mxLevel);
         3152  +    assert( rc==SQLITE_OK || mxLevel==0 );
         3153  +    A = p->nLeafAdd * mxLevel;
         3154  +    A += (A/2);
         3155  +    if( A>(int)nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, 8);
         3156  +  }
         3157  +  sqlite3Fts3SegmentsClose(p);
  3107   3158     return rc;
  3108   3159   }
  3109   3160   
  3110   3161   /*
  3111   3162   ** Implementation of xBegin() method. This is a no-op.
  3112   3163   */
  3113   3164   static int fts3BeginMethod(sqlite3_vtab *pVtab){
  3114         -  TESTONLY( Fts3Table *p = (Fts3Table*)pVtab );
         3165  +  Fts3Table *p = (Fts3Table*)pVtab;
  3115   3166     UNUSED_PARAMETER(pVtab);
  3116   3167     assert( p->pSegments==0 );
  3117   3168     assert( p->nPendingData==0 );
  3118   3169     assert( p->inTransaction!=1 );
  3119   3170     TESTONLY( p->inTransaction = 1 );
  3120   3171     TESTONLY( p->mxSavepoint = -1; );
         3172  +  p->nLeafAdd = 0;
  3121   3173     return SQLITE_OK;
  3122   3174   }
  3123   3175   
  3124   3176   /*
  3125   3177   ** Implementation of xCommit() method. This is a no-op. The contents of
  3126   3178   ** the pending-terms hash-table have already been flushed into the database
  3127   3179   ** by fts3SyncMethod().
................................................................................
  3408   3460   
  3409   3461   /*
  3410   3462   ** The xSavepoint() method.
  3411   3463   **
  3412   3464   ** Flush the contents of the pending-terms table to disk.
  3413   3465   */
  3414   3466   static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
         3467  +  int rc = SQLITE_OK;
  3415   3468     UNUSED_PARAMETER(iSavepoint);
  3416   3469     assert( ((Fts3Table *)pVtab)->inTransaction );
  3417   3470     assert( ((Fts3Table *)pVtab)->mxSavepoint < iSavepoint );
  3418   3471     TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint );
  3419         -  return fts3SyncMethod(pVtab);
         3472  +  if( ((Fts3Table *)pVtab)->bIgnoreSavepoint==0 ){
         3473  +    rc = fts3SyncMethod(pVtab);
         3474  +  }
         3475  +  return rc;
  3420   3476   }
  3421   3477   
  3422   3478   /*
  3423   3479   ** The xRelease() method.
  3424   3480   **
  3425   3481   ** This is a no-op.
  3426   3482   */
................................................................................
  4327   4383     int nToken = 0;
  4328   4384     int nOr = 0;
  4329   4385   
  4330   4386     /* Allocate a MultiSegReader for each token in the expression. */
  4331   4387     fts3EvalAllocateReaders(pCsr, pCsr->pExpr, &nToken, &nOr, &rc);
  4332   4388   
  4333   4389     /* Determine which, if any, tokens in the expression should be deferred. */
  4334         -  if( rc==SQLITE_OK && nToken>1 && pTab->bHasStat ){
         4390  +  if( rc==SQLITE_OK && nToken>1 && pTab->bFts4 ){
  4335   4391       Fts3TokenAndCost *aTC;
  4336   4392       Fts3Expr **apOr;
  4337   4393       aTC = (Fts3TokenAndCost *)sqlite3_malloc(
  4338   4394           sizeof(Fts3TokenAndCost) * nToken
  4339   4395         + sizeof(Fts3Expr *) * nOr * 2
  4340   4396       );
  4341   4397       apOr = (Fts3Expr **)&aTC[nToken];
................................................................................
  5142   5198       memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist));
  5143   5199       for(i=0; i<pPhrase->nToken; i++){
  5144   5200         fts3SegReaderCursorFree(pPhrase->aToken[i].pSegcsr);
  5145   5201         pPhrase->aToken[i].pSegcsr = 0;
  5146   5202       }
  5147   5203     }
  5148   5204   }
         5205  +
  5149   5206   
  5150   5207   /*
  5151   5208   ** Return SQLITE_CORRUPT_VTAB.
  5152   5209   */
  5153   5210   #ifdef SQLITE_DEBUG
  5154   5211   int sqlite3Fts3Corrupt(){
  5155   5212     return SQLITE_CORRUPT_VTAB;

Changes to ext/fts3/fts3Int.h.

    63     63   */
    64     64   #define SizeofArray(X) ((int)(sizeof(X)/sizeof(X[0])))
    65     65   
    66     66   
    67     67   #ifndef MIN
    68     68   # define MIN(x,y) ((x)<(y)?(x):(y))
    69     69   #endif
           70  +#ifndef MAX
           71  +# define MAX(x,y) ((x)>(y)?(x):(y))
           72  +#endif
    70     73   
    71     74   /*
    72     75   ** Maximum length of a varint encoded integer. The varint format is different
    73     76   ** from that used by SQLite, so the maximum length is 10, not 9.
    74     77   */
    75     78   #define FTS3_VARINT_MAX 10
    76     79   
................................................................................
   117    120   ** false.
   118    121   */
   119    122   #ifdef SQLITE_COVERAGE_TEST
   120    123   # define ALWAYS(x) (1)
   121    124   # define NEVER(X)  (0)
   122    125   #else
   123    126   # define ALWAYS(x) (x)
   124         -# define NEVER(X)  (x)
          127  +# define NEVER(x)  (x)
   125    128   #endif
   126    129   
   127    130   /*
   128    131   ** Internal types used by SQLite.
   129    132   */
   130    133   typedef unsigned char u8;         /* 1-byte (or larger) unsigned integer */
   131    134   typedef short int i16;            /* 2-byte (or larger) signed integer */
   132    135   typedef unsigned int u32;         /* 4-byte unsigned integer */
   133    136   typedef sqlite3_uint64 u64;       /* 8-byte unsigned integer */
          137  +typedef sqlite3_int64 i64;        /* 8-byte signed integer */
   134    138   
   135    139   /*
   136    140   ** Macro used to suppress compiler warnings for unused parameters.
   137    141   */
   138    142   #define UNUSED_PARAMETER(x) (void)(x)
   139    143   
   140    144   /*
................................................................................
   189    193     const char *zDb;                /* logical database name */
   190    194     const char *zName;              /* virtual table name */
   191    195     int nColumn;                    /* number of named columns in virtual table */
   192    196     char **azColumn;                /* column names.  malloced */
   193    197     sqlite3_tokenizer *pTokenizer;  /* tokenizer for inserts and queries */
   194    198     char *zContentTbl;              /* content=xxx option, or NULL */
   195    199     char *zLanguageid;              /* languageid=xxx option, or NULL */
          200  +  u8 bAutoincrmerge;              /* True if automerge=1 */
          201  +  u32 nLeafAdd;                   /* Number of leaf blocks added this trans */
   196    202   
   197    203     /* Precompiled statements used by the implementation. Each of these 
   198    204     ** statements is run and reset within a single virtual table API call. 
   199    205     */
   200         -  sqlite3_stmt *aStmt[28];
          206  +  sqlite3_stmt *aStmt[37];
   201    207   
   202    208     char *zReadExprlist;
   203    209     char *zWriteExprlist;
   204    210   
   205    211     int nNodeSize;                  /* Soft limit for node size */
          212  +  u8 bFts4;                       /* True for FTS4, false for FTS3 */
   206    213     u8 bHasStat;                    /* True if %_stat table exists */
   207    214     u8 bHasDocsize;                 /* True if %_docsize table exists */
   208    215     u8 bDescIdx;                    /* True if doclists are in reverse order */
          216  +  u8 bIgnoreSavepoint;            /* True to ignore xSavepoint invocations */
   209    217     int nPgsz;                      /* Page size for host database */
   210    218     char *zSegmentsTbl;             /* Name of %_segments table */
   211    219     sqlite3_blob *pSegments;        /* Blob handle open on %_segments table */
   212    220   
   213         -  /* TODO: Fix the first paragraph of this comment.
   214         -  **
          221  +  /* 
   215    222     ** The following array of hash tables is used to buffer pending index 
   216         -  ** updates during transactions. Variable nPendingData estimates the memory 
   217         -  ** size of the pending data, including hash table overhead, not including
   218         -  ** malloc overhead.  When nPendingData exceeds nMaxPendingData, the buffer 
   219         -  ** is flushed automatically. Variable iPrevDocid is the docid of the most 
   220         -  ** recently inserted record.
          223  +  ** updates during transactions. All pending updates buffered at any one
          224  +  ** time must share a common language-id (see the FTS4 langid= feature).
          225  +  ** The current language id is stored in variable iPrevLangid.
   221    226     **
   222    227     ** A single FTS4 table may have multiple full-text indexes. For each index
   223    228     ** there is an entry in the aIndex[] array. Index 0 is an index of all the
   224    229     ** terms that appear in the document set. Each subsequent index in aIndex[]
   225    230     ** is an index of prefixes of a specific length.
          231  +  **
          232  +  ** Variable nPendingData contains an estimate the memory consumed by the 
          233  +  ** pending data structures, including hash table overhead, but not including
          234  +  ** malloc overhead.  When nPendingData exceeds nMaxPendingData, all hash
          235  +  ** tables are flushed to disk. Variable iPrevDocid is the docid of the most 
          236  +  ** recently inserted record.
   226    237     */
   227    238     int nIndex;                     /* Size of aIndex[] */
   228    239     struct Fts3Index {
   229    240       int nPrefix;                  /* Prefix length (0 for main terms index) */
   230    241       Fts3Hash hPending;            /* Pending terms table for this index */
   231    242     } *aIndex;
   232    243     int nMaxPendingData;            /* Max pending data before flush to disk */
................................................................................
   417    428   int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **);
   418    429   
   419    430   void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *);
   420    431   int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int);
   421    432   int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *);
   422    433   void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *);
   423    434   void sqlite3Fts3SegmentsClose(Fts3Table *);
          435  +int sqlite3Fts3MaxLevel(Fts3Table *, int *);
   424    436   
   425    437   /* Special values interpreted by sqlite3SegReaderCursor() */
   426    438   #define FTS3_SEGCURSOR_PENDING        -1
   427    439   #define FTS3_SEGCURSOR_ALL            -2
   428    440   
   429    441   int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3MultiSegReader*, Fts3SegFilter*);
   430    442   int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3MultiSegReader *);
................................................................................
   468    480     /* Output values. Valid only after Fts3SegReaderStep() returns SQLITE_ROW. */
   469    481     char *zTerm;                    /* Pointer to term buffer */
   470    482     int nTerm;                      /* Size of zTerm in bytes */
   471    483     char *aDoclist;                 /* Pointer to doclist buffer */
   472    484     int nDoclist;                   /* Size of aDoclist[] in bytes */
   473    485   };
   474    486   
          487  +int sqlite3Fts3Incrmerge(Fts3Table*,int,int);
          488  +
   475    489   /* fts3.c */
   476    490   int sqlite3Fts3PutVarint(char *, sqlite3_int64);
   477    491   int sqlite3Fts3GetVarint(const char *, sqlite_int64 *);
   478    492   int sqlite3Fts3GetVarint32(const char *, int *);
   479    493   int sqlite3Fts3VarintLen(sqlite3_uint64);
   480    494   void sqlite3Fts3Dequote(char *);
   481    495   void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*);
   482    496   int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *);
   483    497   int sqlite3Fts3FirstFilter(sqlite3_int64, char *, int, char *);
          498  +void sqlite3Fts3CreateStatTable(int*, Fts3Table*);
   484    499   
   485    500   /* fts3_tokenizer.c */
   486    501   const char *sqlite3Fts3NextToken(const char *, int *);
   487    502   int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *);
   488    503   int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, const char *, 
   489    504       sqlite3_tokenizer **, char **
   490    505   );

Changes to ext/fts3/fts3_snippet.c.

   790    790   static int fts3MatchinfoCheck(
   791    791     Fts3Table *pTab, 
   792    792     char cArg,
   793    793     char **pzErr
   794    794   ){
   795    795     if( (cArg==FTS3_MATCHINFO_NPHRASE)
   796    796      || (cArg==FTS3_MATCHINFO_NCOL)
   797         -   || (cArg==FTS3_MATCHINFO_NDOC && pTab->bHasStat)
   798         -   || (cArg==FTS3_MATCHINFO_AVGLENGTH && pTab->bHasStat)
          797  +   || (cArg==FTS3_MATCHINFO_NDOC && pTab->bFts4)
          798  +   || (cArg==FTS3_MATCHINFO_AVGLENGTH && pTab->bFts4)
   799    799      || (cArg==FTS3_MATCHINFO_LENGTH && pTab->bHasDocsize)
   800    800      || (cArg==FTS3_MATCHINFO_LCS)
   801    801      || (cArg==FTS3_MATCHINFO_HITS)
   802    802     ){
   803    803       return SQLITE_OK;
   804    804     }
   805    805     *pzErr = sqlite3_mprintf("unrecognized matchinfo request: %c", cArg);

Changes to ext/fts3/fts3_write.c.

    20     20   #include "fts3Int.h"
    21     21   #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
    22     22   
    23     23   #include <string.h>
    24     24   #include <assert.h>
    25     25   #include <stdlib.h>
    26     26   
           27  +
           28  +#define FTS_MAX_APPENDABLE_HEIGHT 16
           29  +
    27     30   /*
    28     31   ** When full-text index nodes are loaded from disk, the buffer that they
    29     32   ** are loaded into has the following number of bytes of padding at the end 
    30     33   ** of it. i.e. if a full-text index node is 900 bytes in size, then a buffer
    31     34   ** of 920 bytes is allocated for it.
    32     35   **
    33     36   ** This means that if we have a pointer into a buffer containing node data,
................................................................................
    58     61   int test_fts3_node_chunk_threshold = (4*1024)*4;
    59     62   # define FTS3_NODE_CHUNKSIZE       test_fts3_node_chunksize
    60     63   # define FTS3_NODE_CHUNK_THRESHOLD test_fts3_node_chunk_threshold
    61     64   #else
    62     65   # define FTS3_NODE_CHUNKSIZE (4*1024) 
    63     66   # define FTS3_NODE_CHUNK_THRESHOLD (FTS3_NODE_CHUNKSIZE*4)
    64     67   #endif
           68  +
           69  +/*
           70  +** The two values that may be meaningfully bound to the :1 parameter in
           71  +** statements SQL_REPLACE_STAT and SQL_SELECT_STAT.
           72  +*/
           73  +#define FTS_STAT_DOCTOTAL      0
           74  +#define FTS_STAT_INCRMERGEHINT 1
           75  +#define FTS_STAT_AUTOINCRMERGE 2
           76  +
           77  +/*
           78  +** If FTS_LOG_MERGES is defined, call sqlite3_log() to report each automatic
           79  +** and incremental merge operation that takes place. This is used for 
           80  +** debugging FTS only, it should not usually be turned on in production
           81  +** systems.
           82  +*/
           83  +#ifdef FTS3_LOG_MERGES
           84  +static void fts3LogMerge(int nMerge, sqlite3_int64 iAbsLevel){
           85  +  sqlite3_log(SQLITE_OK, "%d-way merge from level %d", nMerge, (int)iAbsLevel);
           86  +}
           87  +#else
           88  +#define fts3LogMerge(x, y)
           89  +#endif
           90  +
    65     91   
    66     92   typedef struct PendingList PendingList;
    67     93   typedef struct SegmentNode SegmentNode;
    68     94   typedef struct SegmentWriter SegmentWriter;
    69     95   
    70     96   /*
    71     97   ** An instance of the following data structure is used to build doclists
................................................................................
   220    246   #define SQL_SELECT_SEGDIR_MAX_LEVEL   15
   221    247   #define SQL_DELETE_SEGDIR_LEVEL       16
   222    248   #define SQL_DELETE_SEGMENTS_RANGE     17
   223    249   #define SQL_CONTENT_INSERT            18
   224    250   #define SQL_DELETE_DOCSIZE            19
   225    251   #define SQL_REPLACE_DOCSIZE           20
   226    252   #define SQL_SELECT_DOCSIZE            21
   227         -#define SQL_SELECT_DOCTOTAL           22
   228         -#define SQL_REPLACE_DOCTOTAL          23
          253  +#define SQL_SELECT_STAT               22
          254  +#define SQL_REPLACE_STAT              23
   229    255   
   230    256   #define SQL_SELECT_ALL_PREFIX_LEVEL   24
   231    257   #define SQL_DELETE_ALL_TERMS_SEGDIR   25
   232         -
   233    258   #define SQL_DELETE_SEGDIR_RANGE       26
   234         -
   235    259   #define SQL_SELECT_ALL_LANGID         27
          260  +#define SQL_FIND_MERGE_LEVEL          28
          261  +#define SQL_MAX_LEAF_NODE_ESTIMATE    29
          262  +#define SQL_DELETE_SEGDIR_ENTRY       30
          263  +#define SQL_SHIFT_SEGDIR_ENTRY        31
          264  +#define SQL_SELECT_SEGDIR             32
          265  +#define SQL_CHOMP_SEGDIR              33
          266  +#define SQL_SEGMENT_IS_APPENDABLE     34
          267  +#define SQL_SELECT_INDEXES            35
          268  +#define SQL_SELECT_MXLEVEL            36
   236    269   
   237    270   /*
   238    271   ** This function is used to obtain an SQLite prepared statement handle
   239    272   ** for the statement identified by the second argument. If successful,
   240    273   ** *pp is set to the requested statement handle and SQLITE_OK returned.
   241    274   ** Otherwise, an SQLite error code is returned and *pp is set to 0.
   242    275   **
................................................................................
   257    290   /* 2  */  "DELETE FROM %Q.'%q_content'",
   258    291   /* 3  */  "DELETE FROM %Q.'%q_segments'",
   259    292   /* 4  */  "DELETE FROM %Q.'%q_segdir'",
   260    293   /* 5  */  "DELETE FROM %Q.'%q_docsize'",
   261    294   /* 6  */  "DELETE FROM %Q.'%q_stat'",
   262    295   /* 7  */  "SELECT %s WHERE rowid=?",
   263    296   /* 8  */  "SELECT (SELECT max(idx) FROM %Q.'%q_segdir' WHERE level = ?) + 1",
   264         -/* 9  */  "INSERT INTO %Q.'%q_segments'(blockid, block) VALUES(?, ?)",
          297  +/* 9  */  "REPLACE INTO %Q.'%q_segments'(blockid, block) VALUES(?, ?)",
   265    298   /* 10 */  "SELECT coalesce((SELECT max(blockid) FROM %Q.'%q_segments') + 1, 1)",
   266         -/* 11 */  "INSERT INTO %Q.'%q_segdir' VALUES(?,?,?,?,?,?)",
          299  +/* 11 */  "REPLACE INTO %Q.'%q_segdir' VALUES(?,?,?,?,?,?)",
   267    300   
   268    301             /* Return segments in order from oldest to newest.*/ 
   269    302   /* 12 */  "SELECT idx, start_block, leaves_end_block, end_block, root "
   270    303               "FROM %Q.'%q_segdir' WHERE level = ? ORDER BY idx ASC",
   271    304   /* 13 */  "SELECT idx, start_block, leaves_end_block, end_block, root "
   272    305               "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?"
   273    306               "ORDER BY level DESC, idx ASC",
................................................................................
   277    310   
   278    311   /* 16 */  "DELETE FROM %Q.'%q_segdir' WHERE level = ?",
   279    312   /* 17 */  "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?",
   280    313   /* 18 */  "INSERT INTO %Q.'%q_content' VALUES(%s)",
   281    314   /* 19 */  "DELETE FROM %Q.'%q_docsize' WHERE docid = ?",
   282    315   /* 20 */  "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)",
   283    316   /* 21 */  "SELECT size FROM %Q.'%q_docsize' WHERE docid=?",
   284         -/* 22 */  "SELECT value FROM %Q.'%q_stat' WHERE id=0",
   285         -/* 23 */  "REPLACE INTO %Q.'%q_stat' VALUES(0,?)",
          317  +/* 22 */  "SELECT value FROM %Q.'%q_stat' WHERE id=?",
          318  +/* 23 */  "REPLACE INTO %Q.'%q_stat' VALUES(?,?)",
   286    319   /* 24 */  "",
   287    320   /* 25 */  "",
   288    321   
   289    322   /* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?",
   290    323   /* 27 */ "SELECT DISTINCT level / (1024 * ?) FROM %Q.'%q_segdir'",
   291    324   
          325  +/* This statement is used to determine which level to read the input from
          326  +** when performing an incremental merge. It returns the absolute level number
          327  +** of the oldest level in the db that contains at least ? segments. Or,
          328  +** if no level in the FTS index contains more than ? segments, the statement
          329  +** returns zero rows.  */
          330  +/* 28 */ "SELECT level FROM %Q.'%q_segdir' GROUP BY level HAVING count(*)>=?"
          331  +         "  ORDER BY (level %% 1024) ASC LIMIT 1",
          332  +
          333  +/* Estimate the upper limit on the number of leaf nodes in a new segment
          334  +** created by merging the oldest :2 segments from absolute level :1. See 
          335  +** function sqlite3Fts3Incrmerge() for details.  */
          336  +/* 29 */ "SELECT 2 * total(1 + leaves_end_block - start_block) "
          337  +         "  FROM %Q.'%q_segdir' WHERE level = ? AND idx < ?",
          338  +
          339  +/* SQL_DELETE_SEGDIR_ENTRY
          340  +**   Delete the %_segdir entry on absolute level :1 with index :2.  */
          341  +/* 30 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ? AND idx = ?",
          342  +
          343  +/* SQL_SHIFT_SEGDIR_ENTRY
          344  +**   Modify the idx value for the segment with idx=:3 on absolute level :2
          345  +**   to :1.  */
          346  +/* 31 */ "UPDATE %Q.'%q_segdir' SET idx = ? WHERE level=? AND idx=?",
          347  +
          348  +/* SQL_SELECT_SEGDIR
          349  +**   Read a single entry from the %_segdir table. The entry from absolute 
          350  +**   level :1 with index value :2.  */
          351  +/* 32 */  "SELECT idx, start_block, leaves_end_block, end_block, root "
          352  +            "FROM %Q.'%q_segdir' WHERE level = ? AND idx = ?",
          353  +
          354  +/* SQL_CHOMP_SEGDIR
          355  +**   Update the start_block (:1) and root (:2) fields of the %_segdir
          356  +**   entry located on absolute level :3 with index :4.  */
          357  +/* 33 */  "UPDATE %Q.'%q_segdir' SET start_block = ?, root = ?"
          358  +            "WHERE level = ? AND idx = ?",
          359  +
          360  +/* SQL_SEGMENT_IS_APPENDABLE
          361  +**   Return a single row if the segment with end_block=? is appendable. Or
          362  +**   no rows otherwise.  */
          363  +/* 34 */  "SELECT 1 FROM %Q.'%q_segments' WHERE blockid=? AND block IS NULL",
          364  +
          365  +/* SQL_SELECT_INDEXES
          366  +**   Return the list of valid segment indexes for absolute level ?  */
          367  +/* 35 */  "SELECT idx FROM %Q.'%q_segdir' WHERE level=? ORDER BY 1 ASC",
          368  +
          369  +/* SQL_SELECT_MXLEVEL
          370  +**   Return the largest relative level in the FTS index or indexes.  */
          371  +/* 36 */  "SELECT max( level %% 1024 ) FROM %Q.'%q_segdir'"
   292    372     };
   293    373     int rc = SQLITE_OK;
   294    374     sqlite3_stmt *pStmt;
   295    375   
   296    376     assert( SizeofArray(azSql)==SizeofArray(p->aStmt) );
   297    377     assert( eStmt<SizeofArray(azSql) && eStmt>=0 );
   298    378     
................................................................................
   321    401       for(i=0; rc==SQLITE_OK && i<nParam; i++){
   322    402         rc = sqlite3_bind_value(pStmt, i+1, apVal[i]);
   323    403       }
   324    404     }
   325    405     *pp = pStmt;
   326    406     return rc;
   327    407   }
          408  +
   328    409   
   329    410   static int fts3SelectDocsize(
   330    411     Fts3Table *pTab,                /* FTS3 table handle */
   331         -  int eStmt,                      /* Either SQL_SELECT_DOCSIZE or DOCTOTAL */
   332    412     sqlite3_int64 iDocid,           /* Docid to bind for SQL_SELECT_DOCSIZE */
   333    413     sqlite3_stmt **ppStmt           /* OUT: Statement handle */
   334    414   ){
   335    415     sqlite3_stmt *pStmt = 0;        /* Statement requested from fts3SqlStmt() */
   336    416     int rc;                         /* Return code */
   337    417   
   338         -  assert( eStmt==SQL_SELECT_DOCSIZE || eStmt==SQL_SELECT_DOCTOTAL );
   339         -
   340         -  rc = fts3SqlStmt(pTab, eStmt, &pStmt, 0);
          418  +  rc = fts3SqlStmt(pTab, SQL_SELECT_DOCSIZE, &pStmt, 0);
   341    419     if( rc==SQLITE_OK ){
   342         -    if( eStmt==SQL_SELECT_DOCSIZE ){
   343         -      sqlite3_bind_int64(pStmt, 1, iDocid);
   344         -    }
          420  +    sqlite3_bind_int64(pStmt, 1, iDocid);
   345    421       rc = sqlite3_step(pStmt);
   346    422       if( rc!=SQLITE_ROW || sqlite3_column_type(pStmt, 0)!=SQLITE_BLOB ){
   347    423         rc = sqlite3_reset(pStmt);
   348    424         if( rc==SQLITE_OK ) rc = FTS_CORRUPT_VTAB;
   349    425         pStmt = 0;
   350    426       }else{
   351    427         rc = SQLITE_OK;
................................................................................
   356    432     return rc;
   357    433   }
   358    434   
   359    435   int sqlite3Fts3SelectDoctotal(
   360    436     Fts3Table *pTab,                /* Fts3 table handle */
   361    437     sqlite3_stmt **ppStmt           /* OUT: Statement handle */
   362    438   ){
   363         -  return fts3SelectDocsize(pTab, SQL_SELECT_DOCTOTAL, 0, ppStmt);
          439  +  sqlite3_stmt *pStmt = 0;
          440  +  int rc;
          441  +  rc = fts3SqlStmt(pTab, SQL_SELECT_STAT, &pStmt, 0);
          442  +  if( rc==SQLITE_OK ){
          443  +    sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL);
          444  +    if( sqlite3_step(pStmt)!=SQLITE_ROW
          445  +     || sqlite3_column_type(pStmt, 0)!=SQLITE_BLOB
          446  +    ){
          447  +      rc = sqlite3_reset(pStmt);
          448  +      if( rc==SQLITE_OK ) rc = FTS_CORRUPT_VTAB;
          449  +      pStmt = 0;
          450  +    }
          451  +  }
          452  +  *ppStmt = pStmt;
          453  +  return rc;
   364    454   }
   365    455   
   366    456   int sqlite3Fts3SelectDocsize(
   367    457     Fts3Table *pTab,                /* Fts3 table handle */
   368    458     sqlite3_int64 iDocid,           /* Docid to read size data for */
   369    459     sqlite3_stmt **ppStmt           /* OUT: Statement handle */
   370    460   ){
   371         -  return fts3SelectDocsize(pTab, SQL_SELECT_DOCSIZE, iDocid, ppStmt);
          461  +  return fts3SelectDocsize(pTab, iDocid, ppStmt);
   372    462   }
   373    463   
   374    464   /*
   375    465   ** Similar to fts3SqlStmt(). Except, after binding the parameters in
   376    466   ** array apVal[] to the SQL statement identified by eStmt, the statement
   377    467   ** is executed.
   378    468   **
................................................................................
   454    544   ** Language 1 indexes are allocated immediately following language 0.
   455    545   **
   456    546   ** So, for a system with nPrefix prefix indexes configured, the block of
   457    547   ** absolute levels that corresponds to language-id iLangid and index 
   458    548   ** iIndex starts at absolute level ((iLangid * (nPrefix+1) + iIndex) * 1024).
   459    549   */
   460    550   static sqlite3_int64 getAbsoluteLevel(
   461         -  Fts3Table *p, 
   462         -  int iLangid, 
   463         -  int iIndex, 
   464         -  int iLevel
          551  +  Fts3Table *p,                   /* FTS3 table handle */
          552  +  int iLangid,                    /* Language id */
          553  +  int iIndex,                     /* Index in p->aIndex[] */
          554  +  int iLevel                      /* Level of segments */
   465    555   ){
   466    556     sqlite3_int64 iBase;            /* First absolute level for iLangid/iIndex */
   467    557     assert( iLangid>=0 );
   468    558     assert( p->nIndex>0 );
   469    559     assert( iIndex>=0 && iIndex<p->nIndex );
   470    560   
   471    561     iBase = ((sqlite3_int64)iLangid * p->nIndex + iIndex) * FTS3_SEGDIR_MAXLEVEL;
   472    562     return iBase + iLevel;
   473    563   }
   474         -
   475    564   
   476    565   /*
   477    566   ** Set *ppStmt to a statement handle that may be used to iterate through
   478    567   ** all rows in the %_segdir table, from oldest to newest. If successful,
   479    568   ** return SQLITE_OK. If an error occurs while preparing the statement, 
   480    569   ** return an SQLite error code.
   481    570   **
................................................................................
  1034   1123     if( rc==SQLITE_OK ){
  1035   1124       /* If iNext is FTS3_MERGE_COUNT, indicating that level iLevel is already
  1036   1125       ** full, merge all segments in level iLevel into a single iLevel+1
  1037   1126       ** segment and allocate (newly freed) index 0 at level iLevel. Otherwise,
  1038   1127       ** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext.
  1039   1128       */
  1040   1129       if( iNext>=FTS3_MERGE_COUNT ){
         1130  +      fts3LogMerge(16, getAbsoluteLevel(p, iLangid, iIndex, iLevel));
  1041   1131         rc = fts3SegmentMerge(p, iLangid, iIndex, iLevel);
  1042   1132         *piIdx = 0;
  1043   1133       }else{
  1044   1134         *piIdx = iNext;
  1045   1135       }
  1046   1136     }
  1047   1137   
................................................................................
  1081   1171     char **paBlob,                  /* OUT: Blob data in malloc'd buffer */
  1082   1172     int *pnBlob,                    /* OUT: Size of blob data */
  1083   1173     int *pnLoad                     /* OUT: Bytes actually loaded */
  1084   1174   ){
  1085   1175     int rc;                         /* Return code */
  1086   1176   
  1087   1177     /* pnBlob must be non-NULL. paBlob may be NULL or non-NULL. */
  1088         -  assert( pnBlob);
         1178  +  assert( pnBlob );
  1089   1179   
  1090   1180     if( p->pSegments ){
  1091   1181       rc = sqlite3_blob_reopen(p->pSegments, iBlockid);
  1092   1182     }else{
  1093   1183       if( 0==p->zSegmentsTbl ){
  1094   1184         p->zSegmentsTbl = sqlite3_mprintf("%s_segments", p->zName);
  1095   1185         if( 0==p->zSegmentsTbl ) return SQLITE_NOMEM;
................................................................................
  1422   1512   ){
  1423   1513     Fts3Table *p = (Fts3Table*)pCsr->base.pVtab;
  1424   1514     int nOvfl = 0;
  1425   1515     int ii;
  1426   1516     int rc = SQLITE_OK;
  1427   1517     int pgsz = p->nPgsz;
  1428   1518   
  1429         -  assert( p->bHasStat );
         1519  +  assert( p->bFts4 );
  1430   1520     assert( pgsz>0 );
  1431   1521   
  1432   1522     for(ii=0; rc==SQLITE_OK && ii<pMsr->nSegment; ii++){
  1433   1523       Fts3SegReader *pReader = pMsr->apSegment[ii];
  1434   1524       if( !fts3SegReaderIsPending(pReader) 
  1435   1525        && !fts3SegReaderIsRootOnly(pReader) 
  1436   1526       ){
................................................................................
  1779   1869       sqlite3_bind_int64(pStmt, 1, iBlock);
  1780   1870       sqlite3_bind_blob(pStmt, 2, z, n, SQLITE_STATIC);
  1781   1871       sqlite3_step(pStmt);
  1782   1872       rc = sqlite3_reset(pStmt);
  1783   1873     }
  1784   1874     return rc;
  1785   1875   }
         1876  +
         1877  +/*
         1878  +** Find the largest relative level number in the table. If successful, set
         1879  +** *pnMax to this value and return SQLITE_OK. Otherwise, if an error occurs,
         1880  +** set *pnMax to zero and return an SQLite error code.
         1881  +*/
         1882  +int sqlite3Fts3MaxLevel(Fts3Table *p, int *pnMax){
         1883  +  int rc;
         1884  +  int mxLevel = 0;
         1885  +  sqlite3_stmt *pStmt = 0;
         1886  +
         1887  +  rc = fts3SqlStmt(p, SQL_SELECT_MXLEVEL, &pStmt, 0);
         1888  +  if( rc==SQLITE_OK ){
         1889  +    if( SQLITE_ROW==sqlite3_step(pStmt) ){
         1890  +      mxLevel = sqlite3_column_int(pStmt, 0);
         1891  +    }
         1892  +    rc = sqlite3_reset(pStmt);
         1893  +  }
         1894  +  *pnMax = mxLevel;
         1895  +  return rc;
         1896  +}
  1786   1897   
  1787   1898   /* 
  1788   1899   ** Insert a record into the %_segdir table.
  1789   1900   */
  1790   1901   static int fts3WriteSegdir(
  1791   1902     Fts3Table *p,                   /* Virtual table handle */
  1792   1903     sqlite3_int64 iLevel,           /* Value for "level" field (absolute level) */
................................................................................
  2096   2207   
  2097   2208     if( nData>0 && nData+nReq>p->nNodeSize ){
  2098   2209       int rc;
  2099   2210   
  2100   2211       /* The current leaf node is full. Write it out to the database. */
  2101   2212       rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, nData);
  2102   2213       if( rc!=SQLITE_OK ) return rc;
         2214  +    p->nLeafAdd++;
  2103   2215   
  2104   2216       /* Add the current term to the interior node tree. The term added to
  2105   2217       ** the interior tree must:
  2106   2218       **
  2107   2219       **   a) be greater than the largest term on the leaf node just written
  2108   2220       **      to the database (still available in pWriter->zTerm), and
  2109   2221       **
................................................................................
  2204   2316             p, iLevel, iIdx, pWriter->iFirst, iLastLeaf, iLast, zRoot, nRoot);
  2205   2317       }
  2206   2318     }else{
  2207   2319       /* The entire tree fits on the root node. Write it to the segdir table. */
  2208   2320       rc = fts3WriteSegdir(
  2209   2321           p, iLevel, iIdx, 0, 0, 0, pWriter->aData, pWriter->nData);
  2210   2322     }
         2323  +  p->nLeafAdd++;
  2211   2324     return rc;
  2212   2325   }
  2213   2326   
  2214   2327   /*
  2215   2328   ** Release all memory held by the SegmentWriter object passed as the 
  2216   2329   ** first argument.
  2217   2330   */
................................................................................
  2284   2397         getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1)
  2285   2398     );
  2286   2399     if( SQLITE_ROW==sqlite3_step(pStmt) ){
  2287   2400       *pnMax = sqlite3_column_int64(pStmt, 0);
  2288   2401     }
  2289   2402     return sqlite3_reset(pStmt);
  2290   2403   }
         2404  +
         2405  +/*
         2406  +** Delete all entries in the %_segments table associated with the segment
         2407  +** opened with seg-reader pSeg. This function does not affect the contents
         2408  +** of the %_segdir table.
         2409  +*/
         2410  +static int fts3DeleteSegment(
         2411  +  Fts3Table *p,                   /* FTS table handle */
         2412  +  Fts3SegReader *pSeg             /* Segment to delete */
         2413  +){
         2414  +  int rc = SQLITE_OK;             /* Return code */
         2415  +  if( pSeg->iStartBlock ){
         2416  +    sqlite3_stmt *pDelete;        /* SQL statement to delete rows */
         2417  +    rc = fts3SqlStmt(p, SQL_DELETE_SEGMENTS_RANGE, &pDelete, 0);
         2418  +    if( rc==SQLITE_OK ){
         2419  +      sqlite3_bind_int64(pDelete, 1, pSeg->iStartBlock);
         2420  +      sqlite3_bind_int64(pDelete, 2, pSeg->iEndBlock);
         2421  +      sqlite3_step(pDelete);
         2422  +      rc = sqlite3_reset(pDelete);
         2423  +    }
         2424  +  }
         2425  +  return rc;
         2426  +}
  2291   2427   
  2292   2428   /*
  2293   2429   ** This function is used after merging multiple segments into a single large
  2294   2430   ** segment to delete the old, now redundant, segment b-trees. Specifically,
  2295   2431   ** it:
  2296   2432   ** 
  2297   2433   **   1) Deletes all %_segments entries for the segments associated with 
................................................................................
  2307   2443     Fts3Table *p,                   /* Virtual table handle */
  2308   2444     int iLangid,                    /* Language id */
  2309   2445     int iIndex,                     /* Index for p->aIndex */
  2310   2446     int iLevel,                     /* Level of %_segdir entries to delete */
  2311   2447     Fts3SegReader **apSegment,      /* Array of SegReader objects */
  2312   2448     int nReader                     /* Size of array apSegment */
  2313   2449   ){
  2314         -  int rc;                         /* Return Code */
         2450  +  int rc = SQLITE_OK;             /* Return Code */
  2315   2451     int i;                          /* Iterator variable */
  2316         -  sqlite3_stmt *pDelete;          /* SQL statement to delete rows */
         2452  +  sqlite3_stmt *pDelete = 0;      /* SQL statement to delete rows */
  2317   2453   
  2318         -  rc = fts3SqlStmt(p, SQL_DELETE_SEGMENTS_RANGE, &pDelete, 0);
  2319   2454     for(i=0; rc==SQLITE_OK && i<nReader; i++){
  2320         -    Fts3SegReader *pSegment = apSegment[i];
  2321         -    if( pSegment->iStartBlock ){
  2322         -      sqlite3_bind_int64(pDelete, 1, pSegment->iStartBlock);
  2323         -      sqlite3_bind_int64(pDelete, 2, pSegment->iEndBlock);
  2324         -      sqlite3_step(pDelete);
  2325         -      rc = sqlite3_reset(pDelete);
  2326         -    }
         2455  +    rc = fts3DeleteSegment(p, apSegment[i]);
  2327   2456     }
  2328   2457     if( rc!=SQLITE_OK ){
  2329   2458       return rc;
  2330   2459     }
  2331   2460   
  2332   2461     assert( iLevel>=0 || iLevel==FTS3_SEGCURSOR_ALL );
  2333   2462     if( iLevel==FTS3_SEGCURSOR_ALL ){
................................................................................
  2895   3024   
  2896   3025   /* 
  2897   3026   ** Flush the contents of pendingTerms to level 0 segments.
  2898   3027   */
  2899   3028   int sqlite3Fts3PendingTermsFlush(Fts3Table *p){
  2900   3029     int rc = SQLITE_OK;
  2901   3030     int i;
         3031  +        
  2902   3032     for(i=0; rc==SQLITE_OK && i<p->nIndex; i++){
  2903   3033       rc = fts3SegmentMerge(p, p->iPrevLangid, i, FTS3_SEGCURSOR_PENDING);
  2904   3034       if( rc==SQLITE_DONE ) rc = SQLITE_OK;
  2905   3035     }
  2906   3036     sqlite3Fts3PendingTermsClear(p);
         3037  +
         3038  +  /* Determine the auto-incr-merge setting if unknown.  If enabled,
         3039  +  ** estimate the number of leaf blocks of content to be written
         3040  +  */
         3041  +  if( rc==SQLITE_OK && p->bHasStat
         3042  +   && p->bAutoincrmerge==0xff && p->nLeafAdd>0
         3043  +  ){
         3044  +    sqlite3_stmt *pStmt = 0;
         3045  +    rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pStmt, 0);
         3046  +    if( rc==SQLITE_OK ){
         3047  +      sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE);
         3048  +      rc = sqlite3_step(pStmt);
         3049  +      p->bAutoincrmerge = (rc==SQLITE_ROW && sqlite3_column_int(pStmt, 0));
         3050  +      rc = sqlite3_reset(pStmt);
         3051  +    }
         3052  +  }
  2907   3053     return rc;
  2908   3054   }
  2909   3055   
  2910   3056   /*
  2911   3057   ** Encode N integers as varints into a blob.
  2912   3058   */
  2913   3059   static void fts3EncodeIntArray(
................................................................................
  3010   3156     if( *pRC ) return;
  3011   3157     a = sqlite3_malloc( (sizeof(u32)+10)*nStat );
  3012   3158     if( a==0 ){
  3013   3159       *pRC = SQLITE_NOMEM;
  3014   3160       return;
  3015   3161     }
  3016   3162     pBlob = (char*)&a[nStat];
  3017         -  rc = fts3SqlStmt(p, SQL_SELECT_DOCTOTAL, &pStmt, 0);
         3163  +  rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pStmt, 0);
  3018   3164     if( rc ){
  3019   3165       sqlite3_free(a);
  3020   3166       *pRC = rc;
  3021   3167       return;
  3022   3168     }
         3169  +  sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL);
  3023   3170     if( sqlite3_step(pStmt)==SQLITE_ROW ){
  3024   3171       fts3DecodeIntArray(nStat, a,
  3025   3172            sqlite3_column_blob(pStmt, 0),
  3026   3173            sqlite3_column_bytes(pStmt, 0));
  3027   3174     }else{
  3028   3175       memset(a, 0, sizeof(u32)*(nStat) );
  3029   3176     }
................................................................................
  3039   3186         x = 0;
  3040   3187       }else{
  3041   3188         x = x + aSzIns[i] - aSzDel[i];
  3042   3189       }
  3043   3190       a[i+1] = x;
  3044   3191     }
  3045   3192     fts3EncodeIntArray(nStat, a, pBlob, &nBlob);
  3046         -  rc = fts3SqlStmt(p, SQL_REPLACE_DOCTOTAL, &pStmt, 0);
         3193  +  rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pStmt, 0);
  3047   3194     if( rc ){
  3048   3195       sqlite3_free(a);
  3049   3196       *pRC = rc;
  3050   3197       return;
  3051   3198     }
  3052         -  sqlite3_bind_blob(pStmt, 1, pBlob, nBlob, SQLITE_STATIC);
         3199  +  sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL);
         3200  +  sqlite3_bind_blob(pStmt, 2, pBlob, nBlob, SQLITE_STATIC);
  3053   3201     sqlite3_step(pStmt);
  3054   3202     *pRC = sqlite3_reset(pStmt);
  3055   3203     sqlite3_free(a);
  3056   3204   }
  3057   3205   
  3058   3206   /*
  3059   3207   ** Merge the entire database so that there is one segment for each 
................................................................................
  3150   3298         }else{
  3151   3299           nEntry++;
  3152   3300           for(iCol=0; iCol<=p->nColumn; iCol++){
  3153   3301             aSzIns[iCol] += aSz[iCol];
  3154   3302           }
  3155   3303         }
  3156   3304       }
  3157         -    if( p->bHasStat ){
         3305  +    if( p->bFts4 ){
  3158   3306         fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nEntry);
  3159   3307       }
  3160   3308       sqlite3_free(aSz);
  3161   3309   
  3162   3310       if( pStmt ){
  3163   3311         int rc2 = sqlite3_finalize(pStmt);
  3164   3312         if( rc==SQLITE_OK ){
................................................................................
  3166   3314         }
  3167   3315       }
  3168   3316     }
  3169   3317   
  3170   3318     return rc;
  3171   3319   }
  3172   3320   
         3321  +
         3322  +/*
         3323  +** This function opens a cursor used to read the input data for an 
         3324  +** incremental merge operation. Specifically, it opens a cursor to scan
         3325  +** the oldest nSeg segments (idx=0 through idx=(nSeg-1)) in absolute 
         3326  +** level iAbsLevel.
         3327  +*/
         3328  +static int fts3IncrmergeCsr(
         3329  +  Fts3Table *p,                   /* FTS3 table handle */
         3330  +  sqlite3_int64 iAbsLevel,        /* Absolute level to open */
         3331  +  int nSeg,                       /* Number of segments to merge */
         3332  +  Fts3MultiSegReader *pCsr        /* Cursor object to populate */
         3333  +){
         3334  +  int rc;                         /* Return Code */
         3335  +  sqlite3_stmt *pStmt = 0;        /* Statement used to read %_segdir entry */  
         3336  +  int nByte;                      /* Bytes allocated at pCsr->apSegment[] */
         3337  +
         3338  +  /* Allocate space for the Fts3MultiSegReader.aCsr[] array */
         3339  +  memset(pCsr, 0, sizeof(*pCsr));
         3340  +  nByte = sizeof(Fts3SegReader *) * nSeg;
         3341  +  pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc(nByte);
         3342  +
         3343  +  if( pCsr->apSegment==0 ){
         3344  +    rc = SQLITE_NOMEM;
         3345  +  }else{
         3346  +    memset(pCsr->apSegment, 0, nByte);
         3347  +    rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0);
         3348  +  }
         3349  +  if( rc==SQLITE_OK ){
         3350  +    int i;
         3351  +    int rc2;
         3352  +    sqlite3_bind_int64(pStmt, 1, iAbsLevel);
         3353  +    assert( pCsr->nSegment==0 );
         3354  +    for(i=0; rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW && i<nSeg; i++){
         3355  +      rc = sqlite3Fts3SegReaderNew(i, 0,
         3356  +          sqlite3_column_int64(pStmt, 1),        /* segdir.start_block */
         3357  +          sqlite3_column_int64(pStmt, 2),        /* segdir.leaves_end_block */
         3358  +          sqlite3_column_int64(pStmt, 3),        /* segdir.end_block */
         3359  +          sqlite3_column_blob(pStmt, 4),         /* segdir.root */
         3360  +          sqlite3_column_bytes(pStmt, 4),        /* segdir.root */
         3361  +          &pCsr->apSegment[i]
         3362  +      );
         3363  +      pCsr->nSegment++;
         3364  +    }
         3365  +    rc2 = sqlite3_reset(pStmt);
         3366  +    if( rc==SQLITE_OK ) rc = rc2;
         3367  +  }
         3368  +
         3369  +  return rc;
         3370  +}
         3371  +
         3372  +typedef struct IncrmergeWriter IncrmergeWriter;
         3373  +typedef struct NodeWriter NodeWriter;
         3374  +typedef struct Blob Blob;
         3375  +typedef struct NodeReader NodeReader;
         3376  +
         3377  +/*
         3378  +** An instance of the following structure is used as a dynamic buffer
         3379  +** to build up nodes or other blobs of data in.
         3380  +**
         3381  +** The function blobGrowBuffer() is used to extend the allocation.
         3382  +*/
         3383  +struct Blob {
         3384  +  char *a;                        /* Pointer to allocation */
         3385  +  int n;                          /* Number of valid bytes of data in a[] */
         3386  +  int nAlloc;                     /* Allocated size of a[] (nAlloc>=n) */
         3387  +};
         3388  +
         3389  +/*
         3390  +** This structure is used to build up buffers containing segment b-tree 
         3391  +** nodes (blocks).
         3392  +*/
         3393  +struct NodeWriter {
         3394  +  sqlite3_int64 iBlock;           /* Current block id */
         3395  +  Blob key;                       /* Last key written to the current block */
         3396  +  Blob block;                     /* Current block image */
         3397  +};
         3398  +
         3399  +/*
         3400  +** An object of this type contains the state required to create or append
         3401  +** to an appendable b-tree segment.
         3402  +*/
         3403  +struct IncrmergeWriter {
         3404  +  int nLeafEst;                   /* Space allocated for leaf blocks */
         3405  +  int nWork;                      /* Number of leaf pages flushed */
         3406  +  sqlite3_int64 iAbsLevel;        /* Absolute level of input segments */
         3407  +  int iIdx;                       /* Index of *output* segment in iAbsLevel+1 */
         3408  +  sqlite3_int64 iStart;           /* Block number of first allocated block */
         3409  +  sqlite3_int64 iEnd;             /* Block number of last allocated block */
         3410  +  NodeWriter aNodeWriter[FTS_MAX_APPENDABLE_HEIGHT];
         3411  +};
         3412  +
         3413  +/*
         3414  +** An object of the following type is used to read data from a single
         3415  +** FTS segment node. See the following functions:
         3416  +**
         3417  +**     nodeReaderInit()
         3418  +**     nodeReaderNext()
         3419  +**     nodeReaderRelease()
         3420  +*/
         3421  +struct NodeReader {
         3422  +  const char *aNode;
         3423  +  int nNode;
         3424  +  int iOff;                       /* Current offset within aNode[] */
         3425  +
         3426  +  /* Output variables. Containing the current node entry. */
         3427  +  sqlite3_int64 iChild;           /* Pointer to child node */
         3428  +  Blob term;                      /* Current term */
         3429  +  const char *aDoclist;           /* Pointer to doclist */
         3430  +  int nDoclist;                   /* Size of doclist in bytes */
         3431  +};
         3432  +
         3433  +/*
         3434  +** If *pRc is not SQLITE_OK when this function is called, it is a no-op.
         3435  +** Otherwise, if the allocation at pBlob->a is not already at least nMin
         3436  +** bytes in size, extend (realloc) it to be so.
         3437  +**
         3438  +** If an OOM error occurs, set *pRc to SQLITE_NOMEM and leave pBlob->a
         3439  +** unmodified. Otherwise, if the allocation succeeds, update pBlob->nAlloc
         3440  +** to reflect the new size of the pBlob->a[] buffer.
         3441  +*/
         3442  +static void blobGrowBuffer(Blob *pBlob, int nMin, int *pRc){
         3443  +  if( *pRc==SQLITE_OK && nMin>pBlob->nAlloc ){
         3444  +    int nAlloc = nMin;
         3445  +    char *a = (char *)sqlite3_realloc(pBlob->a, nAlloc);
         3446  +    if( a ){
         3447  +      pBlob->nAlloc = nAlloc;
         3448  +      pBlob->a = a;
         3449  +    }else{
         3450  +      *pRc = SQLITE_NOMEM;
         3451  +    }
         3452  +  }
         3453  +}
         3454  +
         3455  +/*
         3456  +** Attempt to advance the node-reader object passed as the first argument to
         3457  +** the next entry on the node. 
         3458  +**
         3459  +** Return an error code if an error occurs (SQLITE_NOMEM is possible). 
         3460  +** Otherwise return SQLITE_OK. If there is no next entry on the node
         3461  +** (e.g. because the current entry is the last) set NodeReader->aNode to
         3462  +** NULL to indicate EOF. Otherwise, populate the NodeReader structure output 
         3463  +** variables for the new entry.
         3464  +*/
         3465  +static int nodeReaderNext(NodeReader *p){
         3466  +  int bFirst = (p->term.n==0);    /* True for first term on the node */
         3467  +  int nPrefix = 0;                /* Bytes to copy from previous term */
         3468  +  int nSuffix = 0;                /* Bytes to append to the prefix */
         3469  +  int rc = SQLITE_OK;             /* Return code */
         3470  +
         3471  +  assert( p->aNode );
         3472  +  if( p->iChild && bFirst==0 ) p->iChild++;
         3473  +  if( p->iOff>=p->nNode ){
         3474  +    /* EOF */
         3475  +    p->aNode = 0;
         3476  +  }else{
         3477  +    if( bFirst==0 ){
         3478  +      p->iOff += sqlite3Fts3GetVarint32(&p->aNode[p->iOff], &nPrefix);
         3479  +    }
         3480  +    p->iOff += sqlite3Fts3GetVarint32(&p->aNode[p->iOff], &nSuffix);
         3481  +
         3482  +    blobGrowBuffer(&p->term, nPrefix+nSuffix, &rc);
         3483  +    if( rc==SQLITE_OK ){
         3484  +      memcpy(&p->term.a[nPrefix], &p->aNode[p->iOff], nSuffix);
         3485  +      p->term.n = nPrefix+nSuffix;
         3486  +      p->iOff += nSuffix;
         3487  +      if( p->iChild==0 ){
         3488  +        p->iOff += sqlite3Fts3GetVarint32(&p->aNode[p->iOff], &p->nDoclist);
         3489  +        p->aDoclist = &p->aNode[p->iOff];
         3490  +        p->iOff += p->nDoclist;
         3491  +      }
         3492  +    }
         3493  +  }
         3494  +
         3495  +  assert( p->iOff<=p->nNode );
         3496  +
         3497  +  return rc;
         3498  +}
         3499  +
         3500  +/*
         3501  +** Release all dynamic resources held by node-reader object *p.
         3502  +*/
         3503  +static void nodeReaderRelease(NodeReader *p){
         3504  +  sqlite3_free(p->term.a);
         3505  +}
         3506  +
         3507  +/*
         3508  +** Initialize a node-reader object to read the node in buffer aNode/nNode.
         3509  +**
         3510  +** If successful, SQLITE_OK is returned and the NodeReader object set to 
         3511  +** point to the first entry on the node (if any). Otherwise, an SQLite
         3512  +** error code is returned.
         3513  +*/
         3514  +static int nodeReaderInit(NodeReader *p, const char *aNode, int nNode){
         3515  +  memset(p, 0, sizeof(NodeReader));
         3516  +  p->aNode = aNode;
         3517  +  p->nNode = nNode;
         3518  +
         3519  +  /* Figure out if this is a leaf or an internal node. */
         3520  +  if( p->aNode[0] ){
         3521  +    /* An internal node. */
         3522  +    p->iOff = 1 + sqlite3Fts3GetVarint(&p->aNode[1], &p->iChild);
         3523  +  }else{
         3524  +    p->iOff = 1;
         3525  +  }
         3526  +
         3527  +  return nodeReaderNext(p);
         3528  +}
         3529  +
         3530  +/*
         3531  +** This function is called while writing an FTS segment each time a leaf o
         3532  +** node is finished and written to disk. The key (zTerm/nTerm) is guaranteed
         3533  +** to be greater than the largest key on the node just written, but smaller
         3534  +** than or equal to the first key that will be written to the next leaf
         3535  +** node.
         3536  +**
         3537  +** The block id of the leaf node just written to disk may be found in
         3538  +** (pWriter->aNodeWriter[0].iBlock) when this function is called.
         3539  +*/
         3540  +static int fts3IncrmergePush(
         3541  +  Fts3Table *p,                   /* Fts3 table handle */
         3542  +  IncrmergeWriter *pWriter,       /* Writer object */
         3543  +  const char *zTerm,              /* Term to write to internal node */
         3544  +  int nTerm                       /* Bytes at zTerm */
         3545  +){
         3546  +  sqlite3_int64 iPtr = pWriter->aNodeWriter[0].iBlock;
         3547  +  int iLayer;
         3548  +
         3549  +  assert( nTerm>0 );
         3550  +  for(iLayer=1; ALWAYS(iLayer<FTS_MAX_APPENDABLE_HEIGHT); iLayer++){
         3551  +    sqlite3_int64 iNextPtr = 0;
         3552  +    NodeWriter *pNode = &pWriter->aNodeWriter[iLayer];
         3553  +    int rc = SQLITE_OK;
         3554  +    int nPrefix;
         3555  +    int nSuffix;
         3556  +    int nSpace;
         3557  +
         3558  +    /* Figure out how much space the key will consume if it is written to
         3559  +    ** the current node of layer iLayer. Due to the prefix compression, 
         3560  +    ** the space required changes depending on which node the key is to
         3561  +    ** be added to.  */
         3562  +    nPrefix = fts3PrefixCompress(pNode->key.a, pNode->key.n, zTerm, nTerm);
         3563  +    nSuffix = nTerm - nPrefix;
         3564  +    nSpace  = sqlite3Fts3VarintLen(nPrefix);
         3565  +    nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix;
         3566  +
         3567  +    if( pNode->key.n==0 || (pNode->block.n + nSpace)<=p->nNodeSize ){ 
         3568  +      /* If the current node of layer iLayer contains zero keys, or if adding
         3569  +      ** the key to it will not cause it to grow to larger than nNodeSize 
         3570  +      ** bytes in size, write the key here.  */
         3571  +
         3572  +      Blob *pBlk = &pNode->block;
         3573  +      if( pBlk->n==0 ){
         3574  +        blobGrowBuffer(pBlk, p->nNodeSize, &rc);
         3575  +        if( rc==SQLITE_OK ){
         3576  +          pBlk->a[0] = (char)iLayer;
         3577  +          pBlk->n = 1 + sqlite3Fts3PutVarint(&pBlk->a[1], iPtr);
         3578  +        }
         3579  +      }
         3580  +      blobGrowBuffer(pBlk, pBlk->n + nSpace, &rc);
         3581  +      blobGrowBuffer(&pNode->key, nTerm, &rc);
         3582  +
         3583  +      if( rc==SQLITE_OK ){
         3584  +        if( pNode->key.n ){
         3585  +          pBlk->n += sqlite3Fts3PutVarint(&pBlk->a[pBlk->n], nPrefix);
         3586  +        }
         3587  +        pBlk->n += sqlite3Fts3PutVarint(&pBlk->a[pBlk->n], nSuffix);
         3588  +        memcpy(&pBlk->a[pBlk->n], &zTerm[nPrefix], nSuffix);
         3589  +        pBlk->n += nSuffix;
         3590  +
         3591  +        memcpy(pNode->key.a, zTerm, nTerm);
         3592  +        pNode->key.n = nTerm;
         3593  +      }
         3594  +    }else{
         3595  +      /* Otherwise, flush the the current node of layer iLayer to disk.
         3596  +      ** Then allocate a new, empty sibling node. The key will be written
         3597  +      ** into the parent of this node. */
         3598  +      rc = fts3WriteSegment(p, pNode->iBlock, pNode->block.a, pNode->block.n);
         3599  +
         3600  +      assert( pNode->block.nAlloc>=p->nNodeSize );
         3601  +      pNode->block.a[0] = (char)iLayer;
         3602  +      pNode->block.n = 1 + sqlite3Fts3PutVarint(&pNode->block.a[1], iPtr+1);
         3603  +
         3604  +      iNextPtr = pNode->iBlock;
         3605  +      pNode->iBlock++;
         3606  +      pNode->key.n = 0;
         3607  +    }
         3608  +
         3609  +    if( rc!=SQLITE_OK || iNextPtr==0 ) return rc;
         3610  +    iPtr = iNextPtr;
         3611  +  }
         3612  +
         3613  +  assert( 0 );
         3614  +  return 0;
         3615  +}
         3616  +
         3617  +/*
         3618  +** Append a term and (optionally) doclist to the FTS segment node currently
         3619  +** stored in blob *pNode. The node need not contain any terms, but the
         3620  +** header must be written before this function is called.
         3621  +**
         3622  +** A node header is a single 0x00 byte for a leaf node, or a height varint
         3623  +** followed by the left-hand-child varint for an internal node.
         3624  +**
         3625  +** The term to be appended is passed via arguments zTerm/nTerm. For a 
         3626  +** leaf node, the doclist is passed as aDoclist/nDoclist. For an internal
         3627  +** node, both aDoclist and nDoclist must be passed 0.
         3628  +**
         3629  +** If the size of the value in blob pPrev is zero, then this is the first
         3630  +** term written to the node. Otherwise, pPrev contains a copy of the 
         3631  +** previous term. Before this function returns, it is updated to contain a
         3632  +** copy of zTerm/nTerm.
         3633  +**
         3634  +** It is assumed that the buffer associated with pNode is already large
         3635  +** enough to accommodate the new entry. The buffer associated with pPrev
         3636  +** is extended by this function if requrired.
         3637  +**
         3638  +** If an error (i.e. OOM condition) occurs, an SQLite error code is
         3639  +** returned. Otherwise, SQLITE_OK.
         3640  +*/
         3641  +static int fts3AppendToNode(
         3642  +  Blob *pNode,                    /* Current node image to append to */
         3643  +  Blob *pPrev,                    /* Buffer containing previous term written */
         3644  +  const char *zTerm,              /* New term to write */
         3645  +  int nTerm,                      /* Size of zTerm in bytes */
         3646  +  const char *aDoclist,           /* Doclist (or NULL) to write */
         3647  +  int nDoclist                    /* Size of aDoclist in bytes */ 
         3648  +){
         3649  +  int rc = SQLITE_OK;             /* Return code */
         3650  +  int bFirst = (pPrev->n==0);     /* True if this is the first term written */
         3651  +  int nPrefix;                    /* Size of term prefix in bytes */
         3652  +  int nSuffix;                    /* Size of term suffix in bytes */
         3653  +
         3654  +  /* Node must have already been started. There must be a doclist for a
         3655  +  ** leaf node, and there must not be a doclist for an internal node.  */
         3656  +  assert( pNode->n>0 );
         3657  +  assert( (pNode->a[0]=='\0')==(aDoclist!=0) );
         3658  +
         3659  +  blobGrowBuffer(pPrev, nTerm, &rc);
         3660  +  if( rc!=SQLITE_OK ) return rc;
         3661  +
         3662  +  nPrefix = fts3PrefixCompress(pPrev->a, pPrev->n, zTerm, nTerm);
         3663  +  nSuffix = nTerm - nPrefix;
         3664  +  memcpy(pPrev->a, zTerm, nTerm);
         3665  +  pPrev->n = nTerm;
         3666  +
         3667  +  if( bFirst==0 ){
         3668  +    pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nPrefix);
         3669  +  }
         3670  +  pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nSuffix);
         3671  +  memcpy(&pNode->a[pNode->n], &zTerm[nPrefix], nSuffix);
         3672  +  pNode->n += nSuffix;
         3673  +
         3674  +  if( aDoclist ){
         3675  +    pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nDoclist);
         3676  +    memcpy(&pNode->a[pNode->n], aDoclist, nDoclist);
         3677  +    pNode->n += nDoclist;
         3678  +  }
         3679  +
         3680  +  assert( pNode->n<=pNode->nAlloc );
         3681  +
         3682  +  return SQLITE_OK;
         3683  +}
         3684  +
         3685  +/*
         3686  +** Append the current term and doclist pointed to by cursor pCsr to the
         3687  +** appendable b-tree segment opened for writing by pWriter.
         3688  +**
         3689  +** Return SQLITE_OK if successful, or an SQLite error code otherwise.
         3690  +*/
         3691  +static int fts3IncrmergeAppend(
         3692  +  Fts3Table *p,                   /* Fts3 table handle */
         3693  +  IncrmergeWriter *pWriter,       /* Writer object */
         3694  +  Fts3MultiSegReader *pCsr        /* Cursor containing term and doclist */
         3695  +){
         3696  +  const char *zTerm = pCsr->zTerm;
         3697  +  int nTerm = pCsr->nTerm;
         3698  +  const char *aDoclist = pCsr->aDoclist;
         3699  +  int nDoclist = pCsr->nDoclist;
         3700  +  int rc = SQLITE_OK;           /* Return code */
         3701  +  int nSpace;                   /* Total space in bytes required on leaf */
         3702  +  int nPrefix;                  /* Size of prefix shared with previous term */
         3703  +  int nSuffix;                  /* Size of suffix (nTerm - nPrefix) */
         3704  +  NodeWriter *pLeaf;            /* Object used to write leaf nodes */
         3705  +
         3706  +  pLeaf = &pWriter->aNodeWriter[0];
         3707  +  nPrefix = fts3PrefixCompress(pLeaf->key.a, pLeaf->key.n, zTerm, nTerm);
         3708  +  nSuffix = nTerm - nPrefix;
         3709  +
         3710  +  nSpace  = sqlite3Fts3VarintLen(nPrefix);
         3711  +  nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix;
         3712  +  nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist;
         3713  +
         3714  +  /* If the current block is not empty, and if adding this term/doclist
         3715  +  ** to the current block would make it larger than Fts3Table.nNodeSize
         3716  +  ** bytes, write this block out to the database. */
         3717  +  if( pLeaf->block.n>0 && (pLeaf->block.n + nSpace)>p->nNodeSize ){
         3718  +    rc = fts3WriteSegment(p, pLeaf->iBlock, pLeaf->block.a, pLeaf->block.n);
         3719  +    pWriter->nWork++;
         3720  +
         3721  +    /* Add the current term to the parent node. The term added to the 
         3722  +    ** parent must:
         3723  +    **
         3724  +    **   a) be greater than the largest term on the leaf node just written
         3725  +    **      to the database (still available in pLeaf->key), and
         3726  +    **
         3727  +    **   b) be less than or equal to the term about to be added to the new
         3728  +    **      leaf node (zTerm/nTerm).
         3729  +    **
         3730  +    ** In other words, it must be the prefix of zTerm 1 byte longer than
         3731  +    ** the common prefix (if any) of zTerm and pWriter->zTerm.
         3732  +    */
         3733  +    if( rc==SQLITE_OK ){
         3734  +      rc = fts3IncrmergePush(p, pWriter, zTerm, nPrefix+1);
         3735  +    }
         3736  +
         3737  +    /* Advance to the next output block */
         3738  +    pLeaf->iBlock++;
         3739  +    pLeaf->key.n = 0;
         3740  +    pLeaf->block.n = 0;
         3741  +
         3742  +    nPrefix = 0;
         3743  +    nSuffix = nTerm;
         3744  +    nSpace  = 1;
         3745  +    nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix;
         3746  +    nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist;
         3747  +  }
         3748  +
         3749  +  blobGrowBuffer(&pLeaf->block, pLeaf->block.n + nSpace, &rc);
         3750  +
         3751  +  if( rc==SQLITE_OK ){
         3752  +    if( pLeaf->block.n==0 ){
         3753  +      pLeaf->block.n = 1;
         3754  +      pLeaf->block.a[0] = '\0';
         3755  +    }
         3756  +    rc = fts3AppendToNode(
         3757  +        &pLeaf->block, &pLeaf->key, zTerm, nTerm, aDoclist, nDoclist
         3758  +    );
         3759  +  }
         3760  +
         3761  +  return rc;
         3762  +}
         3763  +
         3764  +/*
         3765  +** This function is called to release all dynamic resources held by the
         3766  +** merge-writer object pWriter, and if no error has occurred, to flush
         3767  +** all outstanding node buffers held by pWriter to disk.
         3768  +**
         3769  +** If *pRc is not SQLITE_OK when this function is called, then no attempt
         3770  +** is made to write any data to disk. Instead, this function serves only
         3771  +** to release outstanding resources.
         3772  +**
         3773  +** Otherwise, if *pRc is initially SQLITE_OK and an error occurs while
         3774  +** flushing buffers to disk, *pRc is set to an SQLite error code before
         3775  +** returning.
         3776  +*/
         3777  +static void fts3IncrmergeRelease(
         3778  +  Fts3Table *p,                   /* FTS3 table handle */
         3779  +  IncrmergeWriter *pWriter,       /* Merge-writer object */
         3780  +  int *pRc                        /* IN/OUT: Error code */
         3781  +){
         3782  +  int i;                          /* Used to iterate through non-root layers */
         3783  +  int iRoot;                      /* Index of root in pWriter->aNodeWriter */
         3784  +  NodeWriter *pRoot;              /* NodeWriter for root node */
         3785  +  int rc = *pRc;                  /* Error code */
         3786  +
         3787  +  /* Set iRoot to the index in pWriter->aNodeWriter[] of the output segment 
         3788  +  ** root node. If the segment fits entirely on a single leaf node, iRoot
         3789  +  ** will be set to 0. If the root node is the parent of the leaves, iRoot
         3790  +  ** will be 1. And so on.  */
         3791  +  for(iRoot=FTS_MAX_APPENDABLE_HEIGHT-1; iRoot>=0; iRoot--){
         3792  +    NodeWriter *pNode = &pWriter->aNodeWriter[iRoot];
         3793  +    if( pNode->block.n>0 ) break;
         3794  +    assert( *pRc || pNode->block.nAlloc==0 );
         3795  +    assert( *pRc || pNode->key.nAlloc==0 );
         3796  +    sqlite3_free(pNode->block.a);
         3797  +    sqlite3_free(pNode->key.a);
         3798  +  }
         3799  +
         3800  +  /* Empty output segment. This is a no-op. */
         3801  +  if( iRoot<0 ) return;
         3802  +
         3803  +  /* The entire output segment fits on a single node. Normally, this means
         3804  +  ** the node would be stored as a blob in the "root" column of the %_segdir
         3805  +  ** table. However, this is not permitted in this case. The problem is that 
         3806  +  ** space has already been reserved in the %_segments table, and so the 
         3807  +  ** start_block and end_block fields of the %_segdir table must be populated. 
         3808  +  ** And, by design or by accident, released versions of FTS cannot handle 
         3809  +  ** segments that fit entirely on the root node with start_block!=0.
         3810  +  **
         3811  +  ** Instead, create a synthetic root node that contains nothing but a 
         3812  +  ** pointer to the single content node. So that the segment consists of a
         3813  +  ** single leaf and a single interior (root) node.
         3814  +  **
         3815  +  ** Todo: Better might be to defer allocating space in the %_segments 
         3816  +  ** table until we are sure it is needed.
         3817  +  */
         3818  +  if( iRoot==0 ){
         3819  +    Blob *pBlock = &pWriter->aNodeWriter[1].block;
         3820  +    blobGrowBuffer(pBlock, 1 + FTS3_VARINT_MAX, &rc);
         3821  +    if( rc==SQLITE_OK ){
         3822  +      pBlock->a[0] = 0x01;
         3823  +      pBlock->n = 1 + sqlite3Fts3PutVarint(
         3824  +          &pBlock->a[1], pWriter->aNodeWriter[0].iBlock
         3825  +      );
         3826  +    }
         3827  +    iRoot = 1;
         3828  +  }
         3829  +  pRoot = &pWriter->aNodeWriter[iRoot];
         3830  +
         3831  +  /* Flush all currently outstanding nodes to disk. */
         3832  +  for(i=0; i<iRoot; i++){
         3833  +    NodeWriter *pNode = &pWriter->aNodeWriter[i];
         3834  +    if( pNode->block.n>0 && rc==SQLITE_OK ){
         3835  +      rc = fts3WriteSegment(p, pNode->iBlock, pNode->block.a, pNode->block.n);
         3836  +    }
         3837  +    sqlite3_free(pNode->block.a);
         3838  +    sqlite3_free(pNode->key.a);
         3839  +  }
         3840  +
         3841  +  /* Write the %_segdir record. */
         3842  +  if( rc==SQLITE_OK ){
         3843  +    rc = fts3WriteSegdir(p, 
         3844  +        pWriter->iAbsLevel+1,               /* level */
         3845  +        pWriter->iIdx,                      /* idx */
         3846  +        pWriter->iStart,                    /* start_block */
         3847  +        pWriter->aNodeWriter[0].iBlock,     /* leaves_end_block */
         3848  +        pWriter->iEnd,                      /* end_block */
         3849  +        pRoot->block.a, pRoot->block.n      /* root */
         3850  +    );
         3851  +  }
         3852  +  sqlite3_free(pRoot->block.a);
         3853  +  sqlite3_free(pRoot->key.a);
         3854  +
         3855  +  *pRc = rc;
         3856  +}
         3857  +
         3858  +/*
         3859  +** Compare the term in buffer zLhs (size in bytes nLhs) with that in
         3860  +** zRhs (size in bytes nRhs) using memcmp. If one term is a prefix of
         3861  +** the other, it is considered to be smaller than the other.
         3862  +**
         3863  +** Return -ve if zLhs is smaller than zRhs, 0 if it is equal, or +ve
         3864  +** if it is greater.
         3865  +*/
         3866  +static int fts3TermCmp(
         3867  +  const char *zLhs, int nLhs,     /* LHS of comparison */
         3868  +  const char *zRhs, int nRhs      /* RHS of comparison */
         3869  +){
         3870  +  int nCmp = MIN(nLhs, nRhs);
         3871  +  int res;
         3872  +
         3873  +  res = memcmp(zLhs, zRhs, nCmp);
         3874  +  if( res==0 ) res = nLhs - nRhs;
         3875  +
         3876  +  return res;
         3877  +}
         3878  +
         3879  +
         3880  +/*
         3881  +** Query to see if the entry in the %_segments table with blockid iEnd is 
         3882  +** NULL. If no error occurs and the entry is NULL, set *pbRes 1 before
         3883  +** returning. Otherwise, set *pbRes to 0. 
         3884  +**
         3885  +** Or, if an error occurs while querying the database, return an SQLite 
         3886  +** error code. The final value of *pbRes is undefined in this case.
         3887  +**
         3888  +** This is used to test if a segment is an "appendable" segment. If it
         3889  +** is, then a NULL entry has been inserted into the %_segments table
         3890  +** with blockid %_segdir.end_block.
         3891  +*/
         3892  +static int fts3IsAppendable(Fts3Table *p, sqlite3_int64 iEnd, int *pbRes){
         3893  +  int bRes = 0;                   /* Result to set *pbRes to */
         3894  +  sqlite3_stmt *pCheck = 0;       /* Statement to query database with */
         3895  +  int rc;                         /* Return code */
         3896  +
         3897  +  rc = fts3SqlStmt(p, SQL_SEGMENT_IS_APPENDABLE, &pCheck, 0);
         3898  +  if( rc==SQLITE_OK ){
         3899  +    sqlite3_bind_int64(pCheck, 1, iEnd);
         3900  +    if( SQLITE_ROW==sqlite3_step(pCheck) ) bRes = 1;
         3901  +    rc = sqlite3_reset(pCheck);
         3902  +  }
         3903  +  
         3904  +  *pbRes = bRes;
         3905  +  return rc;
         3906  +}
         3907  +
         3908  +/*
         3909  +** This function is called when initializing an incremental-merge operation.
         3910  +** It checks if the existing segment with index value iIdx at absolute level 
         3911  +** (iAbsLevel+1) can be appended to by the incremental merge. If it can, the
         3912  +** merge-writer object *pWriter is initialized to write to it.
         3913  +**
         3914  +** An existing segment can be appended to by an incremental merge if:
         3915  +**
         3916  +**   * It was initially created as an appendable segment (with all required
         3917  +**     space pre-allocated), and
         3918  +**
         3919  +**   * The first key read from the input (arguments zKey and nKey) is 
         3920  +**     greater than the largest key currently stored in the potential
         3921  +**     output segment.
         3922  +*/
         3923  +static int fts3IncrmergeLoad(
         3924  +  Fts3Table *p,                   /* Fts3 table handle */
         3925  +  sqlite3_int64 iAbsLevel,        /* Absolute level of input segments */
         3926  +  int iIdx,                       /* Index of candidate output segment */
         3927  +  const char *zKey,               /* First key to write */
         3928  +  int nKey,                       /* Number of bytes in nKey */
         3929  +  IncrmergeWriter *pWriter        /* Populate this object */
         3930  +){
         3931  +  int rc;                         /* Return code */
         3932  +  sqlite3_stmt *pSelect = 0;      /* SELECT to read %_segdir entry */
         3933  +
         3934  +  rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR, &pSelect, 0);
         3935  +  if( rc==SQLITE_OK ){
         3936  +    sqlite3_int64 iStart = 0;     /* Value of %_segdir.start_block */
         3937  +    sqlite3_int64 iLeafEnd = 0;   /* Value of %_segdir.leaves_end_block */
         3938  +    sqlite3_int64 iEnd = 0;       /* Value of %_segdir.end_block */
         3939  +    const char *aRoot = 0;        /* Pointer to %_segdir.root buffer */
         3940  +    int nRoot = 0;                /* Size of aRoot[] in bytes */
         3941  +    int rc2;                      /* Return code from sqlite3_reset() */
         3942  +    int bAppendable = 0;          /* Set to true if segment is appendable */
         3943  +
         3944  +    /* Read the %_segdir entry for index iIdx absolute level (iAbsLevel+1) */
         3945  +    sqlite3_bind_int64(pSelect, 1, iAbsLevel+1);
         3946  +    sqlite3_bind_int(pSelect, 2, iIdx);
         3947  +    if( sqlite3_step(pSelect)==SQLITE_ROW ){
         3948  +      iStart = sqlite3_column_int64(pSelect, 1);
         3949  +      iLeafEnd = sqlite3_column_int64(pSelect, 2);
         3950  +      iEnd = sqlite3_column_int64(pSelect, 3);
         3951  +      nRoot = sqlite3_column_bytes(pSelect, 4);
         3952  +      aRoot = sqlite3_column_blob(pSelect, 4);
         3953  +    }else{
         3954  +      return sqlite3_reset(pSelect);
         3955  +    }
         3956  +
         3957  +    /* Check for the zero-length marker in the %_segments table */
         3958  +    rc = fts3IsAppendable(p, iEnd, &bAppendable);
         3959  +
         3960  +    /* Check that zKey/nKey is larger than the largest key the candidate */
         3961  +    if( rc==SQLITE_OK && bAppendable ){
         3962  +      char *aLeaf = 0;
         3963  +      int nLeaf = 0;
         3964  +
         3965  +      rc = sqlite3Fts3ReadBlock(p, iLeafEnd, &aLeaf, &nLeaf, 0);
         3966  +      if( rc==SQLITE_OK ){
         3967  +        NodeReader reader;
         3968  +        for(rc = nodeReaderInit(&reader, aLeaf, nLeaf);
         3969  +            rc==SQLITE_OK && reader.aNode;
         3970  +            rc = nodeReaderNext(&reader)
         3971  +        ){
         3972  +          assert( reader.aNode );
         3973  +        }
         3974  +        if( fts3TermCmp(zKey, nKey, reader.term.a, reader.term.n)<=0 ){
         3975  +          bAppendable = 0;
         3976  +        }
         3977  +        nodeReaderRelease(&reader);
         3978  +      }
         3979  +      sqlite3_free(aLeaf);
         3980  +    }
         3981  +
         3982  +    if( rc==SQLITE_OK && bAppendable ){
         3983  +      /* It is possible to append to this segment. Set up the IncrmergeWriter
         3984  +      ** object to do so.  */
         3985  +      int i;
         3986  +      int nHeight = (int)aRoot[0];
         3987  +      NodeWriter *pNode;
         3988  +
         3989  +      pWriter->nLeafEst = (int)((iEnd - iStart) + 1)/FTS_MAX_APPENDABLE_HEIGHT;
         3990  +      pWriter->iStart = iStart;
         3991  +      pWriter->iEnd = iEnd;
         3992  +      pWriter->iAbsLevel = iAbsLevel;
         3993  +      pWriter->iIdx = iIdx;
         3994  +
         3995  +      for(i=nHeight+1; i<FTS_MAX_APPENDABLE_HEIGHT; i++){
         3996  +        pWriter->aNodeWriter[i].iBlock = pWriter->iStart + i*pWriter->nLeafEst;
         3997  +      }
         3998  +
         3999  +      pNode = &pWriter->aNodeWriter[nHeight];
         4000  +      pNode->iBlock = pWriter->iStart + pWriter->nLeafEst*nHeight;
         4001  +      blobGrowBuffer(&pNode->block, MAX(nRoot, p->nNodeSize), &rc);
         4002  +      if( rc==SQLITE_OK ){
         4003  +        memcpy(pNode->block.a, aRoot, nRoot);
         4004  +        pNode->block.n = nRoot;
         4005  +      }
         4006  +
         4007  +      for(i=nHeight; i>=0 && rc==SQLITE_OK; i--){
         4008  +        NodeReader reader;
         4009  +        pNode = &pWriter->aNodeWriter[i];
         4010  +
         4011  +        rc = nodeReaderInit(&reader, pNode->block.a, pNode->block.n);
         4012  +        while( reader.aNode && rc==SQLITE_OK ) rc = nodeReaderNext(&reader);
         4013  +        blobGrowBuffer(&pNode->key, reader.term.n, &rc);
         4014  +        if( rc==SQLITE_OK ){
         4015  +          memcpy(pNode->key.a, reader.term.a, reader.term.n);
         4016  +          pNode->key.n = reader.term.n;
         4017  +          if( i>0 ){
         4018  +            char *aBlock = 0;
         4019  +            int nBlock = 0;
         4020  +            pNode = &pWriter->aNodeWriter[i-1];
         4021  +            pNode->iBlock = reader.iChild;
         4022  +            rc = sqlite3Fts3ReadBlock(p, reader.iChild, &aBlock, &nBlock, 0);
         4023  +            blobGrowBuffer(&pNode->block, MAX(nBlock, p->nNodeSize), &rc);
         4024  +            if( rc==SQLITE_OK ){
         4025  +              memcpy(pNode->block.a, aBlock, nBlock);
         4026  +              pNode->block.n = nBlock;
         4027  +            }
         4028  +            sqlite3_free(aBlock);
         4029  +          }
         4030  +        }
         4031  +        nodeReaderRelease(&reader);
         4032  +      }
         4033  +    }
         4034  +
         4035  +    rc2 = sqlite3_reset(pSelect);
         4036  +    if( rc==SQLITE_OK ) rc = rc2;
         4037  +  }
         4038  +
         4039  +  return rc;
         4040  +}
         4041  +
         4042  +/*
         4043  +** Determine the largest segment index value that exists within absolute
         4044  +** level iAbsLevel+1. If no error occurs, set *piIdx to this value plus
         4045  +** one before returning SQLITE_OK. Or, if there are no segments at all 
         4046  +** within level iAbsLevel, set *piIdx to zero.
         4047  +**
         4048  +** If an error occurs, return an SQLite error code. The final value of
         4049  +** *piIdx is undefined in this case.
         4050  +*/
         4051  +static int fts3IncrmergeOutputIdx( 
         4052  +  Fts3Table *p,                   /* FTS Table handle */
         4053  +  sqlite3_int64 iAbsLevel,        /* Absolute index of input segments */
         4054  +  int *piIdx                      /* OUT: Next free index at iAbsLevel+1 */
         4055  +){
         4056  +  int rc;
         4057  +  sqlite3_stmt *pOutputIdx = 0;   /* SQL used to find output index */
         4058  +
         4059  +  rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pOutputIdx, 0);
         4060  +  if( rc==SQLITE_OK ){
         4061  +    sqlite3_bind_int64(pOutputIdx, 1, iAbsLevel+1);
         4062  +    sqlite3_step(pOutputIdx);
         4063  +    *piIdx = sqlite3_column_int(pOutputIdx, 0);
         4064  +    rc = sqlite3_reset(pOutputIdx);
         4065  +  }
         4066  +
         4067  +  return rc;
         4068  +}
         4069  +
         4070  +/* 
         4071  +** Allocate an appendable output segment on absolute level iAbsLevel+1
         4072  +** with idx value iIdx.
         4073  +**
         4074  +** In the %_segdir table, a segment is defined by the values in three
         4075  +** columns:
         4076  +**
         4077  +**     start_block
         4078  +**     leaves_end_block
         4079  +**     end_block
         4080  +**
         4081  +** When an appendable segment is allocated, it is estimated that the
         4082  +** maximum number of leaf blocks that may be required is the sum of the
         4083  +** number of leaf blocks consumed by the input segments, plus the number
         4084  +** of input segments, multiplied by two. This value is stored in stack 
         4085  +** variable nLeafEst.
         4086  +**
         4087  +** A total of 16*nLeafEst blocks are allocated when an appendable segment
         4088  +** is created ((1 + end_block - start_block)==16*nLeafEst). The contiguous
         4089  +** array of leaf nodes starts at the first block allocated. The array
         4090  +** of interior nodes that are parents of the leaf nodes start at block
         4091  +** (start_block + (1 + end_block - start_block) / 16). And so on.
         4092  +**
         4093  +** In the actual code below, the value "16" is replaced with the 
         4094  +** pre-processor macro FTS_MAX_APPENDABLE_HEIGHT.
         4095  +*/
         4096  +static int fts3IncrmergeWriter( 
         4097  +  Fts3Table *p,                   /* Fts3 table handle */
         4098  +  sqlite3_int64 iAbsLevel,        /* Absolute level of input segments */
         4099  +  int iIdx,                       /* Index of new output segment */
         4100  +  Fts3MultiSegReader *pCsr,       /* Cursor that data will be read from */
         4101  +  IncrmergeWriter *pWriter        /* Populate this object */
         4102  +){
         4103  +  int rc;                         /* Return Code */
         4104  +  int i;                          /* Iterator variable */
         4105  +  int nLeafEst = 0;               /* Blocks allocated for leaf nodes */
         4106  +  sqlite3_stmt *pLeafEst = 0;     /* SQL used to determine nLeafEst */
         4107  +  sqlite3_stmt *pFirstBlock = 0;  /* SQL used to determine first block */
         4108  +
         4109  +  /* Calculate nLeafEst. */
         4110  +  rc = fts3SqlStmt(p, SQL_MAX_LEAF_NODE_ESTIMATE, &pLeafEst, 0);
         4111  +  if( rc==SQLITE_OK ){
         4112  +    sqlite3_bind_int64(pLeafEst, 1, iAbsLevel);
         4113  +    sqlite3_bind_int64(pLeafEst, 2, pCsr->nSegment);
         4114  +    if( SQLITE_ROW==sqlite3_step(pLeafEst) ){
         4115  +      nLeafEst = sqlite3_column_int(pLeafEst, 0);
         4116  +    }
         4117  +    rc = sqlite3_reset(pLeafEst);
         4118  +  }
         4119  +  if( rc!=SQLITE_OK ) return rc;
         4120  +
         4121  +  /* Calculate the first block to use in the output segment */
         4122  +  rc = fts3SqlStmt(p, SQL_NEXT_SEGMENTS_ID, &pFirstBlock, 0);
         4123  +  if( rc==SQLITE_OK ){
         4124  +    if( SQLITE_ROW==sqlite3_step(pFirstBlock) ){
         4125  +      pWriter->iStart = sqlite3_column_int64(pFirstBlock, 0);
         4126  +      pWriter->iEnd = pWriter->iStart - 1;
         4127  +      pWriter->iEnd += nLeafEst * FTS_MAX_APPENDABLE_HEIGHT;
         4128  +    }
         4129  +    rc = sqlite3_reset(pFirstBlock);
         4130  +  }
         4131  +  if( rc!=SQLITE_OK ) return rc;
         4132  +
         4133  +  /* Insert the marker in the %_segments table to make sure nobody tries
         4134  +  ** to steal the space just allocated. This is also used to identify 
         4135  +  ** appendable segments.  */
         4136  +  rc = fts3WriteSegment(p, pWriter->iEnd, 0, 0);
         4137  +  if( rc!=SQLITE_OK ) return rc;
         4138  +
         4139  +  pWriter->iAbsLevel = iAbsLevel;
         4140  +  pWriter->nLeafEst = nLeafEst;
         4141  +  pWriter->iIdx = iIdx;
         4142  +
         4143  +  /* Set up the array of NodeWriter objects */
         4144  +  for(i=0; i<FTS_MAX_APPENDABLE_HEIGHT; i++){
         4145  +    pWriter->aNodeWriter[i].iBlock = pWriter->iStart + i*pWriter->nLeafEst;
         4146  +  }
         4147  +  return SQLITE_OK;
         4148  +}
         4149  +
         4150  +/*
         4151  +** Remove an entry from the %_segdir table. This involves running the 
         4152  +** following two statements:
         4153  +**
         4154  +**   DELETE FROM %_segdir WHERE level = :iAbsLevel AND idx = :iIdx
         4155  +**   UPDATE %_segdir SET idx = idx - 1 WHERE level = :iAbsLevel AND idx > :iIdx
         4156  +**
         4157  +** The DELETE statement removes the specific %_segdir level. The UPDATE 
         4158  +** statement ensures that the remaining segments have contiguously allocated
         4159  +** idx values.
         4160  +*/
         4161  +static int fts3RemoveSegdirEntry(
         4162  +  Fts3Table *p,                   /* FTS3 table handle */
         4163  +  sqlite3_int64 iAbsLevel,        /* Absolute level to delete from */
         4164  +  int iIdx                        /* Index of %_segdir entry to delete */
         4165  +){
         4166  +  int rc;                         /* Return code */
         4167  +  sqlite3_stmt *pDelete = 0;      /* DELETE statement */
         4168  +
         4169  +  rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_ENTRY, &pDelete, 0);
         4170  +  if( rc==SQLITE_OK ){
         4171  +    sqlite3_bind_int64(pDelete, 1, iAbsLevel);
         4172  +    sqlite3_bind_int(pDelete, 2, iIdx);
         4173  +    sqlite3_step(pDelete);
         4174  +    rc = sqlite3_reset(pDelete);
         4175  +  }
         4176  +
         4177  +  return rc;
         4178  +}
         4179  +
         4180  +/*
         4181  +** One or more segments have just been removed from absolute level iAbsLevel.
         4182  +** Update the 'idx' values of the remaining segments in the level so that
         4183  +** the idx values are a contiguous sequence starting from 0.
         4184  +*/
         4185  +static int fts3RepackSegdirLevel(
         4186  +  Fts3Table *p,                   /* FTS3 table handle */
         4187  +  sqlite3_int64 iAbsLevel         /* Absolute level to repack */
         4188  +){
         4189  +  int rc;                         /* Return code */
         4190  +  int *aIdx = 0;                  /* Array of remaining idx values */
         4191  +  int nIdx = 0;                   /* Valid entries in aIdx[] */
         4192  +  int nAlloc = 0;                 /* Allocated size of aIdx[] */
         4193  +  int i;                          /* Iterator variable */
         4194  +  sqlite3_stmt *pSelect = 0;      /* Select statement to read idx values */
         4195  +  sqlite3_stmt *pUpdate = 0;      /* Update statement to modify idx values */
         4196  +
         4197  +  rc = fts3SqlStmt(p, SQL_SELECT_INDEXES, &pSelect, 0);
         4198  +  if( rc==SQLITE_OK ){
         4199  +    int rc2;
         4200  +    sqlite3_bind_int64(pSelect, 1, iAbsLevel);
         4201  +    while( SQLITE_ROW==sqlite3_step(pSelect) ){
         4202  +      if( nIdx>=nAlloc ){
         4203  +        int *aNew;
         4204  +        nAlloc += 16;
         4205  +        aNew = sqlite3_realloc(aIdx, nAlloc*sizeof(int));
         4206  +        if( !aNew ){
         4207  +          rc = SQLITE_NOMEM;
         4208  +          break;
         4209  +        }
         4210  +        aIdx = aNew;
         4211  +      }
         4212  +      aIdx[nIdx++] = sqlite3_column_int(pSelect, 0);
         4213  +    }
         4214  +    rc2 = sqlite3_reset(pSelect);
         4215  +    if( rc==SQLITE_OK ) rc = rc2;
         4216  +  }
         4217  +
         4218  +  if( rc==SQLITE_OK ){
         4219  +    rc = fts3SqlStmt(p, SQL_SHIFT_SEGDIR_ENTRY, &pUpdate, 0);
         4220  +  }
         4221  +  if( rc==SQLITE_OK ){
         4222  +    sqlite3_bind_int64(pUpdate, 2, iAbsLevel);
         4223  +  }
         4224  +
         4225  +  assert( p->bIgnoreSavepoint==0 );
         4226  +  p->bIgnoreSavepoint = 1;
         4227  +  for(i=0; rc==SQLITE_OK && i<nIdx; i++){
         4228  +    if( aIdx[i]!=i ){
         4229  +      sqlite3_bind_int(pUpdate, 3, aIdx[i]);
         4230  +      sqlite3_bind_int(pUpdate, 1, i);
         4231  +      sqlite3_step(pUpdate);
         4232  +      rc = sqlite3_reset(pUpdate);
         4233  +    }
         4234  +  }
         4235  +  p->bIgnoreSavepoint = 0;
         4236  +
         4237  +  sqlite3_free(aIdx);
         4238  +  return rc;
         4239  +}
         4240  +
         4241  +static void fts3StartNode(Blob *pNode, int iHeight, sqlite3_int64 iChild){
         4242  +  pNode->a[0] = (char)iHeight;
         4243  +  if( iChild ){
         4244  +    assert( pNode->nAlloc>=1+sqlite3Fts3VarintLen(iChild) );
         4245  +    pNode->n = 1 + sqlite3Fts3PutVarint(&pNode->a[1], iChild);
         4246  +  }else{
         4247  +    assert( pNode->nAlloc>=1 );
         4248  +    pNode->n = 1;
         4249  +  }
         4250  +}
         4251  +
         4252  +/*
         4253  +** The first two arguments are a pointer to and the size of a segment b-tree
         4254  +** node. The node may be a leaf or an internal node.
         4255  +**
         4256  +** This function creates a new node image in blob object *pNew by copying
         4257  +** all terms that are greater than or equal to zTerm/nTerm (for leaf nodes)
         4258  +** or greater than zTerm/nTerm (for internal nodes) from aNode/nNode.
         4259  +*/
         4260  +static int fts3TruncateNode(
         4261  +  const char *aNode,              /* Current node image */
         4262  +  int nNode,                      /* Size of aNode in bytes */
         4263  +  Blob *pNew,                     /* OUT: Write new node image here */
         4264  +  const char *zTerm,              /* Omit all terms smaller than this */
         4265  +  int nTerm,                      /* Size of zTerm in bytes */
         4266  +  sqlite3_int64 *piBlock          /* OUT: Block number in next layer down */
         4267  +){
         4268  +  NodeReader reader;              /* Reader object */
         4269  +  Blob prev = {0, 0, 0};          /* Previous term written to new node */
         4270  +  int rc = SQLITE_OK;             /* Return code */
         4271  +  int bLeaf = aNode[0]=='\0';     /* True for a leaf node */
         4272  +
         4273  +  /* Allocate required output space */
         4274  +  blobGrowBuffer(pNew, nNode, &rc);
         4275  +  if( rc!=SQLITE_OK ) return rc;
         4276  +  pNew->n = 0;
         4277  +
         4278  +  /* Populate new node buffer */
         4279  +  for(rc = nodeReaderInit(&reader, aNode, nNode); 
         4280  +      rc==SQLITE_OK && reader.aNode; 
         4281  +      rc = nodeReaderNext(&reader)
         4282  +  ){
         4283  +    if( pNew->n==0 ){
         4284  +      int res = fts3TermCmp(reader.term.a, reader.term.n, zTerm, nTerm);
         4285  +      if( res<0 || (bLeaf==0 && res==0) ) continue;
         4286  +      fts3StartNode(pNew, (int)aNode[0], reader.iChild);
         4287  +      *piBlock = reader.iChild;
         4288  +    }
         4289  +    rc = fts3AppendToNode(
         4290  +        pNew, &prev, reader.term.a, reader.term.n,
         4291  +        reader.aDoclist, reader.nDoclist
         4292  +    );
         4293  +    if( rc!=SQLITE_OK ) break;
         4294  +  }
         4295  +  if( pNew->n==0 ){
         4296  +    fts3StartNode(pNew, (int)aNode[0], reader.iChild);
         4297  +    *piBlock = reader.iChild;
         4298  +  }
         4299  +  assert( pNew->n<=pNew->nAlloc );
         4300  +
         4301  +  nodeReaderRelease(&reader);
         4302  +  sqlite3_free(prev.a);
         4303  +  return rc;
         4304  +}
         4305  +
         4306  +/*
         4307  +** Remove all terms smaller than zTerm/nTerm from segment iIdx in absolute 
         4308  +** level iAbsLevel. This may involve deleting entries from the %_segments
         4309  +** table, and modifying existing entries in both the %_segments and %_segdir
         4310  +** tables.
         4311  +**
         4312  +** SQLITE_OK is returned if the segment is updated successfully. Or an
         4313  +** SQLite error code otherwise.
         4314  +*/
         4315  +static int fts3TruncateSegment(
         4316  +  Fts3Table *p,                   /* FTS3 table handle */
         4317  +  sqlite3_int64 iAbsLevel,        /* Absolute level of segment to modify */
         4318  +  int iIdx,                       /* Index within level of segment to modify */
         4319  +  const char *zTerm,              /* Remove terms smaller than this */
         4320  +  int nTerm                      /* Number of bytes in buffer zTerm */
         4321  +){
         4322  +  int rc = SQLITE_OK;             /* Return code */
         4323  +  Blob root = {0,0,0};            /* New root page image */
         4324  +  Blob block = {0,0,0};           /* Buffer used for any other block */
         4325  +  sqlite3_int64 iBlock = 0;       /* Block id */
         4326  +  sqlite3_int64 iNewStart = 0;    /* New value for iStartBlock */
         4327  +  sqlite3_int64 iOldStart = 0;    /* Old value for iStartBlock */
         4328  +  sqlite3_stmt *pFetch = 0;       /* Statement used to fetch segdir */
         4329  +
         4330  +  rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR, &pFetch, 0);
         4331  +  if( rc==SQLITE_OK ){
         4332  +    int rc2;                      /* sqlite3_reset() return code */
         4333  +    sqlite3_bind_int64(pFetch, 1, iAbsLevel);
         4334  +    sqlite3_bind_int(pFetch, 2, iIdx);
         4335  +    if( SQLITE_ROW==sqlite3_step(pFetch) ){
         4336  +      const char *aRoot = sqlite3_column_blob(pFetch, 4);
         4337  +      int nRoot = sqlite3_column_bytes(pFetch, 4);
         4338  +      iOldStart = sqlite3_column_int64(pFetch, 1);
         4339  +      rc = fts3TruncateNode(aRoot, nRoot, &root, zTerm, nTerm, &iBlock);
         4340  +    }
         4341  +    rc2 = sqlite3_reset(pFetch);
         4342  +    if( rc==SQLITE_OK ) rc = rc2;
         4343  +  }
         4344  +
         4345  +  while( rc==SQLITE_OK && iBlock ){
         4346  +    char *aBlock = 0;
         4347  +    int nBlock = 0;
         4348  +    iNewStart = iBlock;
         4349  +
         4350  +    rc = sqlite3Fts3ReadBlock(p, iBlock, &aBlock, &nBlock, 0);
         4351  +    if( rc==SQLITE_OK ){
         4352  +      rc = fts3TruncateNode(aBlock, nBlock, &block, zTerm, nTerm, &iBlock);
         4353  +    }
         4354  +    if( rc==SQLITE_OK ){
         4355  +      rc = fts3WriteSegment(p, iNewStart, block.a, block.n);
         4356  +    }
         4357  +    sqlite3_free(aBlock);
         4358  +  }
         4359  +
         4360  +  /* Variable iNewStart now contains the first valid leaf node. */
         4361  +  if( rc==SQLITE_OK && iNewStart ){
         4362  +    sqlite3_stmt *pDel = 0;
         4363  +    rc = fts3SqlStmt(p, SQL_DELETE_SEGMENTS_RANGE, &pDel, 0);
         4364  +    if( rc==SQLITE_OK ){
         4365  +      sqlite3_bind_int64(pDel, 1, iOldStart);
         4366  +      sqlite3_bind_int64(pDel, 2, iNewStart-1);
         4367  +      sqlite3_step(pDel);
         4368  +      rc = sqlite3_reset(pDel);
         4369  +    }
         4370  +  }
         4371  +
         4372  +  if( rc==SQLITE_OK ){
         4373  +    sqlite3_stmt *pChomp = 0;
         4374  +    rc = fts3SqlStmt(p, SQL_CHOMP_SEGDIR, &pChomp, 0);
         4375  +    if( rc==SQLITE_OK ){
         4376  +      sqlite3_bind_int64(pChomp, 1, iNewStart);
         4377  +      sqlite3_bind_blob(pChomp, 2, root.a, root.n, SQLITE_STATIC);
         4378  +      sqlite3_bind_int64(pChomp, 3, iAbsLevel);
         4379  +      sqlite3_bind_int(pChomp, 4, iIdx);
         4380  +      sqlite3_step(pChomp);
         4381  +      rc = sqlite3_reset(pChomp);
         4382  +    }
         4383  +  }
         4384  +
         4385  +  sqlite3_free(root.a);
         4386  +  sqlite3_free(block.a);
         4387  +  return rc;
         4388  +}
         4389  +
         4390  +/*
         4391  +** This function is called after an incrmental-merge operation has run to
         4392  +** merge (or partially merge) two or more segments from absolute level
         4393  +** iAbsLevel.
         4394  +**
         4395  +** Each input segment is either removed from the db completely (if all of
         4396  +** its data was copied to the output segment by the incrmerge operation)
         4397  +** or modified in place so that it no longer contains those entries that
         4398  +** have been duplicated in the output segment.
         4399  +*/
         4400  +static int fts3IncrmergeChomp(
         4401  +  Fts3Table *p,                   /* FTS table handle */
         4402  +  sqlite3_int64 iAbsLevel,        /* Absolute level containing segments */
         4403  +  Fts3MultiSegReader *pCsr,       /* Chomp all segments opened by this cursor */
         4404  +  int *pnRem                      /* Number of segments not deleted */
         4405  +){
         4406  +  int i;
         4407  +  int nRem = 0;
         4408  +  int rc = SQLITE_OK;
         4409  +
         4410  +  for(i=pCsr->nSegment-1; i>=0 && rc==SQLITE_OK; i--){
         4411  +    Fts3SegReader *pSeg = 0;
         4412  +    int j;
         4413  +
         4414  +    /* Find the Fts3SegReader object with Fts3SegReader.iIdx==i. It is hiding
         4415  +    ** somewhere in the pCsr->apSegment[] array.  */
         4416  +    for(j=0; ALWAYS(j<pCsr->nSegment); j++){
         4417  +      pSeg = pCsr->apSegment[j];
         4418  +      if( pSeg->iIdx==i ) break;
         4419  +    }
         4420  +    assert( j<pCsr->nSegment && pSeg->iIdx==i );
         4421  +
         4422  +    if( pSeg->aNode==0 ){
         4423  +      /* Seg-reader is at EOF. Remove the entire input segment. */
         4424  +      rc = fts3DeleteSegment(p, pSeg);
         4425  +      if( rc==SQLITE_OK ){
         4426  +        rc = fts3RemoveSegdirEntry(p, iAbsLevel, pSeg->iIdx);
         4427  +      }
         4428  +      *pnRem = 0;
         4429  +    }else{
         4430  +      /* The incremental merge did not copy all the data from this 
         4431  +      ** segment to the upper level. The segment is modified in place
         4432  +      ** so that it contains no keys smaller than zTerm/nTerm. */ 
         4433  +      const char *zTerm = pSeg->zTerm;
         4434  +      int nTerm = pSeg->nTerm;
         4435  +      rc = fts3TruncateSegment(p, iAbsLevel, pSeg->iIdx, zTerm, nTerm);
         4436  +      nRem++;
         4437  +    }
         4438  +  }
         4439  +
         4440  +  if( rc==SQLITE_OK && nRem!=pCsr->nSegment ){
         4441  +    rc = fts3RepackSegdirLevel(p, iAbsLevel);
         4442  +  }
         4443  +
         4444  +  *pnRem = nRem;
         4445  +  return rc;
         4446  +}
         4447  +
         4448  +/*
         4449  +** Store an incr-merge hint in the database.
         4450  +*/
         4451  +static int fts3IncrmergeHintStore(Fts3Table *p, Blob *pHint){
         4452  +  sqlite3_stmt *pReplace = 0;
         4453  +  int rc;                         /* Return code */
         4454  +
         4455  +  rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pReplace, 0);
         4456  +  if( rc==SQLITE_OK ){
         4457  +    sqlite3_bind_int(pReplace, 1, FTS_STAT_INCRMERGEHINT);
         4458  +    sqlite3_bind_blob(pReplace, 2, pHint->a, pHint->n, SQLITE_STATIC);
         4459  +    sqlite3_step(pReplace);
         4460  +    rc = sqlite3_reset(pReplace);
         4461  +  }
         4462  +
         4463  +  return rc;
         4464  +}
         4465  +
         4466  +/*
         4467  +** Load an incr-merge hint from the database. The incr-merge hint, if one 
         4468  +** exists, is stored in the rowid==1 row of the %_stat table.
         4469  +**
         4470  +** If successful, populate blob *pHint with the value read from the %_stat
         4471  +** table and return SQLITE_OK. Otherwise, if an error occurs, return an
         4472  +** SQLite error code.
         4473  +*/
         4474  +static int fts3IncrmergeHintLoad(Fts3Table *p, Blob *pHint){
         4475  +  sqlite3_stmt *pSelect = 0;
         4476  +  int rc;
         4477  +
         4478  +  pHint->n = 0;
         4479  +  rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pSelect, 0);
         4480  +  if( rc==SQLITE_OK ){
         4481  +    int rc2;
         4482  +    sqlite3_bind_int(pSelect, 1, FTS_STAT_INCRMERGEHINT);
         4483  +    if( SQLITE_ROW==sqlite3_step(pSelect) ){
         4484  +      const char *aHint = sqlite3_column_blob(pSelect, 0);
         4485  +      int nHint = sqlite3_column_bytes(pSelect, 0);
         4486  +      if( aHint ){
         4487  +        blobGrowBuffer(pHint, nHint, &rc);
         4488  +        if( rc==SQLITE_OK ){
         4489  +          memcpy(pHint->a, aHint, nHint);
         4490  +          pHint->n = nHint;
         4491  +        }
         4492  +      }
         4493  +    }
         4494  +    rc2 = sqlite3_reset(pSelect);
         4495  +    if( rc==SQLITE_OK ) rc = rc2;
         4496  +  }
         4497  +
         4498  +  return rc;
         4499  +}
         4500  +
         4501  +/*
         4502  +** If *pRc is not SQLITE_OK when this function is called, it is a no-op.
         4503  +** Otherwise, append an entry to the hint stored in blob *pHint. Each entry
         4504  +** consists of two varints, the absolute level number of the input segments 
         4505  +** and the number of input segments.
         4506  +**
         4507  +** If successful, leave *pRc set to SQLITE_OK and return. If an error occurs,
         4508  +** set *pRc to an SQLite error code before returning.
         4509  +*/
         4510  +static void fts3IncrmergeHintPush(
         4511  +  Blob *pHint,                    /* Hint blob to append to */
         4512  +  i64 iAbsLevel,                  /* First varint to store in hint */
         4513  +  int nInput,                     /* Second varint to store in hint */
         4514  +  int *pRc                        /* IN/OUT: Error code */
         4515  +){
         4516  +  blobGrowBuffer(pHint, pHint->n + 2*FTS3_VARINT_MAX, pRc);
         4517  +  if( *pRc==SQLITE_OK ){
         4518  +    pHint->n += sqlite3Fts3PutVarint(&pHint->a[pHint->n], iAbsLevel);
         4519  +    pHint->n += sqlite3Fts3PutVarint(&pHint->a[pHint->n], (i64)nInput);
         4520  +  }
         4521  +}
         4522  +
         4523  +/*
         4524  +** Read the last entry (most recently pushed) from the hint blob *pHint
         4525  +** and then remove the entry. Write the two values read to *piAbsLevel and 
         4526  +** *pnInput before returning.
         4527  +**
         4528  +** If no error occurs, return SQLITE_OK. If the hint blob in *pHint does
         4529  +** not contain at least two valid varints, return SQLITE_CORRUPT_VTAB.
         4530  +*/
         4531  +static int fts3IncrmergeHintPop(Blob *pHint, i64 *piAbsLevel, int *pnInput){
         4532  +  const int nHint = pHint->n;
         4533  +  int i;
         4534  +
         4535  +  i = pHint->n-2;
         4536  +  while( i>0 && (pHint->a[i-1] & 0x80) ) i--;
         4537  +  while( i>0 && (pHint->a[i-1] & 0x80) ) i--;
         4538  +
         4539  +  pHint->n = i;
         4540  +  i += sqlite3Fts3GetVarint(&pHint->a[i], piAbsLevel);
         4541  +  i += sqlite3Fts3GetVarint32(&pHint->a[i], pnInput);
         4542  +  if( i!=nHint ) return SQLITE_CORRUPT_VTAB;
         4543  +
         4544  +  return SQLITE_OK;
         4545  +}
         4546  +
         4547  +
         4548  +/*
         4549  +** Attempt an incremental merge that writes nMerge leaf blocks.
         4550  +**
         4551  +** Incremental merges happen nMin segments at a time. The two
         4552  +** segments to be merged are the nMin oldest segments (the ones with
         4553  +** the smallest indexes) in the highest level that contains at least
         4554  +** nMin segments. Multiple merges might occur in an attempt to write the 
         4555  +** quota of nMerge leaf blocks.
         4556  +*/
         4557  +int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){
         4558  +  int rc;                         /* Return code */
         4559  +  int nRem = nMerge;              /* Number of leaf pages yet to  be written */
         4560  +  Fts3MultiSegReader *pCsr;       /* Cursor used to read input data */
         4561  +  Fts3SegFilter *pFilter;         /* Filter used with cursor pCsr */
         4562  +  IncrmergeWriter *pWriter;       /* Writer object */
         4563  +  int nSeg = 0;                   /* Number of input segments */
         4564  +  sqlite3_int64 iAbsLevel = 0;    /* Absolute level number to work on */
         4565  +  Blob hint = {0, 0, 0};          /* Hint read from %_stat table */
         4566  +  int bDirtyHint = 0;             /* True if blob 'hint' has been modified */
         4567  +
         4568  +  /* Allocate space for the cursor, filter and writer objects */
         4569  +  const int nAlloc = sizeof(*pCsr) + sizeof(*pFilter) + sizeof(*pWriter);
         4570  +  pWriter = (IncrmergeWriter *)sqlite3_malloc(nAlloc);
         4571  +  if( !pWriter ) return SQLITE_NOMEM;
         4572  +  pFilter = (Fts3SegFilter *)&pWriter[1];
         4573  +  pCsr = (Fts3MultiSegReader *)&pFilter[1];
         4574  +
         4575  +  rc = fts3IncrmergeHintLoad(p, &hint);
         4576  +  while( rc==SQLITE_OK && nRem>0 ){
         4577  +    const i64 nMod = FTS3_SEGDIR_MAXLEVEL * p->nIndex;
         4578  +    sqlite3_stmt *pFindLevel = 0; /* SQL used to determine iAbsLevel */
         4579  +    int bUseHint = 0;             /* True if attempting to append */
         4580  +
         4581  +    /* Search the %_segdir table for the absolute level with the smallest
         4582  +    ** relative level number that contains at least nMin segments, if any.
         4583  +    ** If one is found, set iAbsLevel to the absolute level number and
         4584  +    ** nSeg to nMin. If no level with at least nMin segments can be found, 
         4585  +    ** set nSeg to -1.
         4586  +    */
         4587  +    rc = fts3SqlStmt(p, SQL_FIND_MERGE_LEVEL, &pFindLevel, 0);
         4588  +    sqlite3_bind_int(pFindLevel, 1, nMin);
         4589  +    if( sqlite3_step(pFindLevel)==SQLITE_ROW ){
         4590  +      iAbsLevel = sqlite3_column_int64(pFindLevel, 0);
         4591  +      nSeg = nMin;
         4592  +    }else{
         4593  +      nSeg = -1;
         4594  +    }
         4595  +    rc = sqlite3_reset(pFindLevel);
         4596  +
         4597  +    /* If the hint read from the %_stat table is not empty, check if the
         4598  +    ** last entry in it specifies a relative level smaller than or equal
         4599  +    ** to the level identified by the block above (if any). If so, this 
         4600  +    ** iteration of the loop will work on merging at the hinted level.
         4601  +    */
         4602  +    if( rc==SQLITE_OK && hint.n ){
         4603  +      int nHint = hint.n;
         4604  +      sqlite3_int64 iHintAbsLevel = 0;      /* Hint level */
         4605  +      int nHintSeg = 0;                     /* Hint number of segments */
         4606  +
         4607  +      rc = fts3IncrmergeHintPop(&hint, &iHintAbsLevel, &nHintSeg);
         4608  +      if( nSeg<0 || (iAbsLevel % nMod) >= (iHintAbsLevel % nMod) ){
         4609  +        iAbsLevel = iHintAbsLevel;
         4610  +        nSeg = nHintSeg;
         4611  +        bUseHint = 1;
         4612  +        bDirtyHint = 1;
         4613  +      }else{
         4614  +        /* This undoes the effect of the HintPop() above - so that no entry
         4615  +        ** is removed from the hint blob.  */
         4616  +        hint.n = nHint;
         4617  +      }
         4618  +    }
         4619  +
         4620  +    /* If nSeg is less that zero, then there is no level with at least
         4621  +    ** nMin segments and no hint in the %_stat table. No work to do.
         4622  +    ** Exit early in this case.  */
         4623  +    if( nSeg<0 ) break;
         4624  +
         4625  +    /* Open a cursor to iterate through the contents of the oldest nSeg 
         4626  +    ** indexes of absolute level iAbsLevel. If this cursor is opened using 
         4627  +    ** the 'hint' parameters, it is possible that there are less than nSeg
         4628  +    ** segments available in level iAbsLevel. In this case, no work is
         4629  +    ** done on iAbsLevel - fall through to the next iteration of the loop 
         4630  +    ** to start work on some other level.  */
         4631  +    memset(pWriter, 0, nAlloc);
         4632  +    pFilter->flags = FTS3_SEGMENT_REQUIRE_POS;
         4633  +    if( rc==SQLITE_OK ){
         4634  +      rc = fts3IncrmergeCsr(p, iAbsLevel, nSeg, pCsr);
         4635  +    }
         4636  +    if( SQLITE_OK==rc && pCsr->nSegment==nSeg
         4637  +     && SQLITE_OK==(rc = sqlite3Fts3SegReaderStart(p, pCsr, pFilter))
         4638  +     && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pCsr))
         4639  +    ){
         4640  +      int iIdx = 0;               /* Largest idx in level (iAbsLevel+1) */
         4641  +      rc = fts3IncrmergeOutputIdx(p, iAbsLevel, &iIdx);
         4642  +      if( rc==SQLITE_OK ){
         4643  +        if( bUseHint && iIdx>0 ){
         4644  +          const char *zKey = pCsr->zTerm;
         4645  +          int nKey = pCsr->nTerm;
         4646  +          rc = fts3IncrmergeLoad(p, iAbsLevel, iIdx-1, zKey, nKey, pWriter);
         4647  +        }else{
         4648  +          rc = fts3IncrmergeWriter(p, iAbsLevel, iIdx, pCsr, pWriter);
         4649  +        }
         4650  +      }
         4651  +
         4652  +      if( rc==SQLITE_OK && pWriter->nLeafEst ){
         4653  +        fts3LogMerge(nSeg, iAbsLevel);
         4654  +        do {
         4655  +          rc = fts3IncrmergeAppend(p, pWriter, pCsr);
         4656  +          if( rc==SQLITE_OK ) rc = sqlite3Fts3SegReaderStep(p, pCsr);
         4657  +          if( pWriter->nWork>=nRem && rc==SQLITE_ROW ) rc = SQLITE_OK;
         4658  +        }while( rc==SQLITE_ROW );
         4659  +
         4660  +        /* Update or delete the input segments */
         4661  +        if( rc==SQLITE_OK ){
         4662  +          nRem -= (1 + pWriter->nWork);
         4663  +          rc = fts3IncrmergeChomp(p, iAbsLevel, pCsr, &nSeg);
         4664  +          if( nSeg!=0 ){
         4665  +            bDirtyHint = 1;
         4666  +            fts3IncrmergeHintPush(&hint, iAbsLevel, nSeg, &rc);
         4667  +          }
         4668  +        }
         4669  +      }
         4670  +
         4671  +      fts3IncrmergeRelease(p, pWriter, &rc);
         4672  +    }
         4673  +
         4674  +    sqlite3Fts3SegReaderFinish(pCsr);
         4675  +  }
         4676  +
         4677  +  /* Write the hint values into the %_stat table for the next incr-merger */
         4678  +  if( bDirtyHint && rc==SQLITE_OK ){
         4679  +    rc = fts3IncrmergeHintStore(p, &hint);
         4680  +  }
         4681  +
         4682  +  sqlite3_free(pWriter);
         4683  +  sqlite3_free(hint.a);
         4684  +  return rc;
         4685  +}
         4686  +
         4687  +/*
         4688  +** Convert the text beginning at *pz into an integer and return
         4689  +** its value.  Advance *pz to point to the first character past
         4690  +** the integer.
         4691  +*/
         4692  +static int fts3Getint(const char **pz){
         4693  +  const char *z = *pz;
         4694  +  int i = 0;
         4695  +  while( (*z)>='0' && (*z)<='9' ) i = 10*i + *(z++) - '0';
         4696  +  *pz = z;
         4697  +  return i;
         4698  +}
         4699  +
         4700  +/*
         4701  +** Process statements of the form:
         4702  +**
         4703  +**    INSERT INTO table(table) VALUES('merge=A,B');
         4704  +**
         4705  +** A and B are integers that decode to be the number of leaf pages
         4706  +** written for the merge, and the minimum number of segments on a level
         4707  +** before it will be selected for a merge, respectively.
         4708  +*/
         4709  +static int fts3DoIncrmerge(
         4710  +  Fts3Table *p,                   /* FTS3 table handle */
         4711  +  const char *zParam              /* Nul-terminated string containing "A,B" */
         4712  +){
         4713  +  int rc;
         4714  +  int nMin = (FTS3_MERGE_COUNT / 2);
         4715  +  int nMerge = 0;
         4716  +  const char *z = zParam;
         4717  +
         4718  +  /* Read the first integer value */
         4719  +  nMerge = fts3Getint(&z);
         4720  +
         4721  +  /* If the first integer value is followed by a ',',  read the second
         4722  +  ** integer value. */
         4723  +  if( z[0]==',' && z[1]!='\0' ){
         4724  +    z++;
         4725  +    nMin = fts3Getint(&z);
         4726  +  }
         4727  +
         4728  +  if( z[0]!='\0' || nMin<2 ){
         4729  +    rc = SQLITE_ERROR;
         4730  +  }else{
         4731  +    rc = SQLITE_OK;
         4732  +    if( !p->bHasStat ){
         4733  +      assert( p->bFts4==0 );
         4734  +      sqlite3Fts3CreateStatTable(&rc, p);
         4735  +    }
         4736  +    if( rc==SQLITE_OK ){
         4737  +      rc = sqlite3Fts3Incrmerge(p, nMerge, nMin);
         4738  +    }
         4739  +    sqlite3Fts3SegmentsClose(p);
         4740  +  }
         4741  +  return rc;
         4742  +}
         4743  +
         4744  +/*
         4745  +** Process statements of the form:
         4746  +**
         4747  +**    INSERT INTO table(table) VALUES('automerge=X');
         4748  +**
         4749  +** where X is an integer.  X==0 means to turn automerge off.  X!=0 means
         4750  +** turn it on.  The setting is persistent.
         4751  +*/
         4752  +static int fts3DoAutoincrmerge(
         4753  +  Fts3Table *p,                   /* FTS3 table handle */
         4754  +  const char *zParam              /* Nul-terminated string containing boolean */
         4755  +){
         4756  +  int rc = SQLITE_OK;
         4757  +  sqlite3_stmt *pStmt = 0;
         4758  +  p->bAutoincrmerge = fts3Getint(&zParam)!=0;
         4759  +  if( !p->bHasStat ){
         4760  +    assert( p->bFts4==0 );
         4761  +    sqlite3Fts3CreateStatTable(&rc, p);
         4762  +    if( rc ) return rc;
         4763  +  }
         4764  +  rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pStmt, 0);
         4765  +  if( rc ) return rc;;
         4766  +  sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE);
         4767  +  sqlite3_bind_int(pStmt, 2, p->bAutoincrmerge);
         4768  +  sqlite3_step(pStmt);
         4769  +  rc = sqlite3_reset(pStmt);
         4770  +  return rc;
         4771  +}
         4772  +
         4773  +/*
         4774  +** Return a 64-bit checksum for the FTS index entry specified by the
         4775  +** arguments to this function.
         4776  +*/
         4777  +static u64 fts3ChecksumEntry(
         4778  +  const char *zTerm,              /* Pointer to buffer containing term */
         4779  +  int nTerm,                      /* Size of zTerm in bytes */
         4780  +  int iLangid,                    /* Language id for current row */
         4781  +  int iIndex,                     /* Index (0..Fts3Table.nIndex-1) */
         4782  +  i64 iDocid,                     /* Docid for current row. */
         4783  +  int iCol,                       /* Column number */
         4784  +  int iPos                        /* Position */
         4785  +){
         4786  +  int i;
         4787  +  u64 ret = (u64)iDocid;
         4788  +
         4789  +  ret += (ret<<3) + iLangid;
         4790  +  ret += (ret<<3) + iIndex;
         4791  +  ret += (ret<<3) + iCol;
         4792  +  ret += (ret<<3) + iPos;
         4793  +  for(i=0; i<nTerm; i++) ret += (ret<<3) + zTerm[i];
         4794  +
         4795  +  return ret;
         4796  +}
         4797  +
         4798  +/*
         4799  +** Return a checksum of all entries in the FTS index that correspond to
         4800  +** language id iLangid. The checksum is calculated by XORing the checksums
         4801  +** of each individual entry (see fts3ChecksumEntry()) together.
         4802  +**
         4803  +** If successful, the checksum value is returned and *pRc set to SQLITE_OK.
         4804  +** Otherwise, if an error occurs, *pRc is set to an SQLite error code. The
         4805  +** return value is undefined in this case.
         4806  +*/
         4807  +static u64 fts3ChecksumIndex(
         4808  +  Fts3Table *p,                   /* FTS3 table handle */
         4809  +  int iLangid,                    /* Language id to return cksum for */
         4810  +  int iIndex,                     /* Index to cksum (0..p->nIndex-1) */
         4811  +  int *pRc                        /* OUT: Return code */
         4812  +){
         4813  +  Fts3SegFilter filter;
         4814  +  Fts3MultiSegReader csr;
         4815  +  int rc;
         4816  +  u64 cksum = 0;
         4817  +
         4818  +  assert( *pRc==SQLITE_OK );
         4819  +
         4820  +  memset(&filter, 0, sizeof(filter));
         4821  +  memset(&csr, 0, sizeof(csr));
         4822  +  filter.flags =  FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY;
         4823  +  filter.flags |= FTS3_SEGMENT_SCAN;
         4824  +
         4825  +  rc = sqlite3Fts3SegReaderCursor(
         4826  +      p, iLangid, iIndex, FTS3_SEGCURSOR_ALL, 0, 0, 0, 1,&csr
         4827  +  );
         4828  +  if( rc==SQLITE_OK ){
         4829  +    rc = sqlite3Fts3SegReaderStart(p, &csr, &filter);
         4830  +  }
         4831  +
         4832  +  if( rc==SQLITE_OK ){
         4833  +    while( SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, &csr)) ){
         4834  +      char *pCsr = csr.aDoclist;
         4835  +      char *pEnd = &pCsr[csr.nDoclist];
         4836  +
         4837  +      i64 iDocid = 0;
         4838  +      i64 iCol = 0;
         4839  +      i64 iPos = 0;
         4840  +
         4841  +      pCsr += sqlite3Fts3GetVarint(pCsr, &iDocid);
         4842  +      while( pCsr<pEnd ){
         4843  +        i64 iVal = 0;
         4844  +        pCsr += sqlite3Fts3GetVarint(pCsr, &iVal);
         4845  +        if( pCsr<pEnd ){
         4846  +          if( iVal==0 || iVal==1 ){
         4847  +            iCol = 0;
         4848  +            iPos = 0;
         4849  +            if( iVal ){
         4850  +              pCsr += sqlite3Fts3GetVarint(pCsr, &iCol);
         4851  +            }else{
         4852  +              pCsr += sqlite3Fts3GetVarint(pCsr, &iVal);
         4853  +              iDocid += iVal;
         4854  +            }
         4855  +          }else{
         4856  +            iPos += (iVal - 2);
         4857  +            cksum = cksum ^ fts3ChecksumEntry(
         4858  +                csr.zTerm, csr.nTerm, iLangid, iIndex, iDocid,
         4859  +                (int)iCol, (int)iPos
         4860  +            );
         4861  +          }
         4862  +        }
         4863  +      }
         4864  +    }
         4865  +  }
         4866  +  sqlite3Fts3SegReaderFinish(&csr);
         4867  +
         4868  +  *pRc = rc;
         4869  +  return cksum;
         4870  +}
         4871  +
         4872  +/*
         4873  +** Check if the contents of the FTS index match the current contents of the
         4874  +** content table. If no error occurs and the contents do match, set *pbOk
         4875  +** to true and return SQLITE_OK. Or if the contents do not match, set *pbOk
         4876  +** to false before returning.
         4877  +**
         4878  +** If an error occurs (e.g. an OOM or IO error), return an SQLite error 
         4879  +** code. The final value of *pbOk is undefined in this case.
         4880  +*/
         4881  +static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){
         4882  +  int rc = SQLITE_OK;             /* Return code */
         4883  +  u64 cksum1 = 0;                 /* Checksum based on FTS index contents */
         4884  +  u64 cksum2 = 0;                 /* Checksum based on %_content contents */
         4885  +  sqlite3_stmt *pAllLangid = 0;   /* Statement to return all language-ids */
         4886  +
         4887  +  /* This block calculates the checksum according to the FTS index. */
         4888  +  rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0);
         4889  +  if( rc==SQLITE_OK ){
         4890  +    int rc2;
         4891  +    sqlite3_bind_int(pAllLangid, 1, p->nIndex);
         4892  +    while( rc==SQLITE_OK && sqlite3_step(pAllLangid)==SQLITE_ROW ){
         4893  +      int iLangid = sqlite3_column_int(pAllLangid, 0);
         4894  +      int i;
         4895  +      for(i=0; i<p->nIndex; i++){
         4896  +        cksum1 = cksum1 ^ fts3ChecksumIndex(p, iLangid, i, &rc);
         4897  +      }
         4898  +    }
         4899  +    rc2 = sqlite3_reset(pAllLangid);
         4900  +    if( rc==SQLITE_OK ) rc = rc2;
         4901  +  }
         4902  +
         4903  +  /* This block calculates the checksum according to the %_content table */
         4904  +  rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0);
         4905  +  if( rc==SQLITE_OK ){
         4906  +    sqlite3_tokenizer_module const *pModule = p->pTokenizer->pModule;
         4907  +    sqlite3_stmt *pStmt = 0;
         4908  +    char *zSql;
         4909  +   
         4910  +    zSql = sqlite3_mprintf("SELECT %s" , p->zReadExprlist);
         4911  +    if( !zSql ){
         4912  +      rc = SQLITE_NOMEM;
         4913  +    }else{
         4914  +      rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
         4915  +      sqlite3_free(zSql);
         4916  +    }
         4917  +
         4918  +    while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
         4919  +      i64 iDocid = sqlite3_column_int64(pStmt, 0);
         4920  +      int iLang = langidFromSelect(p, pStmt);
         4921  +      int iCol;
         4922  +
         4923  +      for(iCol=0; rc==SQLITE_OK && iCol<p->nColumn; iCol++){
         4924  +        const char *zText = (const char *)sqlite3_column_text(pStmt, iCol+1);
         4925  +        int nText = sqlite3_column_bytes(pStmt, iCol+1);
         4926  +        sqlite3_tokenizer_cursor *pT = 0;
         4927  +
         4928  +        rc = sqlite3Fts3OpenTokenizer(p->pTokenizer, iLang, zText, nText, &pT);
         4929  +        while( rc==SQLITE_OK ){
         4930  +          char const *zToken;       /* Buffer containing token */
         4931  +          int nToken;               /* Number of bytes in token */
         4932  +          int iDum1, iDum2;         /* Dummy variables */
         4933  +          int iPos;                 /* Position of token in zText */
         4934  +
         4935  +          rc = pModule->xNext(pT, &zToken, &nToken, &iDum1, &iDum2, &iPos);
         4936  +          if( rc==SQLITE_OK ){
         4937  +            int i;
         4938  +            cksum2 = cksum2 ^ fts3ChecksumEntry(
         4939  +                zToken, nToken, iLang, 0, iDocid, iCol, iPos
         4940  +            );
         4941  +            for(i=1; i<p->nIndex; i++){
         4942  +              if( p->aIndex[i].nPrefix<=nToken ){
         4943  +                cksum2 = cksum2 ^ fts3ChecksumEntry(
         4944  +                  zToken, p->aIndex[i].nPrefix, iLang, i, iDocid, iCol, iPos
         4945  +                );
         4946  +              }
         4947  +            }
         4948  +          }
         4949  +        }
         4950  +        if( pT ) pModule->xClose(pT);
         4951  +        if( rc==SQLITE_DONE ) rc = SQLITE_OK;
         4952  +      }
         4953  +    }
         4954  +
         4955  +    sqlite3_finalize(pStmt);
         4956  +  }
         4957  +
         4958  +  *pbOk = (cksum1==cksum2);
         4959  +  return rc;
         4960  +}
         4961  +
         4962  +/*
         4963  +** Run the integrity-check. If no error occurs and the current contents of
         4964  +** the FTS index are correct, return SQLITE_OK. Or, if the contents of the
         4965  +** FTS index are incorrect, return SQLITE_CORRUPT_VTAB.
         4966  +**
         4967  +** Or, if an error (e.g. an OOM or IO error) occurs, return an SQLite 
         4968  +** error code.
         4969  +**
         4970  +** The integrity-check works as follows. For each token and indexed token
         4971  +** prefix in the document set, a 64-bit checksum is calculated (by code
         4972  +** in fts3ChecksumEntry()) based on the following:
         4973  +**
         4974  +**     + The index number (0 for the main index, 1 for the first prefix
         4975  +**       index etc.),
         4976  +**     + The token (or token prefix) text itself, 
         4977  +**     + The language-id of the row it appears in,
         4978  +**     + The docid of the row it appears in,
         4979  +**     + The column it appears in, and
         4980  +**     + The tokens position within that column.
         4981  +**
         4982  +** The checksums for all entries in the index are XORed together to create
         4983  +** a single checksum for the entire index.
         4984  +**
         4985  +** The integrity-check code calculates the same checksum in two ways:
         4986  +**
         4987  +**     1. By scanning the contents of the FTS index, and 
         4988  +**     2. By scanning and tokenizing the content table.
         4989  +**
         4990  +** If the two checksums are identical, the integrity-check is deemed to have
         4991  +** passed.
         4992  +*/
         4993  +static int fts3DoIntegrityCheck(
         4994  +  Fts3Table *p                    /* FTS3 table handle */
         4995  +){
         4996  +  int rc;
         4997  +  int bOk = 0;
         4998  +  rc = fts3IntegrityCheck(p, &bOk);
         4999  +  if( rc==SQLITE_OK && bOk==0 ) rc = SQLITE_CORRUPT_VTAB;
         5000  +  return rc;
         5001  +}
         5002  +
  3173   5003   /*
  3174   5004   ** Handle a 'special' INSERT of the form:
  3175   5005   **
  3176   5006   **   "INSERT INTO tbl(tbl) VALUES(<expr>)"
  3177   5007   **
  3178   5008   ** Argument pVal contains the result of <expr>. Currently the only 
  3179   5009   ** meaningful value to insert is the text 'optimize'.
................................................................................
  3185   5015   
  3186   5016     if( !zVal ){
  3187   5017       return SQLITE_NOMEM;
  3188   5018     }else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){
  3189   5019       rc = fts3DoOptimize(p, 0);
  3190   5020     }else if( nVal==7 && 0==sqlite3_strnicmp(zVal, "rebuild", 7) ){
  3191   5021       rc = fts3DoRebuild(p);
         5022  +  }else if( nVal==15 && 0==sqlite3_strnicmp(zVal, "integrity-check", 15) ){
         5023  +    rc = fts3DoIntegrityCheck(p);
         5024  +  }else if( nVal>6 && 0==sqlite3_strnicmp(zVal, "merge=", 6) ){
         5025  +    rc = fts3DoIncrmerge(p, &zVal[6]);
         5026  +  }else if( nVal>10 && 0==sqlite3_strnicmp(zVal, "automerge=", 10) ){
         5027  +    rc = fts3DoAutoincrmerge(p, &zVal[10]);
  3192   5028   #ifdef SQLITE_TEST
  3193   5029     }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){
  3194   5030       p->nNodeSize = atoi(&zVal[9]);
  3195   5031       rc = SQLITE_OK;
  3196   5032     }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){
  3197   5033       p->nMaxPendingData = atoi(&zVal[11]);
  3198   5034       rc = SQLITE_OK;
................................................................................
  3380   5216   }
  3381   5217   
  3382   5218   /*
  3383   5219   ** This function does the work for the xUpdate method of FTS3 virtual
  3384   5220   ** tables. The schema of the virtual table being:
  3385   5221   **
  3386   5222   **     CREATE TABLE <table name>( 
  3387         -**       <user COLUMns>,
         5223  +**       <user columns>,
  3388   5224   **       <table name> HIDDEN, 
  3389   5225   **       docid HIDDEN, 
  3390   5226   **       <langid> HIDDEN
  3391   5227   **     );
  3392   5228   **
  3393   5229   ** 
  3394   5230   */
................................................................................
  3512   5348       }
  3513   5349       if( p->bHasDocsize ){
  3514   5350         fts3InsertDocsize(&rc, p, aSzIns);
  3515   5351       }
  3516   5352       nChng++;
  3517   5353     }
  3518   5354   
  3519         -  if( p->bHasStat ){
         5355  +  if( p->bFts4 ){
  3520   5356       fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nChng);
  3521   5357     }
  3522   5358   
  3523   5359    update_out:
  3524   5360     sqlite3_free(aSzIns);
  3525   5361     sqlite3Fts3SegmentsClose(p);
  3526   5362     return rc;

Added ext/fts3/tool/fts3view.c.

            1  +/*
            2  +** This program is a debugging and analysis utility that displays
            3  +** information about an FTS3 or FTS4 index.
            4  +**
            5  +** Link this program against the SQLite3 amalgamation with the
            6  +** SQLITE_ENABLE_FTS4 compile-time option.  Then run it as:
            7  +**
            8  +**    fts3view DATABASE
            9  +**
           10  +** to get a list of all FTS3/4 tables in DATABASE, or do
           11  +**
           12  +**    fts3view DATABASE TABLE COMMAND ....
           13  +**
           14  +** to see various aspects of the TABLE table.  Type fts3view with no
           15  +** arguments for a list of available COMMANDs.
           16  +*/
           17  +#include <stdio.h>
           18  +#include <stdarg.h>
           19  +#include <stdlib.h>
           20  +#include <string.h>
           21  +#include <ctype.h>
           22  +#include "sqlite3.h"
           23  +
           24  +/*
           25  +** Extra command-line arguments:
           26  +*/
           27  +int nExtra;
           28  +char **azExtra;
           29  +
           30  +/*
           31  +** Look for a command-line argument.
           32  +*/
           33  +const char *findOption(const char *zName, int hasArg, const char *zDefault){
           34  +  int i;
           35  +  const char *zResult = zDefault;
           36  +  for(i=0; i<nExtra; i++){
           37  +    const char *z = azExtra[i];
           38  +    while( z[0]=='-' ) z++;
           39  +    if( strcmp(z, zName)==0 ){
           40  +      int j = 1;
           41  +      if( hasArg==0 || i==nExtra-1 ) j = 0;
           42  +      zResult = azExtra[i+j];
           43  +      while( i+j<nExtra ){
           44  +        azExtra[i] = azExtra[i+j+1];
           45  +        i++;
           46  +      }
           47  +      break;
           48  +    }
           49  +  }
           50  +  return zResult;       
           51  +}
           52  +
           53  +
           54  +/*
           55  +** Prepare an SQL query
           56  +*/
           57  +static sqlite3_stmt *prepare(sqlite3 *db, const char *zFormat, ...){
           58  +  va_list ap;
           59  +  char *zSql;
           60  +  sqlite3_stmt *pStmt;
           61  +  int rc;
           62  +
           63  +  va_start(ap, zFormat);
           64  +  zSql = sqlite3_vmprintf(zFormat, ap);
           65  +  va_end(ap);
           66  +  rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
           67  +  if( rc ){
           68  +    fprintf(stderr, "Error: %s\nSQL: %s\n", sqlite3_errmsg(db), zSql);
           69  +    exit(1);
           70  +  }
           71  +  sqlite3_free(zSql);
           72  +  return pStmt;
           73  +}
           74  +
           75  +/*
           76  +** Run an SQL statement
           77  +*/
           78  +static int runSql(sqlite3 *db, const char *zFormat, ...){
           79  +  va_list ap;
           80  +  char *zSql;
           81  +  int rc;
           82  +
           83  +  va_start(ap, zFormat);
           84  +  zSql = sqlite3_vmprintf(zFormat, ap);
           85  +  rc = sqlite3_exec(db, zSql, 0, 0, 0);
           86  +  va_end(ap);
           87  +  return rc;
           88  +}
           89  +
           90  +/*
           91  +** Show the table schema
           92  +*/
           93  +static void showSchema(sqlite3 *db, const char *zTab){
           94  +  sqlite3_stmt *pStmt;
           95  +  pStmt = prepare(db,
           96  +            "SELECT sql FROM sqlite_master"
           97  +            " WHERE name LIKE '%q%%'"
           98  +            " ORDER BY 1",
           99  +            zTab);
          100  +  while( sqlite3_step(pStmt)==SQLITE_ROW ){
          101  +    printf("%s;\n", sqlite3_column_text(pStmt, 0));
          102  +  }
          103  +  sqlite3_finalize(pStmt);
          104  +  pStmt = prepare(db, "PRAGMA page_size");
          105  +  while( sqlite3_step(pStmt)==SQLITE_ROW ){
          106  +    printf("PRAGMA page_size=%s;\n", sqlite3_column_text(pStmt, 0));
          107  +  }
          108  +  sqlite3_finalize(pStmt);
          109  +  pStmt = prepare(db, "PRAGMA journal_mode");
          110  +  while( sqlite3_step(pStmt)==SQLITE_ROW ){
          111  +    printf("PRAGMA journal_mode=%s;\n", sqlite3_column_text(pStmt, 0));
          112  +  }
          113  +  sqlite3_finalize(pStmt);
          114  +  pStmt = prepare(db, "PRAGMA auto_vacuum");
          115  +  while( sqlite3_step(pStmt)==SQLITE_ROW ){
          116  +    const char *zType = "???";
          117  +    switch( sqlite3_column_int(pStmt, 0) ){
          118  +      case 0:  zType = "OFF";         break;
          119  +      case 1:  zType = "FULL";        break;
          120  +      case 2:  zType = "INCREMENTAL"; break;
          121  +    }
          122  +    printf("PRAGMA auto_vacuum=%s;\n", zType);
          123  +  }
          124  +  sqlite3_finalize(pStmt);
          125  +  pStmt = prepare(db, "PRAGMA encoding");
          126  +  while( sqlite3_step(pStmt)==SQLITE_ROW ){
          127  +    printf("PRAGMA encoding=%s;\n", sqlite3_column_text(pStmt, 0));
          128  +  }
          129  +  sqlite3_finalize(pStmt);
          130  +}
          131  +
          132  +/* 
          133  +** Read a 64-bit variable-length integer from memory starting at p[0].
          134  +** Return the number of bytes read, or 0 on error.
          135  +** The value is stored in *v.
          136  +*/
          137  +int getVarint(const unsigned char *p, sqlite_int64 *v){
          138  +  const unsigned char *q = p;
          139  +  sqlite_uint64 x = 0, y = 1;
          140  +  while( (*q&0x80)==0x80 && q-(unsigned char *)p<9 ){
          141  +    x += y * (*q++ & 0x7f);
          142  +    y <<= 7;
          143  +  }
          144  +  x += y * (*q++);
          145  +  *v = (sqlite_int64) x;
          146  +  return (int) (q - (unsigned char *)p);
          147  +}
          148  +
          149  +
          150  +/* Show the content of the %_stat table
          151  +*/
          152  +static void showStat(sqlite3 *db, const char *zTab){
          153  +  sqlite3_stmt *pStmt;
          154  +  pStmt = prepare(db, "SELECT id, value FROM '%q_stat'", zTab);
          155  +  while( sqlite3_step(pStmt)==SQLITE_ROW ){
          156  +    printf("stat[%d] =", sqlite3_column_int(pStmt, 0));
          157  +    switch( sqlite3_column_type(pStmt, 1) ){
          158  +      case SQLITE_INTEGER: {
          159  +        printf(" %d\n", sqlite3_column_int(pStmt, 1));
          160  +        break;
          161  +      }
          162  +      case SQLITE_BLOB: {
          163  +        unsigned char *x = (unsigned char*)sqlite3_column_blob(pStmt, 1);
          164  +        int len = sqlite3_column_bytes(pStmt, 1);
          165  +        int i = 0;
          166  +        sqlite3_int64 v;
          167  +        while( i<len ){
          168  +          i += getVarint(x, &v);
          169  +          printf(" %lld", v);
          170  +        }
          171  +        printf("\n");
          172  +        break;
          173  +      }
          174  +    }
          175  +  }
          176  +  sqlite3_finalize(pStmt);
          177  +}
          178  +
          179  +/*
          180  +** Report on the vocabulary.  This creates an fts4aux table with a random
          181  +** name, but deletes it in the end.
          182  +*/
          183  +static void showVocabulary(sqlite3 *db, const char *zTab){
          184  +  char *zAux;
          185  +  sqlite3_uint64 r;
          186  +  sqlite3_stmt *pStmt;
          187  +  int nDoc = 0;
          188  +  int nToken = 0;
          189  +  int nOccurrence = 0;
          190  +  int nTop;
          191  +  int n, i;
          192  +
          193  +  sqlite3_randomness(sizeof(r), &r);
          194  +  zAux = sqlite3_mprintf("viewer_%llx", zTab, r);
          195  +  runSql(db, "BEGIN");
          196  +  pStmt = prepare(db, "SELECT count(*) FROM %Q", zTab);
          197  +  while( sqlite3_step(pStmt)==SQLITE_ROW ){
          198  +    nDoc = sqlite3_column_int(pStmt, 0);
          199  +  }
          200  +  sqlite3_finalize(pStmt);
          201  +  printf("Number of documents...................... %9d\n", nDoc);
          202  +
          203  +  runSql(db, "CREATE VIRTUAL TABLE %s USING fts4aux(%Q)", zAux, zTab);
          204  +  pStmt = prepare(db, 
          205  +             "SELECT count(*), sum(occurrences) FROM %s WHERE col='*'",
          206  +             zAux);
          207  +  while( sqlite3_step(pStmt)==SQLITE_ROW ){
          208  +    nToken = sqlite3_column_int(pStmt, 0);
          209  +    nOccurrence = sqlite3_column_int(pStmt, 1);
          210  +  }
          211  +  sqlite3_finalize(pStmt);
          212  +  printf("Total tokens in all documents............ %9d\n", nOccurrence);
          213  +  printf("Total number of distinct tokens.......... %9d\n", nToken);
          214  +  if( nToken==0 ) goto end_vocab;
          215  +
          216  +  n = 0;
          217  +  pStmt = prepare(db, "SELECT count(*) FROM %s"
          218  +                      " WHERE col='*' AND occurrences==1", zAux);
          219  +  while( sqlite3_step(pStmt)==SQLITE_ROW ){
          220  +    n = sqlite3_column_int(pStmt, 0);
          221  +  }
          222  +  sqlite3_finalize(pStmt);
          223  +  printf("Tokens used exactly once................. %9d %5.2f%%\n",
          224  +          n, n*100.0/nToken);
          225  +
          226  +  n = 0;
          227  +  pStmt = prepare(db, "SELECT count(*) FROM %s"
          228  +                      " WHERE col='*' AND documents==1", zAux);
          229  +  while( sqlite3_step(pStmt)==SQLITE_ROW ){
          230  +    n = sqlite3_column_int(pStmt, 0);
          231  +  }
          232  +  sqlite3_finalize(pStmt);
          233  +  printf("Tokens used in only one document......... %9d %5.2f%%\n",
          234  +          n, n*100.0/nToken);
          235  +
          236  +  if( nDoc>=2000 ){
          237  +    n = 0;
          238  +    pStmt = prepare(db, "SELECT count(*) FROM %s"
          239  +                        " WHERE col='*' AND occurrences<=%d", zAux, nDoc/1000);
          240  +    while( sqlite3_step(pStmt)==SQLITE_ROW ){
          241  +      n = sqlite3_column_int(pStmt, 0);
          242  +    }
          243  +    sqlite3_finalize(pStmt);
          244  +    printf("Tokens used in 0.1%% or less of docs...... %9d %5.2f%%\n",
          245  +            n, n*100.0/nToken);
          246  +  }
          247  +
          248  +  if( nDoc>=200 ){
          249  +    n = 0;
          250  +    pStmt = prepare(db, "SELECT count(*) FROM %s"
          251  +                        " WHERE col='*' AND occurrences<=%d", zAux, nDoc/100);
          252  +    while( sqlite3_step(pStmt)==SQLITE_ROW ){
          253  +      n = sqlite3_column_int(pStmt, 0);
          254  +    }
          255  +    sqlite3_finalize(pStmt);
          256  +    printf("Tokens used in 1%% or less of docs........ %9d %5.2f%%\n",
          257  +            n, n*100.0/nToken);
          258  +  }
          259  +
          260  +  nTop = atoi(findOption("top", 1, "25"));
          261  +  printf("The %d most common tokens:\n", nTop);
          262  +  pStmt = prepare(db,
          263  +            "SELECT term, documents FROM %s"
          264  +            " WHERE col='*'"
          265  +            " ORDER BY documents DESC, term"
          266  +            " LIMIT %d", zAux, nTop);
          267  +  i = 0;
          268  +  while( sqlite3_step(pStmt)==SQLITE_ROW ){
          269  +    i++;
          270  +    n = sqlite3_column_int(pStmt, 1);
          271  +    printf("  %2d. %-30s %9d docs %5.2f%%\n", i,
          272  +      sqlite3_column_text(pStmt, 0), n, n*100.0/nDoc);
          273  +  }
          274  +  sqlite3_finalize(pStmt);
          275  +
          276  +end_vocab:
          277  +  runSql(db, "ROLLBACK");
          278  +  sqlite3_free(zAux);
          279  +}
          280  +
          281  +/*
          282  +** Report on the number and sizes of segments
          283  +*/
          284  +static void showSegmentStats(sqlite3 *db, const char *zTab){
          285  +  sqlite3_stmt *pStmt;
          286  +  int nSeg = 0;
          287  +  sqlite3_int64 szSeg = 0, mxSeg = 0;
          288  +  int nIdx = 0;
          289  +  sqlite3_int64 szIdx = 0, mxIdx = 0;
          290  +  int nRoot = 0;
          291  +  sqlite3_int64 szRoot = 0, mxRoot = 0;
          292  +  sqlite3_int64 mx;
          293  +  int nLeaf;
          294  +  int n;
          295  +  int pgsz;
          296  +  int mxLevel;
          297  +  int i;
          298  +
          299  +  pStmt = prepare(db,
          300  +                  "SELECT count(*), sum(length(block)), max(length(block))"
          301  +                  " FROM '%q_segments'",
          302  +                  zTab);
          303  +  while( sqlite3_step(pStmt)==SQLITE_ROW ){
          304  +    nSeg = sqlite3_column_int(pStmt, 0);
          305  +    szSeg = sqlite3_column_int64(pStmt, 1);
          306  +    mxSeg = sqlite3_column_int64(pStmt, 2);
          307  +  }
          308  +  sqlite3_finalize(pStmt);
          309  +  pStmt = prepare(db,
          310  +            "SELECT count(*), sum(length(block)), max(length(block))"
          311  +            "  FROM '%q_segments' a JOIN '%q_segdir' b"
          312  +            " WHERE a.blockid BETWEEN b.leaves_end_block+1 AND b.end_block",
          313  +            zTab, zTab);
          314  +  while( sqlite3_step(pStmt)==SQLITE_ROW ){
          315  +    nIdx = sqlite3_column_int(pStmt, 0);
          316  +    szIdx = sqlite3_column_int64(pStmt, 1);
          317  +    mxIdx = sqlite3_column_int64(pStmt, 2);
          318  +  }
          319  +  sqlite3_finalize(pStmt);
          320  +  pStmt = prepare(db,
          321  +            "SELECT count(*), sum(length(root)), max(length(root))"
          322  +            "  FROM '%q_segdir'",
          323  +            zTab);
          324  +  while( sqlite3_step(pStmt)==SQLITE_ROW ){
          325  +    nRoot = sqlite3_column_int(pStmt, 0);
          326  +    szRoot = sqlite3_column_int64(pStmt, 1);
          327  +    mxRoot = sqlite3_column_int64(pStmt, 2);
          328  +  }
          329  +  sqlite3_finalize(pStmt);
          330  +
          331  +  printf("Number of segments....................... %9d\n", nSeg+nRoot);
          332  +  printf("Number of leaf segments.................. %9d\n", nSeg-nIdx);
          333  +  printf("Number of index segments................. %9d\n", nIdx);
          334  +  printf("Number of root segments.................. %9d\n", nRoot);
          335  +  printf("Total size of all segments............... %9lld\n", szSeg+szRoot);
          336  +  printf("Total size of all leaf segments.......... %9lld\n", szSeg-szIdx);
          337  +  printf("Total size of all index segments......... %9lld\n", szIdx);
          338  +  printf("Total size of all root segments.......... %9lld\n", szRoot);
          339  +  if( nSeg>0 ){
          340  +    printf("Average size of all segments............. %11.1f\n",
          341  +            (double)(szSeg+szRoot)/(double)(nSeg+nRoot));
          342  +    printf("Average size of leaf segments............ %11.1f\n",
          343  +            (double)(szSeg-szIdx)/(double)(nSeg-nIdx));
          344  +  }
          345  +  if( nIdx>0 ){
          346  +    printf("Average size of index segments........... %11.1f\n",
          347  +            (double)szIdx/(double)nIdx);
          348  +  }
          349  +  if( nRoot>0 ){
          350  +    printf("Average size of root segments............ %11.1f\n",
          351  +            (double)szRoot/(double)nRoot);
          352  +  }
          353  +  mx = mxSeg;
          354  +  if( mx<mxRoot ) mx = mxRoot;
          355  +  printf("Maximum segment size..................... %9lld\n", mx);
          356  +  printf("Maximum index segment size............... %9lld\n", mxIdx);
          357  +  printf("Maximum root segment size................ %9lld\n", mxRoot);
          358  +
          359  +  pStmt = prepare(db, "PRAGMA page_size");
          360  +  pgsz = 1024;
          361  +  while( sqlite3_step(pStmt)==SQLITE_ROW ){
          362  +    pgsz = sqlite3_column_int(pStmt, 0);
          363  +  }
          364  +  sqlite3_finalize(pStmt);
          365  +  printf("Database page size....................... %9d\n", pgsz);
          366  +  pStmt = prepare(db,
          367  +            "SELECT count(*)"
          368  +            "  FROM '%q_segments' a JOIN '%q_segdir' b"
          369  +            " WHERE a.blockid BETWEEN b.start_block AND b.leaves_end_block"
          370  +            "   AND length(a.block)>%d",
          371  +            zTab, zTab, pgsz-45);
          372  +  n = 0;
          373  +  while( sqlite3_step(pStmt)==SQLITE_ROW ){
          374  +    n = sqlite3_column_int(pStmt, 0);
          375  +  }
          376  +  sqlite3_finalize(pStmt);
          377  +  nLeaf = nSeg - nIdx;
          378  +  printf("Leaf segments larger than %5d bytes.... %9d   %5.2f%%\n",
          379  +         pgsz-45, n, n*100.0/nLeaf);
          380  +
          381  +  pStmt = prepare(db, "SELECT max(level%%1024) FROM '%q_segdir'", zTab);
          382  +  mxLevel = 0;
          383  +  while( sqlite3_step(pStmt)==SQLITE_ROW ){
          384  +    mxLevel = sqlite3_column_int(pStmt, 0);
          385  +  }
          386  +  sqlite3_finalize(pStmt);
          387  +
          388  +  for(i=0; i<=mxLevel; i++){
          389  +    pStmt = prepare(db,
          390  +           "SELECT count(*), sum(len), avg(len), max(len), sum(len>%d),"
          391  +           "       count(distinct idx)"
          392  +           "  FROM (SELECT length(a.block) AS len, idx"
          393  +           "          FROM '%q_segments' a JOIN '%q_segdir' b"
          394  +           "         WHERE (a.blockid BETWEEN b.start_block"
          395  +                                       " AND b.leaves_end_block)"
          396  +           "           AND (b.level%%1024)==%d)",
          397  +           pgsz-45, zTab, zTab, i);
          398  +    if( sqlite3_step(pStmt)==SQLITE_ROW
          399  +     && (nLeaf = sqlite3_column_int(pStmt, 0))>0
          400  +    ){
          401  +      int nIdx = sqlite3_column_int(pStmt, 5);
          402  +      sqlite3_int64 sz;
          403  +      printf("For level %d:\n", i);
          404  +      printf("  Number of indexes...................... %9d\n", nIdx);
          405  +      printf("  Number of leaf segments................ %9d\n", nLeaf);
          406  +      if( nIdx>1 ){
          407  +        printf("  Average leaf segments per index........ %11.1f\n",
          408  +               (double)nLeaf/(double)nIdx);
          409  +      }
          410  +      printf("  Total size of all leaf segments........ %9lld\n",
          411  +             (sz = sqlite3_column_int64(pStmt, 1)));
          412  +      printf("  Average size of leaf segments.......... %11.1f\n",
          413  +             sqlite3_column_double(pStmt, 2));
          414  +      if( nIdx>1 ){
          415  +        printf("  Average leaf segment size per index.... %11.1f\n",
          416  +               (double)sz/(double)nIdx);
          417  +      }
          418  +      printf("  Maximum leaf segment size.............. %9lld\n",
          419  +             sqlite3_column_int64(pStmt, 3));
          420  +      n = sqlite3_column_int(pStmt, 4);
          421  +      printf("  Leaf segments larger than %5d bytes.. %9d   %5.2f%%\n",
          422  +             pgsz-45, n, n*100.0/nLeaf);
          423  +    }
          424  +    sqlite3_finalize(pStmt);
          425  +  }
          426  +}
          427  +
          428  +/*
          429  +** Print a single "tree" line of the segdir map output.
          430  +*/
          431  +static void printTreeLine(sqlite3_int64 iLower, sqlite3_int64 iUpper){
          432  +  printf("                 tree   %9lld", iLower);
          433  +  if( iUpper>iLower ){
          434  +    printf(" thru %9lld  (%lld blocks)", iUpper, iUpper-iLower+1);
          435  +  }
          436  +  printf("\n");
          437  +}
          438  +
          439  +/*
          440  +** Check to see if the block of a %_segments entry is NULL.
          441  +*/
          442  +static int isNullSegment(sqlite3 *db, const char *zTab, sqlite3_int64 iBlockId){
          443  +  sqlite3_stmt *pStmt;
          444  +  int rc = 1;
          445  +
          446  +  pStmt = prepare(db, "SELECT block IS NULL FROM '%q_segments'"
          447  +                      " WHERE blockid=%lld", zTab, iBlockId);
          448  +  if( sqlite3_step(pStmt)==SQLITE_ROW ){
          449  +    rc = sqlite3_column_int(pStmt, 0);
          450  +  }
          451  +  sqlite3_finalize(pStmt);
          452  +  return rc;
          453  +}
          454  +
          455  +/*
          456  +** Show a map of segments derived from the %_segdir table.
          457  +*/
          458  +static void showSegdirMap(sqlite3 *db, const char *zTab){
          459  +  int mxIndex, iIndex;
          460  +  sqlite3_stmt *pStmt = 0;
          461  +  sqlite3_stmt *pStmt2 = 0;
          462  +  int prevLevel;
          463  +
          464  +  pStmt = prepare(db, "SELECT max(level/1024) FROM '%q_segdir'", zTab);
          465  +  if( sqlite3_step(pStmt)==SQLITE_ROW ){
          466  +    mxIndex = sqlite3_column_int(pStmt, 0);
          467  +  }else{
          468  +    mxIndex = 0;
          469  +  }
          470  +  sqlite3_finalize(pStmt);
          471  +
          472  +  printf("Number of inverted indices............... %3d\n", mxIndex+1);
          473  +  pStmt = prepare(db,
          474  +    "SELECT level, idx, start_block, leaves_end_block, end_block, rowid"
          475  +    "  FROM '%q_segdir'"
          476  +    " WHERE level/1024==?"
          477  +    " ORDER BY level DESC, idx",
          478  +    zTab);
          479  +  pStmt2 = prepare(db,
          480  +    "SELECT blockid FROM '%q_segments'"
          481  +    " WHERE blockid BETWEEN ? AND ? ORDER BY blockid",
          482  +    zTab);
          483  +  for(iIndex=0; iIndex<=mxIndex; iIndex++){
          484  +    if( mxIndex>0 ){
          485  +      printf("**************************** Index %d "
          486  +             "****************************\n", iIndex);
          487  +    }
          488  +    sqlite3_bind_int(pStmt, 1, iIndex);
          489  +    prevLevel = -1;
          490  +    while( sqlite3_step(pStmt)==SQLITE_ROW ){
          491  +      int iLevel = sqlite3_column_int(pStmt, 0)%1024;
          492  +      int iIdx = sqlite3_column_int(pStmt, 1);
          493  +      sqlite3_int64 iStart = sqlite3_column_int64(pStmt, 2);
          494  +      sqlite3_int64 iLEnd = sqlite3_column_int64(pStmt, 3);
          495  +      sqlite3_int64 iEnd = sqlite3_column_int64(pStmt, 4);
          496  +      char rtag[20];
          497  +      if( iLevel!=prevLevel ){
          498  +        printf("level %2d idx %2d", iLevel, iIdx);
          499  +        prevLevel = iLevel;
          500  +      }else{
          501  +        printf("         idx %2d", iIdx);
          502  +      }
          503  +      sqlite3_snprintf(sizeof(rtag), rtag, "r%lld",
          504  +                       sqlite3_column_int64(pStmt,5));
          505  +      printf("  root   %9s\n", rtag);
          506  +      if( iLEnd>iStart ){
          507  +        sqlite3_int64 iLower, iPrev, iX;
          508  +        if( iLEnd+1<=iEnd ){
          509  +          sqlite3_bind_int64(pStmt2, 1, iLEnd+1);
          510  +          sqlite3_bind_int64(pStmt2, 2, iEnd);
          511  +          iLower = -1;        
          512  +          while( sqlite3_step(pStmt2)==SQLITE_ROW ){
          513  +            iX = sqlite3_column_int64(pStmt2, 0);
          514  +            if( iLower<0 ){
          515  +              iLower = iPrev = iX;
          516  +            }else if( iX==iPrev+1 ){
          517  +              iPrev = iX;
          518  +            }else{
          519  +              printTreeLine(iLower, iPrev);
          520  +              iLower = iPrev = iX;
          521  +            }
          522  +          }
          523  +          sqlite3_reset(pStmt2);
          524  +          if( iLower>=0 ){
          525  +            if( iLower==iPrev && iLower==iEnd
          526  +             && isNullSegment(db,zTab,iLower)
          527  +            ){
          528  +              printf("                 null   %9lld\n", iLower);
          529  +            }else{
          530  +              printTreeLine(iLower, iPrev);
          531  +            }
          532  +          }
          533  +        }
          534  +        printf("                 leaves %9lld thru %9lld  (%lld blocks)\n",
          535  +               iStart, iLEnd, iLEnd - iStart + 1);
          536  +      }
          537  +    }
          538  +    sqlite3_reset(pStmt);
          539  +  }
          540  +  sqlite3_finalize(pStmt);
          541  +  sqlite3_finalize(pStmt2);
          542  +}
          543  +
          544  +/*
          545  +** Decode a single segment block and display the results on stdout.
          546  +*/
          547  +static void decodeSegment(
          548  +  const unsigned char *aData,   /* Content to print */
          549  +  int nData                     /* Number of bytes of content */
          550  +){
          551  +  sqlite3_int64 iChild;
          552  +  sqlite3_int64 iPrefix;
          553  +  sqlite3_int64 nTerm;
          554  +  sqlite3_int64 n;
          555  +  sqlite3_int64 iDocsz;
          556  +  int iHeight;
          557  +  int i = 0;
          558  +  int cnt = 0;
          559  +  char zTerm[1000];
          560  +
          561  +  i += getVarint(aData, &n);
          562  +  iHeight = (int)n;
          563  +  printf("height: %d\n", iHeight);
          564  +  if( iHeight>0 ){
          565  +    i += getVarint(aData+i, &iChild);
          566  +    printf("left-child: %lld\n", iChild);
          567  +  }
          568  +  while( i<nData ){
          569  +    if( (cnt++)>0 ){
          570  +      i += getVarint(aData+i, &iPrefix);
          571  +    }else{
          572  +      iPrefix = 0;
          573  +    }
          574  +    i += getVarint(aData+i, &nTerm);
          575  +    if( iPrefix+nTerm+1 >= sizeof(zTerm) ){
          576  +      fprintf(stderr, "term to long\n");
          577  +      exit(1);
          578  +    }
          579  +    memcpy(zTerm+iPrefix, aData+i, nTerm);
          580  +    zTerm[iPrefix+nTerm] = 0;
          581  +    i += nTerm;
          582  +    if( iHeight==0 ){
          583  +      i += getVarint(aData+i, &iDocsz);
          584  +      printf("term: %-25s doclist %7lld bytes offset %d\n", zTerm, iDocsz, i);
          585  +      i += iDocsz;
          586  +    }else{
          587  +      printf("term: %-25s child %lld\n", zTerm, ++iChild);
          588  +    }
          589  +  }
          590  +}
          591  +  
          592  +  
          593  +/*
          594  +** Print a a blob as hex and ascii.
          595  +*/
          596  +static void printBlob(
          597  +  const unsigned char *aData,   /* Content to print */
          598  +  int nData                     /* Number of bytes of content */
          599  +){
          600  +  int i, j;
          601  +  const char *zOfstFmt;
          602  +  const int perLine = 16;
          603  +
          604  +  if( (nData&~0xfff)==0 ){
          605  +    zOfstFmt = " %03x: ";
          606  +  }else if( (nData&~0xffff)==0 ){
          607  +    zOfstFmt = " %04x: ";
          608  +  }else if( (nData&~0xfffff)==0 ){
          609  +    zOfstFmt = " %05x: ";
          610  +  }else if( (nData&~0xffffff)==0 ){
          611  +    zOfstFmt = " %06x: ";
          612  +  }else{
          613  +    zOfstFmt = " %08x: ";
          614  +  }
          615  +
          616  +  for(i=0; i<nData; i += perLine){
          617  +    fprintf(stdout, zOfstFmt, i);
          618  +    for(j=0; j<perLine; j++){
          619  +      if( i+j>nData ){
          620  +        fprintf(stdout, "   ");
          621  +      }else{
          622  +        fprintf(stdout,"%02x ", aData[i+j]);
          623  +      }
          624  +    }
          625  +    for(j=0; j<perLine; j++){
          626  +      if( i+j>nData ){
          627  +        fprintf(stdout, " ");
          628  +      }else{
          629  +        fprintf(stdout,"%c", isprint(aData[i+j]) ? aData[i+j] : '.');
          630  +      }
          631  +    }
          632  +    fprintf(stdout,"\n");
          633  +  }
          634  +}
          635  +
          636  +/*
          637  +** Convert text to a 64-bit integer
          638  +*/
          639  +static sqlite3_int64 atoi64(const char *z){
          640  +  sqlite3_int64 v = 0;
          641  +  while( z[0]>='0' && z[0]<='9' ){
          642  +     v = v*10 + z[0] - '0';
          643  +     z++;
          644  +  }
          645  +  return v;
          646  +}
          647  +
          648  +/*
          649  +** Return a prepared statement which, when stepped, will return in its
          650  +** first column the blob associated with segment zId.  If zId begins with
          651  +** 'r' then it is a rowid of a %_segdir entry.  Otherwise it is a
          652  +** %_segment entry.
          653  +*/
          654  +static sqlite3_stmt *prepareToGetSegment(
          655  +  sqlite3 *db,         /* The database */
          656  +  const char *zTab,    /* The FTS3/4 table name */
          657  +  const char *zId      /* ID of the segment to open */
          658  +){
          659  +  sqlite3_stmt *pStmt;
          660  +  if( zId[0]=='r' ){
          661  +    pStmt = prepare(db, "SELECT root FROM '%q_segdir' WHERE rowid=%lld",
          662  +                    zTab, atoi64(zId+1));
          663  +  }else{
          664  +    pStmt = prepare(db, "SELECT block FROM '%q_segments' WHERE blockid=%lld",
          665  +                    zTab, atoi64(zId));
          666  +  }
          667  +  return pStmt;
          668  +}
          669  +
          670  +/*
          671  +** Print the content of a segment or of the root of a segdir.  The segment
          672  +** or root is identified by azExtra[0].  If the first character of azExtra[0]
          673  +** is 'r' then the remainder is the integer rowid of the %_segdir entry.
          674  +** If the first character of azExtra[0] is not 'r' then, then all of
          675  +** azExtra[0] is an integer which is the block number.
          676  +**
          677  +** If the --raw option is present in azExtra, then a hex dump is provided.
          678  +** Otherwise a decoding is shown.
          679  +*/
          680  +static void showSegment(sqlite3 *db, const char *zTab){
          681  +  const unsigned char *aData;
          682  +  int nData;
          683  +  sqlite3_stmt *pStmt;
          684  +
          685  +  pStmt = prepareToGetSegment(db, zTab, azExtra[0]);
          686  +  if( sqlite3_step(pStmt)!=SQLITE_ROW ){
          687  +    sqlite3_finalize(pStmt);
          688  +    return;
          689  +  }
          690  +  nData = sqlite3_column_bytes(pStmt, 0);
          691  +  aData = sqlite3_column_blob(pStmt, 0);
          692  +  printf("Segment %s of size %d bytes:\n", azExtra[0], nData);
          693  +  if( findOption("raw", 0, 0)!=0 ){
          694  +    printBlob(aData, nData);
          695  +  }else{
          696  +    decodeSegment(aData, nData);
          697  +  }
          698  +  sqlite3_finalize(pStmt);
          699  +}
          700  +
          701  +/*
          702  +** Decode a single doclist and display the results on stdout.
          703  +*/
          704  +static void decodeDoclist(
          705  +  const unsigned char *aData,   /* Content to print */
          706  +  int nData                     /* Number of bytes of content */
          707  +){
          708  +  sqlite3_int64 iPrevDocid = 0;
          709  +  sqlite3_int64 iDocid;
          710  +  sqlite3_int64 iPos;
          711  +  sqlite3_int64 iPrevPos = 0;
          712  +  sqlite3_int64 iCol;
          713  +  int i = 0;
          714  +
          715  +  while( i<nData ){
          716  +    i += getVarint(aData+i, &iDocid);
          717  +    printf("docid %lld col0", iDocid+iPrevDocid);
          718  +    iPrevDocid += iDocid;
          719  +    iPrevPos = 0;
          720  +    while( 1 ){
          721  +      i += getVarint(aData+i, &iPos);
          722  +      if( iPos==1 ){
          723  +        i += getVarint(aData+i, &iCol);
          724  +        printf(" col%lld", iCol);
          725  +        iPrevPos = 0;
          726  +      }else if( iPos==0 ){
          727  +        printf("\n");
          728  +        break;
          729  +      }else{
          730  +        iPrevPos += iPos - 2;
          731  +        printf(" %lld", iPrevPos);
          732  +      }
          733  +    }
          734  +  }
          735  +}
          736  +  
          737  +
          738  +/*
          739  +** Print the content of a doclist.  The segment or segdir-root is
          740  +** identified by azExtra[0].  If the first character of azExtra[0]
          741  +** is 'r' then the remainder is the integer rowid of the %_segdir entry.
          742  +** If the first character of azExtra[0] is not 'r' then, then all of
          743  +** azExtra[0] is an integer which is the block number.  The offset
          744  +** into the segment is identified by azExtra[1].  The size of the doclist
          745  +** is azExtra[2].
          746  +**
          747  +** If the --raw option is present in azExtra, then a hex dump is provided.
          748  +** Otherwise a decoding is shown.
          749  +*/
          750  +static void showDoclist(sqlite3 *db, const char *zTab){
          751  +  const unsigned char *aData;
          752  +  sqlite3_int64 offset, nData;
          753  +  sqlite3_stmt *pStmt;
          754  +
          755  +  offset = atoi64(azExtra[1]);
          756  +  nData = atoi64(azExtra[2]);
          757  +  pStmt = prepareToGetSegment(db, zTab, azExtra[0]);
          758  +  if( sqlite3_step(pStmt)!=SQLITE_ROW ){
          759  +    sqlite3_finalize(pStmt);
          760  +    return;
          761  +  }
          762  +  aData = sqlite3_column_blob(pStmt, 0);
          763  +  printf("Doclist at %s offset %lld of size %lld bytes:\n",
          764  +         azExtra[0], offset, nData);
          765  +  if( findOption("raw", 0, 0)!=0 ){
          766  +    printBlob(aData+offset, nData);
          767  +  }else{
          768  +    decodeDoclist(aData+offset, nData);
          769  +  }
          770  +  sqlite3_finalize(pStmt);
          771  +}
          772  +
          773  +/*
          774  +** Show the top N largest segments
          775  +*/
          776  +static void listBigSegments(sqlite3 *db, const char *zTab){
          777  +  int nTop, i;
          778  +  sqlite3_stmt *pStmt;
          779  +  sqlite3_int64 sz;
          780  +  sqlite3_int64 id;
          781  +
          782  +  nTop = atoi(findOption("top", 1, "25"));
          783  +  printf("The %d largest segments:\n", nTop);
          784  +  pStmt = prepare(db,
          785  +            "SELECT blockid, length(block) AS len FROM '%q_segments'"
          786  +            " ORDER BY 2 DESC, 1"
          787  +            " LIMIT %d", zTab, nTop);
          788  +  i = 0;
          789  +  while( sqlite3_step(pStmt)==SQLITE_ROW ){
          790  +    i++;
          791  +    id = sqlite3_column_int64(pStmt, 0);
          792  +    sz = sqlite3_column_int64(pStmt, 1);
          793  +    printf("  %2d. %9lld size %lld\n", i, id, sz);
          794  +  }
          795  +  sqlite3_finalize(pStmt);
          796  +}
          797  +
          798  +
          799  +
          800  +static void usage(const char *argv0){
          801  +  fprintf(stderr, "Usage: %s DATABASE\n"
          802  +                  "   or: %s DATABASE FTS3TABLE ARGS...\n", argv0, argv0);
          803  +  fprintf(stderr,
          804  +    "ARGS:\n"
          805  +    "  big-segments [--top N]                    show the largest segments\n"
          806  +    "  doclist BLOCKID OFFSET SIZE [--raw]       Decode a doclist\n"
          807  +    "  schema                                    FTS table schema\n"
          808  +    "  segdir                                    directory of segments\n"
          809  +    "  segment BLOCKID [--raw]                   content of a segment\n"
          810  +    "  segment-stats                             info on segment sizes\n"
          811  +    "  stat                                      the %%_stat table\n"
          812  +    "  vocabulary [--top N]                      document vocabulary\n"
          813  +  );
          814  +  exit(1);
          815  +}
          816  +
          817  +int main(int argc, char **argv){
          818  +  sqlite3 *db;
          819  +  int rc;
          820  +  const char *zTab;
          821  +  const char *zCmd;
          822  +
          823  +  if( argc<2 ) usage(argv[0]);
          824  +  rc = sqlite3_open(argv[1], &db);
          825  +  if( rc ){
          826  +    fprintf(stderr, "Cannot open %s\n", argv[1]);
          827  +    exit(1);
          828  +  }
          829  +  if( argc==2 ){
          830  +    sqlite3_stmt *pStmt;
          831  +    int cnt = 0;
          832  +    pStmt = prepare(db, "SELECT b.sql"
          833  +                        "  FROM sqlite_master a, sqlite_master b"
          834  +                        " WHERE a.name GLOB '*_segdir'"
          835  +                        "   AND b.name=substr(a.name,1,length(a.name)-7)"
          836  +                        " ORDER BY 1");
          837  +    while( sqlite3_step(pStmt)==SQLITE_ROW ){
          838  +      cnt++;
          839  +      printf("%s;\n", sqlite3_column_text(pStmt, 0));
          840  +    }
          841  +    sqlite3_finalize(pStmt);
          842  +    if( cnt==0 ){
          843  +      printf("/* No FTS3/4 tables found in database %s */\n", argv[1]);
          844  +    }
          845  +    return 0;
          846  +  }
          847  +  if( argc<4 ) usage(argv[0]);
          848  +  zTab = argv[2];
          849  +  zCmd = argv[3];
          850  +  nExtra = argc-4;
          851  +  azExtra = argv+4;
          852  +  if( strcmp(zCmd,"big-segments")==0 ){
          853  +    listBigSegments(db, zTab);
          854  +  }else if( strcmp(zCmd,"doclist")==0 ){
          855  +    if( argc<7 ) usage(argv[0]);
          856  +    showDoclist(db, zTab);
          857  +  }else if( strcmp(zCmd,"schema")==0 ){
          858  +    showSchema(db, zTab);
          859  +  }else if( strcmp(zCmd,"segdir")==0 ){
          860  +    showSegdirMap(db, zTab);
          861  +  }else if( strcmp(zCmd,"segment")==0 ){
          862  +    if( argc<5 ) usage(argv[0]);
          863  +    showSegment(db, zTab);
          864  +  }else if( strcmp(zCmd,"segment-stats")==0 ){
          865  +    showSegmentStats(db, zTab);
          866  +  }else if( strcmp(zCmd,"stat")==0 ){
          867  +    showStat(db, zTab);
          868  +  }else if( strcmp(zCmd,"vocabulary")==0 ){
          869  +    showVocabulary(db, zTab);
          870  +  }else{
          871  +    usage(argv[0]);
          872  +  }
          873  +  return 0; 
          874  +}

Changes to src/btree.c.

  6785   6785     /* Assert that the caller has been consistent. If this cursor was opened
  6786   6786     ** expecting an index b-tree, then the caller should be inserting blob
  6787   6787     ** keys with no associated data. If the cursor was opened expecting an
  6788   6788     ** intkey table, the caller should be inserting integer keys with a
  6789   6789     ** blob of associated data.  */
  6790   6790     assert( (pKey==0)==(pCur->pKeyInfo==0) );
  6791   6791   
  6792         -  /* If this is an insert into a table b-tree, invalidate any incrblob 
  6793         -  ** cursors open on the row being replaced (assuming this is a replace
  6794         -  ** operation - if it is not, the following is a no-op).  */
  6795         -  if( pCur->pKeyInfo==0 ){
  6796         -    invalidateIncrblobCursors(p, nKey, 0);
  6797         -  }
  6798         -
  6799   6792     /* Save the positions of any other cursors open on this table.
  6800   6793     **
  6801   6794     ** In some cases, the call to btreeMoveto() below is a no-op. For
  6802   6795     ** example, when inserting data into a table with auto-generated integer
  6803   6796     ** keys, the VDBE layer invokes sqlite3BtreeLast() to figure out the 
  6804   6797     ** integer key to use. It then calls this function to actually insert the 
  6805   6798     ** data into the intkey B-Tree. In this case btreeMoveto() recognizes
  6806   6799     ** that the cursor is already where it needs to be and returns without
  6807   6800     ** doing any work. To avoid thwarting these optimizations, it is important
  6808   6801     ** not to clear the cursor here.
  6809   6802     */
  6810   6803     rc = saveAllCursors(pBt, pCur->pgnoRoot, pCur);
  6811   6804     if( rc ) return rc;
         6805  +
         6806  +  /* If this is an insert into a table b-tree, invalidate any incrblob 
         6807  +  ** cursors open on the row being replaced (assuming this is a replace
         6808  +  ** operation - if it is not, the following is a no-op).  */
         6809  +  if( pCur->pKeyInfo==0 ){
         6810  +    invalidateIncrblobCursors(p, nKey, 0);
         6811  +  }
         6812  +
  6812   6813     if( !loc ){
  6813   6814       rc = btreeMoveto(pCur, pKey, nKey, appendBias, &loc);
  6814   6815       if( rc ) return rc;
  6815   6816     }
  6816   6817     assert( pCur->eState==CURSOR_VALID || (pCur->eState==CURSOR_INVALID && loc) );
  6817   6818   
  6818   6819     pPage = pCur->apPage[pCur->iPage];
................................................................................
  6915   6916   
  6916   6917     if( NEVER(pCur->aiIdx[pCur->iPage]>=pCur->apPage[pCur->iPage]->nCell) 
  6917   6918      || NEVER(pCur->eState!=CURSOR_VALID)
  6918   6919     ){
  6919   6920       return SQLITE_ERROR;  /* Something has gone awry. */
  6920   6921     }
  6921   6922   
  6922         -  /* If this is a delete operation to remove a row from a table b-tree,
  6923         -  ** invalidate any incrblob cursors open on the row being deleted.  */
  6924         -  if( pCur->pKeyInfo==0 ){
  6925         -    invalidateIncrblobCursors(p, pCur->info.nKey, 0);
  6926         -  }
  6927         -
  6928   6923     iCellDepth = pCur->iPage;
  6929   6924     iCellIdx = pCur->aiIdx[iCellDepth];
  6930   6925     pPage = pCur->apPage[iCellDepth];
  6931   6926     pCell = findCell(pPage, iCellIdx);
  6932   6927   
  6933   6928     /* If the page containing the entry to delete is not a leaf page, move
  6934   6929     ** the cursor to the largest entry in the tree that is smaller than
................................................................................
  6946   6941     /* Save the positions of any other cursors open on this table before
  6947   6942     ** making any modifications. Make the page containing the entry to be 
  6948   6943     ** deleted writable. Then free any overflow pages associated with the 
  6949   6944     ** entry and finally remove the cell itself from within the page.  
  6950   6945     */
  6951   6946     rc = saveAllCursors(pBt, pCur->pgnoRoot, pCur);
  6952   6947     if( rc ) return rc;
         6948  +
         6949  +  /* If this is a delete operation to remove a row from a table b-tree,
         6950  +  ** invalidate any incrblob cursors open on the row being deleted.  */
         6951  +  if( pCur->pKeyInfo==0 ){
         6952  +    invalidateIncrblobCursors(p, pCur->info.nKey, 0);
         6953  +  }
         6954  +
  6953   6955     rc = sqlite3PagerWrite(pPage->pDbPage);
  6954   6956     if( rc ) return rc;
  6955   6957     rc = clearCell(pPage, pCell);
  6956   6958     dropCell(pPage, iCellIdx, cellSizePtr(pPage, pCell), &rc);
  6957   6959     if( rc ) return rc;
  6958   6960   
  6959   6961     /* If the cell deleted was not located on a leaf page, then the cursor
................................................................................
  7227   7229   */
  7228   7230   int sqlite3BtreeClearTable(Btree *p, int iTable, int *pnChange){
  7229   7231     int rc;
  7230   7232     BtShared *pBt = p->pBt;
  7231   7233     sqlite3BtreeEnter(p);
  7232   7234     assert( p->inTrans==TRANS_WRITE );
  7233   7235   
  7234         -  /* Invalidate all incrblob cursors open on table iTable (assuming iTable
  7235         -  ** is the root of a table b-tree - if it is not, the following call is
  7236         -  ** a no-op).  */
  7237         -  invalidateIncrblobCursors(p, 0, 1);
  7238         -
  7239   7236     rc = saveAllCursors(pBt, (Pgno)iTable, 0);
         7237  +
  7240   7238     if( SQLITE_OK==rc ){
         7239  +    /* Invalidate all incrblob cursors open on table iTable (assuming iTable
         7240  +    ** is the root of a table b-tree - if it is not, the following call is
         7241  +    ** a no-op).  */
         7242  +    invalidateIncrblobCursors(p, 0, 1);
  7241   7243       rc = clearDatabasePage(pBt, (Pgno)iTable, 0, pnChange);
  7242   7244     }
  7243   7245     sqlite3BtreeLeave(p);
  7244   7246     return rc;
  7245   7247   }
  7246   7248   
  7247   7249   /*

Changes to src/build.c.

  3036   3036     }
  3037   3037   
  3038   3038   exit_drop_index:
  3039   3039     sqlite3SrcListDelete(db, pName);
  3040   3040   }
  3041   3041   
  3042   3042   /*
  3043         -** pArray is a pointer to an array of objects.  Each object in the
  3044         -** array is szEntry bytes in size.  This routine allocates a new
  3045         -** object on the end of the array.
         3043  +** pArray is a pointer to an array of objects. Each object in the
         3044  +** array is szEntry bytes in size. This routine uses sqlite3DbRealloc()
         3045  +** to extend the array so that there is space for a new object at the end.
  3046   3046   **
  3047         -** *pnEntry is the number of entries already in use.  *pnAlloc is
  3048         -** the previously allocated size of the array.  initSize is the
  3049         -** suggested initial array size allocation.
         3047  +** When this function is called, *pnEntry contains the current size of
         3048  +** the array (in entries - so the allocation is ((*pnEntry) * szEntry) bytes
         3049  +** in total).
  3050   3050   **
  3051         -** The index of the new entry is returned in *pIdx.
         3051  +** If the realloc() is successful (i.e. if no OOM condition occurs), the
         3052  +** space allocated for the new object is zeroed, *pnEntry updated to
         3053  +** reflect the new size of the array and a pointer to the new allocation
         3054  +** returned. *pIdx is set to the index of the new array entry in this case.
  3052   3055   **
  3053         -** This routine returns a pointer to the array of objects.  This
  3054         -** might be the same as the pArray parameter or it might be a different
  3055         -** pointer if the array was resized.
         3056  +** Otherwise, if the realloc() fails, *pIdx is set to -1, *pnEntry remains
         3057  +** unchanged and a copy of pArray returned.
  3056   3058   */
  3057   3059   void *sqlite3ArrayAllocate(
  3058   3060     sqlite3 *db,      /* Connection to notify of malloc failures */
  3059   3061     void *pArray,     /* Array of objects.  Might be reallocated */
  3060   3062     int szEntry,      /* Size of each object in the array */
  3061   3063     int *pnEntry,     /* Number of objects currently in use */
  3062   3064     int *pIdx         /* Write the index of a new slot here */

Changes to src/delete.c.

   370    370       /* Collect rowids of every row to be deleted.
   371    371       */
   372    372       sqlite3VdbeAddOp2(v, OP_Null, 0, iRowSet);
   373    373       pWInfo = sqlite3WhereBegin(
   374    374           pParse, pTabList, pWhere, 0, 0, WHERE_DUPLICATES_OK
   375    375       );
   376    376       if( pWInfo==0 ) goto delete_from_cleanup;
   377         -    regRowid = sqlite3ExprCodeGetColumn(pParse, pTab, -1, iCur, iRowid);
          377  +    regRowid = sqlite3ExprCodeGetColumn(pParse, pTab, -1, iCur, iRowid, 0);
   378    378       sqlite3VdbeAddOp2(v, OP_RowSetAdd, iRowSet, regRowid);
   379    379       if( db->flags & SQLITE_CountRows ){
   380    380         sqlite3VdbeAddOp2(v, OP_AddImm, memCnt, 1);
   381    381       }
   382    382       sqlite3WhereEnd(pWInfo);
   383    383   
   384    384       /* Delete every item whose key was written to the list during the

Changes to src/expr.c.

  2028   2028     /* First replace any existing entry.
  2029   2029     **
  2030   2030     ** Actually, the way the column cache is currently used, we are guaranteed
  2031   2031     ** that the object will never already be in cache.  Verify this guarantee.
  2032   2032     */
  2033   2033   #ifndef NDEBUG
  2034   2034     for(i=0, p=pParse->aColCache; i<SQLITE_N_COLCACHE; i++, p++){
  2035         -#if 0 /* This code wold remove the entry from the cache if it existed */
  2036         -    if( p->iReg && p->iTable==iTab && p->iColumn==iCol ){
  2037         -      cacheEntryClear(pParse, p);
  2038         -      p->iLevel = pParse->iCacheLevel;
  2039         -      p->iReg = iReg;
  2040         -      p->lru = pParse->iCacheCnt++;
  2041         -      return;
  2042         -    }
  2043         -#endif
  2044   2035       assert( p->iReg==0 || p->iTable!=iTab || p->iColumn!=iCol );
  2045   2036     }
  2046   2037   #endif
  2047   2038   
  2048   2039     /* Find an empty slot and replace it */
  2049   2040     for(i=0, p=pParse->aColCache; i<SQLITE_N_COLCACHE; i++, p++){
  2050   2041       if( p->iReg==0 ){
................................................................................
  2171   2162   ** is called.  If iColumn<0 then code is generated that extracts the rowid.
  2172   2163   */
  2173   2164   int sqlite3ExprCodeGetColumn(
  2174   2165     Parse *pParse,   /* Parsing and code generating context */
  2175   2166     Table *pTab,     /* Description of the table we are reading from */
  2176   2167     int iColumn,     /* Index of the table column */
  2177   2168     int iTable,      /* The cursor pointing to the table */
  2178         -  int iReg         /* Store results here */
         2169  +  int iReg,        /* Store results here */
         2170  +  u8 p5            /* P5 value for OP_Column */
  2179   2171   ){
  2180   2172     Vdbe *v = pParse->pVdbe;
  2181   2173     int i;
  2182   2174     struct yColCache *p;
  2183   2175   
  2184   2176     for(i=0, p=pParse->aColCache; i<SQLITE_N_COLCACHE; i++, p++){
  2185   2177       if( p->iReg>0 && p->iTable==iTable && p->iColumn==iColumn ){
................................................................................
  2186   2178         p->lru = pParse->iCacheCnt++;
  2187   2179         sqlite3ExprCachePinRegister(pParse, p->iReg);
  2188   2180         return p->iReg;
  2189   2181       }
  2190   2182     }  
  2191   2183     assert( v!=0 );
  2192   2184     sqlite3ExprCodeGetColumnOfTable(v, pTab, iTable, iColumn, iReg);
  2193         -  sqlite3ExprCacheStore(pParse, iTable, iColumn, iReg);
         2185  +  if( p5 ){
         2186  +    sqlite3VdbeChangeP5(v, p5);
         2187  +  }else{   
         2188  +    sqlite3ExprCacheStore(pParse, iTable, iColumn, iReg);
         2189  +  }
  2194   2190     return iReg;
  2195   2191   }
  2196   2192   
  2197   2193   /*
  2198   2194   ** Clear all column cache entries.
  2199   2195   */
  2200   2196   void sqlite3ExprCacheClear(Parse *pParse){
................................................................................
  2314   2310       case TK_COLUMN: {
  2315   2311         if( pExpr->iTable<0 ){
  2316   2312           /* This only happens when coding check constraints */
  2317   2313           assert( pParse->ckBase>0 );
  2318   2314           inReg = pExpr->iColumn + pParse->ckBase;
  2319   2315         }else{
  2320   2316           inReg = sqlite3ExprCodeGetColumn(pParse, pExpr->pTab,
  2321         -                                 pExpr->iColumn, pExpr->iTable, target);
         2317  +                                 pExpr->iColumn, pExpr->iTable, target,
         2318  +                                 pExpr->op2);
  2322   2319         }
  2323   2320         break;
  2324   2321       }
  2325   2322       case TK_INTEGER: {
  2326   2323         codeInteger(pParse, pExpr, 0, target);
  2327   2324         break;
  2328   2325       }
................................................................................
  2591   2588           sqlite3VdbeResolveLabel(v, endCoalesce);
  2592   2589           break;
  2593   2590         }
  2594   2591   
  2595   2592   
  2596   2593         if( pFarg ){
  2597   2594           r1 = sqlite3GetTempRange(pParse, nFarg);
         2595  +
         2596  +        /* For length() and typeof() functions with a column argument,
         2597  +        ** set the P5 parameter to the OP_Column opcode to OPFLAG_LENGTHARG
         2598  +        ** or OPFLAG_TYPEOFARG respectively, to avoid unnecessary data
         2599  +        ** loading.
         2600  +        */
         2601  +        if( (pDef->flags & (SQLITE_FUNC_LENGTH|SQLITE_FUNC_TYPEOF))!=0 ){
         2602  +          u8 exprOp;
         2603  +          assert( nFarg==1 );
         2604  +          assert( pFarg->a[0].pExpr!=0 );
         2605  +          exprOp = pFarg->a[0].pExpr->op;
         2606  +          if( exprOp==TK_COLUMN || exprOp==TK_AGG_COLUMN ){
         2607  +            assert( SQLITE_FUNC_LENGTH==OPFLAG_LENGTHARG );
         2608  +            assert( SQLITE_FUNC_TYPEOF==OPFLAG_TYPEOFARG );
         2609  +            testcase( pDef->flags==SQLITE_FUNC_LENGTH );
         2610  +            pFarg->a[0].pExpr->op2 = pDef->flags;
         2611  +          }
         2612  +        }
         2613  +
  2598   2614           sqlite3ExprCachePush(pParse);     /* Ticket 2ea2425d34be */
  2599   2615           sqlite3ExprCodeExprList(pParse, pFarg, r1, 1);
  2600   2616           sqlite3ExprCachePop(pParse, 1);   /* Ticket 2ea2425d34be */
  2601   2617         }else{
  2602   2618           r1 = 0;
  2603   2619         }
  2604   2620   #ifndef SQLITE_OMIT_VIRTUALTABLE

Changes to src/func.c.

  1538   1538       FUNCTION(trim,               2, 3, 0, trimFunc         ),
  1539   1539       FUNCTION(min,               -1, 0, 1, minmaxFunc       ),
  1540   1540       FUNCTION(min,                0, 0, 1, 0                ),
  1541   1541       AGGREGATE(min,               1, 0, 1, minmaxStep,      minMaxFinalize ),
  1542   1542       FUNCTION(max,               -1, 1, 1, minmaxFunc       ),
  1543   1543       FUNCTION(max,                0, 1, 1, 0                ),
  1544   1544       AGGREGATE(max,               1, 1, 1, minmaxStep,      minMaxFinalize ),
  1545         -    FUNCTION(typeof,             1, 0, 0, typeofFunc       ),
  1546         -    FUNCTION(length,             1, 0, 0, lengthFunc       ),
         1545  +    FUNCTION2(typeof,            1, 0, 0, typeofFunc,  SQLITE_FUNC_TYPEOF),
         1546  +    FUNCTION2(length,            1, 0, 0, lengthFunc,  SQLITE_FUNC_LENGTH),
  1547   1547       FUNCTION(substr,             2, 0, 0, substrFunc       ),
  1548   1548       FUNCTION(substr,             3, 0, 0, substrFunc       ),
  1549   1549       FUNCTION(abs,                1, 0, 0, absFunc          ),
  1550   1550   #ifndef SQLITE_OMIT_FLOATING_POINT
  1551   1551       FUNCTION(round,              1, 0, 0, roundFunc        ),
  1552   1552       FUNCTION(round,              2, 0, 0, roundFunc        ),
  1553   1553   #endif
  1554   1554       FUNCTION(upper,              1, 0, 0, upperFunc        ),
  1555   1555       FUNCTION(lower,              1, 0, 0, lowerFunc        ),
  1556   1556       FUNCTION(coalesce,           1, 0, 0, 0                ),
  1557   1557       FUNCTION(coalesce,           0, 0, 0, 0                ),
  1558         -/*  FUNCTION(coalesce,          -1, 0, 0, ifnullFunc       ), */
  1559         -    {-1,SQLITE_UTF8,SQLITE_FUNC_COALESCE,0,0,ifnullFunc,0,0,"coalesce",0,0},
         1558  +    FUNCTION2(coalesce,         -1, 0, 0, ifnullFunc,  SQLITE_FUNC_COALESCE),
  1560   1559       FUNCTION(hex,                1, 0, 0, hexFunc          ),
  1561         -/*  FUNCTION(ifnull,             2, 0, 0, ifnullFunc       ), */
  1562         -    {2,SQLITE_UTF8,SQLITE_FUNC_COALESCE,0,0,ifnullFunc,0,0,"ifnull",0,0},
         1560  +    FUNCTION2(ifnull,            2, 0, 0, ifnullFunc,  SQLITE_FUNC_COALESCE),
  1563   1561       FUNCTION(random,             0, 0, 0, randomFunc       ),
  1564   1562       FUNCTION(randomblob,         1, 0, 0, randomBlob       ),
  1565   1563       FUNCTION(nullif,             2, 0, 1, nullifFunc       ),
  1566   1564       FUNCTION(sqlite_version,     0, 0, 0, versionFunc      ),
  1567   1565       FUNCTION(sqlite_source_id,   0, 0, 0, sourceidFunc     ),
  1568   1566       FUNCTION(sqlite_log,         2, 0, 0, errlogFunc       ),
  1569   1567   #ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS

Changes to src/pager.c.

   666    666     int pageSize;               /* Number of bytes in a page */
   667    667     Pgno mxPgno;                /* Maximum allowed size of the database */
   668    668     i64 journalSizeLimit;       /* Size limit for persistent journal files */
   669    669     char *zFilename;            /* Name of the database file */
   670    670     char *zJournal;             /* Name of the journal file */
   671    671     int (*xBusyHandler)(void*); /* Function to call when busy */
   672    672     void *pBusyHandlerArg;      /* Context argument for xBusyHandler */
   673         -  int nHit, nMiss;            /* Total cache hits and misses */
          673  +  int aStat[3];               /* Total cache hits, misses and writes */
   674    674   #ifdef SQLITE_TEST
   675         -  int nRead, nWrite;          /* Database pages read/written */
          675  +  int nRead;                  /* Database pages read */
   676    676   #endif
   677    677     void (*xReiniter)(DbPage*); /* Call this routine when reloading pages */
   678    678   #ifdef SQLITE_HAS_CODEC
   679    679     void *(*xCodec)(void*,void*,Pgno,int); /* Routine for en/decoding data */
   680    680     void (*xCodecSizeChng)(void*,int,int); /* Notify of page size changes */
   681    681     void (*xCodecFree)(void*);             /* Destructor for the codec */
   682    682     void *pCodec;               /* First argument to xCodec... methods */
................................................................................
   685    685     PCache *pPCache;            /* Pointer to page cache object */
   686    686   #ifndef SQLITE_OMIT_WAL
   687    687     Wal *pWal;                  /* Write-ahead log used by "journal_mode=wal" */
   688    688     char *zWal;                 /* File name for write-ahead log */
   689    689   #endif
   690    690   };
   691    691   
          692  +/*
          693  +** Indexes for use with Pager.aStat[]. The Pager.aStat[] array contains
          694  +** the values accessed by passing SQLITE_DBSTATUS_CACHE_HIT, CACHE_MISS 
          695  +** or CACHE_WRITE to sqlite3_db_status().
          696  +*/
          697  +#define PAGER_STAT_HIT   0
          698  +#define PAGER_STAT_MISS  1
          699  +#define PAGER_STAT_WRITE 2
          700  +
   692    701   /*
   693    702   ** The following global variables hold counters used for
   694    703   ** testing purposes only.  These variables do not exist in
   695    704   ** a non-testing build.  These variables are not thread-safe.
   696    705   */
   697    706   #ifdef SQLITE_TEST
   698    707   int sqlite3_pager_readdb_count = 0;    /* Number of full pages read from DB */
................................................................................
  2967   2976   static int pagerWalFrames(
  2968   2977     Pager *pPager,                  /* Pager object */
  2969   2978     PgHdr *pList,                   /* List of frames to log */
  2970   2979     Pgno nTruncate,                 /* Database size after this commit */
  2971   2980     int isCommit                    /* True if this is a commit */
  2972   2981   ){
  2973   2982     int rc;                         /* Return code */
         2983  +  int nList;                      /* Number of pages in pList */
  2974   2984   #if defined(SQLITE_DEBUG) || defined(SQLITE_CHECK_PAGES)
  2975   2985     PgHdr *p;                       /* For looping over pages */
  2976   2986   #endif
  2977   2987   
  2978   2988     assert( pPager->pWal );
  2979   2989     assert( pList );
  2980   2990   #ifdef SQLITE_DEBUG
  2981   2991     /* Verify that the page list is in accending order */
  2982   2992     for(p=pList; p && p->pDirty; p=p->pDirty){
  2983   2993       assert( p->pgno < p->pDirty->pgno );
  2984   2994     }
  2985   2995   #endif
  2986   2996   
         2997  +  assert( pList->pDirty==0 || isCommit );
  2987   2998     if( isCommit ){
  2988   2999       /* If a WAL transaction is being committed, there is no point in writing
  2989   3000       ** any pages with page numbers greater than nTruncate into the WAL file.
  2990   3001       ** They will never be read by any client. So remove them from the pDirty
  2991   3002       ** list here. */
  2992   3003       PgHdr *p;
  2993   3004       PgHdr **ppNext = &pList;
         3005  +    nList = 0;
  2994   3006       for(p=pList; (*ppNext = p); p=p->pDirty){
  2995         -      if( p->pgno<=nTruncate ) ppNext = &p->pDirty;
         3007  +      if( p->pgno<=nTruncate ){
         3008  +        ppNext = &p->pDirty;
         3009  +        nList++;
         3010  +      }
  2996   3011       }
  2997   3012       assert( pList );
         3013  +  }else{
         3014  +    nList = 1;
  2998   3015     }
         3016  +  pPager->aStat[PAGER_STAT_WRITE] += nList;
  2999   3017   
  3000   3018     if( pList->pgno==1 ) pager_write_changecounter(pList);
  3001   3019     rc = sqlite3WalFrames(pPager->pWal, 
  3002   3020         pPager->pageSize, pList, nTruncate, isCommit, pPager->walSyncFlags
  3003   3021     );
  3004   3022     if( rc==SQLITE_OK && pPager->pBackup ){
  3005   3023       PgHdr *p;
................................................................................
  4059   4077         */
  4060   4078         if( pgno==1 ){
  4061   4079           memcpy(&pPager->dbFileVers, &pData[24], sizeof(pPager->dbFileVers));
  4062   4080         }
  4063   4081         if( pgno>pPager->dbFileSize ){
  4064   4082           pPager->dbFileSize = pgno;
  4065   4083         }
         4084  +      pPager->aStat[PAGER_STAT_WRITE]++;
  4066   4085   
  4067   4086         /* Update any backup objects copying the contents of this pager. */
  4068   4087         sqlite3BackupUpdate(pPager->pBackup, pgno, (u8*)pList->pData);
  4069   4088   
  4070   4089         PAGERTRACE(("STORE %d page %d hash(%08x)\n",
  4071   4090                      PAGERID(pPager), pgno, pager_pagehash(pList)));
  4072   4091         IOTRACE(("PGOUT %p %d\n", pPager, pgno));
  4073   4092         PAGER_INCR(sqlite3_pager_writedb_count);
  4074         -      PAGER_INCR(pPager->nWrite);
  4075   4093       }else{
  4076   4094         PAGERTRACE(("NOSTORE %d page %d\n", PAGERID(pPager), pgno));
  4077   4095       }
  4078   4096       pager_set_pagehash(pList);
  4079   4097       pList = pList->pDirty;
  4080   4098     }
  4081   4099   
................................................................................
  5025   5043     assert( (*ppPage)->pgno==pgno );
  5026   5044     assert( (*ppPage)->pPager==pPager || (*ppPage)->pPager==0 );
  5027   5045   
  5028   5046     if( (*ppPage)->pPager && !noContent ){
  5029   5047       /* In this case the pcache already contains an initialized copy of
  5030   5048       ** the page. Return without further ado.  */
  5031   5049       assert( pgno<=PAGER_MAX_PGNO && pgno!=PAGER_MJ_PGNO(pPager) );
  5032         -    pPager->nHit++;
         5050  +    pPager->aStat[PAGER_STAT_HIT]++;
  5033   5051       return SQLITE_OK;
  5034   5052   
  5035   5053     }else{
  5036   5054       /* The pager cache has created a new page. Its content needs to 
  5037   5055       ** be initialized.  */
  5038   5056   
  5039   5057       pPg = *ppPage;
................................................................................
  5067   5085           testcase( rc==SQLITE_NOMEM );
  5068   5086           sqlite3EndBenignMalloc();
  5069   5087         }
  5070   5088         memset(pPg->pData, 0, pPager->pageSize);
  5071   5089         IOTRACE(("ZERO %p %d\n", pPager, pgno));
  5072   5090       }else{
  5073   5091         assert( pPg->pPager==pPager );
  5074         -      pPager->nMiss++;
         5092  +      pPager->aStat[PAGER_STAT_MISS]++;
  5075   5093         rc = readDbPage(pPg);
  5076   5094         if( rc!=SQLITE_OK ){
  5077   5095           goto pager_acquire_err;
  5078   5096         }
  5079   5097       }
  5080   5098       pager_set_pagehash(pPg);
  5081   5099     }
................................................................................
  5652   5670         /* If running in direct mode, write the contents of page 1 to the file. */
  5653   5671         if( DIRECT_MODE ){
  5654   5672           const void *zBuf;
  5655   5673           assert( pPager->dbFileSize>0 );
  5656   5674           CODEC2(pPager, pPgHdr->pData, 1, 6, rc=SQLITE_NOMEM, zBuf);
  5657   5675           if( rc==SQLITE_OK ){
  5658   5676             rc = sqlite3OsWrite(pPager->fd, zBuf, pPager->pageSize, 0);
         5677  +          pPager->aStat[PAGER_STAT_WRITE]++;
  5659   5678           }
  5660   5679           if( rc==SQLITE_OK ){
  5661   5680             pPager->changeCountDone = 1;
  5662   5681           }
  5663   5682         }else{
  5664   5683           pPager->changeCountDone = 1;
  5665   5684         }
................................................................................
  6095   6114     static int a[11];
  6096   6115     a[0] = sqlite3PcacheRefCount(pPager->pPCache);
  6097   6116     a[1] = sqlite3PcachePagecount(pPager->pPCache);
  6098   6117     a[2] = sqlite3PcacheGetCachesize(pPager->pPCache);
  6099   6118     a[3] = pPager->eState==PAGER_OPEN ? -1 : (int) pPager->dbSize;
  6100   6119     a[4] = pPager->eState;
  6101   6120     a[5] = pPager->errCode;
  6102         -  a[6] = pPager->nHit;
  6103         -  a[7] = pPager->nMiss;
         6121  +  a[6] = pPager->aStat[PAGER_STAT_HIT];
         6122  +  a[7] = pPager->aStat[PAGER_STAT_MISS];
  6104   6123     a[8] = 0;  /* Used to be pPager->nOvfl */
  6105   6124     a[9] = pPager->nRead;
  6106         -  a[10] = pPager->nWrite;
         6125  +  a[10] = pPager->aStat[PAGER_STAT_WRITE];
  6107   6126     return a;
  6108   6127   }
  6109   6128   #endif
  6110   6129   
  6111   6130   /*
  6112   6131   ** Parameter eStat must be either SQLITE_DBSTATUS_CACHE_HIT or
  6113   6132   ** SQLITE_DBSTATUS_CACHE_MISS. Before returning, *pnVal is incremented by the
  6114   6133   ** current cache hit or miss count, according to the value of eStat. If the 
  6115   6134   ** reset parameter is non-zero, the cache hit or miss count is zeroed before 
  6116   6135   ** returning.
  6117   6136   */
  6118   6137   void sqlite3PagerCacheStat(Pager *pPager, int eStat, int reset, int *pnVal){
  6119         -  int *piStat;
  6120   6138   
  6121   6139     assert( eStat==SQLITE_DBSTATUS_CACHE_HIT
  6122   6140          || eStat==SQLITE_DBSTATUS_CACHE_MISS
         6141  +       || eStat==SQLITE_DBSTATUS_CACHE_WRITE
  6123   6142     );
  6124         -  if( eStat==SQLITE_DBSTATUS_CACHE_HIT ){
  6125         -    piStat = &pPager->nHit;
  6126         -  }else{
  6127         -    piStat = &pPager->nMiss;
  6128         -  }
  6129   6143   
  6130         -  *pnVal += *piStat;
         6144  +  assert( SQLITE_DBSTATUS_CACHE_HIT+1==SQLITE_DBSTATUS_CACHE_MISS );
         6145  +  assert( SQLITE_DBSTATUS_CACHE_HIT+2==SQLITE_DBSTATUS_CACHE_WRITE );
         6146  +  assert( PAGER_STAT_HIT==0 && PAGER_STAT_MISS==1 && PAGER_STAT_WRITE==2 );
         6147  +
         6148  +  *pnVal += pPager->aStat[eStat - SQLITE_DBSTATUS_CACHE_HIT];
  6131   6149     if( reset ){
  6132         -    *piStat = 0;
         6150  +    pPager->aStat[eStat - SQLITE_DBSTATUS_CACHE_HIT] = 0;
  6133   6151     }
  6134   6152   }
  6135   6153   
  6136   6154   /*
  6137   6155   ** Return true if this is an in-memory pager.
  6138   6156   */
  6139   6157   int sqlite3PagerIsMemdb(Pager *pPager){

Changes to src/parse.y.

    71     71   
    72     72   /*
    73     73   ** An instance of this structure is used to store the LIKE,
    74     74   ** GLOB, NOT LIKE, and NOT GLOB operators.
    75     75   */
    76     76   struct LikeOp {
    77     77     Token eOperator;  /* "like" or "glob" or "regexp" */
    78         -  int not;         /* True if the NOT keyword is present */
           78  +  int bNot;         /* True if the NOT keyword is present */
    79     79   };
    80     80   
    81     81   /*
    82     82   ** An instance of the following structure describes the event of a
    83     83   ** TRIGGER.  "a" is the event type, one of TK_UPDATE, TK_INSERT,
    84     84   ** TK_DELETE, or TK_INSTEAD.  If the event is of the form
    85     85   **
................................................................................
   877    877                                           {spanBinaryExpr(&A,pParse,@OP,&X,&Y);}
   878    878   expr(A) ::= expr(X) PLUS|MINUS(OP) expr(Y).
   879    879                                           {spanBinaryExpr(&A,pParse,@OP,&X,&Y);}
   880    880   expr(A) ::= expr(X) STAR|SLASH|REM(OP) expr(Y).
   881    881                                           {spanBinaryExpr(&A,pParse,@OP,&X,&Y);}
   882    882   expr(A) ::= expr(X) CONCAT(OP) expr(Y). {spanBinaryExpr(&A,pParse,@OP,&X,&Y);}
   883    883   %type likeop {struct LikeOp}
   884         -likeop(A) ::= LIKE_KW(X).     {A.eOperator = X; A.not = 0;}
   885         -likeop(A) ::= NOT LIKE_KW(X). {A.eOperator = X; A.not = 1;}
   886         -likeop(A) ::= MATCH(X).       {A.eOperator = X; A.not = 0;}
   887         -likeop(A) ::= NOT MATCH(X).   {A.eOperator = X; A.not = 1;}
          884  +likeop(A) ::= LIKE_KW(X).     {A.eOperator = X; A.bNot = 0;}
          885  +likeop(A) ::= NOT LIKE_KW(X). {A.eOperator = X; A.bNot = 1;}
          886  +likeop(A) ::= MATCH(X).       {A.eOperator = X; A.bNot = 0;}
          887  +likeop(A) ::= NOT MATCH(X).   {A.eOperator = X; A.bNot = 1;}
   888    888   expr(A) ::= expr(X) likeop(OP) expr(Y).  [LIKE_KW]  {
   889    889     ExprList *pList;
   890    890     pList = sqlite3ExprListAppend(pParse,0, Y.pExpr);
   891    891     pList = sqlite3ExprListAppend(pParse,pList, X.pExpr);
   892    892     A.pExpr = sqlite3ExprFunction(pParse, pList, &OP.eOperator);
   893         -  if( OP.not ) A.pExpr = sqlite3PExpr(pParse, TK_NOT, A.pExpr, 0, 0);
          893  +  if( OP.bNot ) A.pExpr = sqlite3PExpr(pParse, TK_NOT, A.pExpr, 0, 0);
   894    894     A.zStart = X.zStart;
   895    895     A.zEnd = Y.zEnd;
   896    896     if( A.pExpr ) A.pExpr->flags |= EP_InfixFunc;
   897    897   }
   898    898   expr(A) ::= expr(X) likeop(OP) expr(Y) ESCAPE expr(E).  [LIKE_KW]  {
   899    899     ExprList *pList;
   900    900     pList = sqlite3ExprListAppend(pParse,0, Y.pExpr);
   901    901     pList = sqlite3ExprListAppend(pParse,pList, X.pExpr);
   902    902     pList = sqlite3ExprListAppend(pParse,pList, E.pExpr);
   903    903     A.pExpr = sqlite3ExprFunction(pParse, pList, &OP.eOperator);
   904         -  if( OP.not ) A.pExpr = sqlite3PExpr(pParse, TK_NOT, A.pExpr, 0, 0);
          904  +  if( OP.bNot ) A.pExpr = sqlite3PExpr(pParse, TK_NOT, A.pExpr, 0, 0);
   905    905     A.zStart = X.zStart;
   906    906     A.zEnd = E.zEnd;
   907    907     if( A.pExpr ) A.pExpr->flags |= EP_InfixFunc;
   908    908   }
   909    909   
   910    910   %include {
   911    911     /* Construct an expression node for a unary postfix operator

Changes to src/select.c.

  4224   4224           for(i=0; i<sAggInfo.nColumn; i++){
  4225   4225             struct AggInfo_col *pCol = &sAggInfo.aCol[i];
  4226   4226             if( pCol->iSorterColumn>=j ){
  4227   4227               int r1 = j + regBase;
  4228   4228               int r2;
  4229   4229   
  4230   4230               r2 = sqlite3ExprCodeGetColumn(pParse, 
  4231         -                               pCol->pTab, pCol->iColumn, pCol->iTable, r1);
         4231  +                               pCol->pTab, pCol->iColumn, pCol->iTable, r1, 0);
  4232   4232               if( r1!=r2 ){
  4233   4233                 sqlite3VdbeAddOp2(v, OP_SCopy, r2, r1);
  4234   4234               }
  4235   4235               j++;
  4236   4236             }
  4237   4237           }
  4238   4238           regRecord = sqlite3GetTempReg(pParse);

Changes to src/shell.c.

    64     64   # define stifle_history(X)
    65     65   #endif
    66     66   
    67     67   #if defined(_WIN32) || defined(WIN32)
    68     68   # include <io.h>
    69     69   #define isatty(h) _isatty(h)
    70     70   #define access(f,m) _access((f),(m))
           71  +#define popen(a,b) _popen((a),(b))
           72  +#define pclose(x) _pclose(x)
    71     73   #else
    72     74   /* Make sure isatty() has a prototype.
    73     75   */
    74     76   extern int isatty(int);
    75     77   #endif
    76     78   
    77     79   #if defined(_WIN32_WCE)
................................................................................
  1073   1075       fprintf(pArg->out, "Pager Heap Usage:                    %d bytes\n", iCur);    iHiwtr = iCur = -1;
  1074   1076       sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_HIT, &iCur, &iHiwtr, 1);
  1075   1077       fprintf(pArg->out, "Page cache hits:                     %d\n", iCur);
  1076   1078       iHiwtr = iCur = -1;
  1077   1079       sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_MISS, &iCur, &iHiwtr, 1);
  1078   1080       fprintf(pArg->out, "Page cache misses:                   %d\n", iCur); 
  1079   1081       iHiwtr = iCur = -1;
         1082  +    sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_WRITE, &iCur, &iHiwtr, 1);
         1083  +    fprintf(pArg->out, "Page cache writes:                   %d\n", iCur); 
         1084  +    iHiwtr = iCur = -1;
  1080   1085       sqlite3_db_status(db, SQLITE_DBSTATUS_SCHEMA_USED, &iCur, &iHiwtr, bReset);
  1081   1086       fprintf(pArg->out, "Schema Heap Usage:                   %d bytes\n", iCur); 
  1082   1087       iHiwtr = iCur = -1;
  1083   1088       sqlite3_db_status(db, SQLITE_DBSTATUS_STMT_USED, &iCur, &iHiwtr, bReset);
  1084   1089       fprintf(pArg->out, "Statement Heap/Lookaside Usage:      %d bytes\n", iCur); 
  1085   1090     }
  1086   1091   
................................................................................
  1283   1288   
  1284   1289     if( strcmp(zType, "table")==0 ){
  1285   1290       sqlite3_stmt *pTableInfo = 0;
  1286   1291       char *zSelect = 0;
  1287   1292       char *zTableInfo = 0;
  1288   1293       char *zTmp = 0;
  1289   1294       int nRow = 0;
  1290         -    int kk;
  1291   1295      
  1292   1296       zTableInfo = appendText(zTableInfo, "PRAGMA table_info(", 0);
  1293   1297       zTableInfo = appendText(zTableInfo, zTable, '"');
  1294   1298       zTableInfo = appendText(zTableInfo, ");", 0);
  1295   1299   
  1296   1300       rc = sqlite3_prepare(p->db, zTableInfo, -1, &pTableInfo, 0);
  1297   1301       free(zTableInfo);
  1298   1302       if( rc!=SQLITE_OK || !pTableInfo ){
  1299   1303         return 1;
  1300   1304       }
  1301   1305   
  1302   1306       zSelect = appendText(zSelect, "SELECT 'INSERT INTO ' || ", 0);
  1303         -    if( !isalpha(zTable[0]) ){
  1304         -      kk = 0;
  1305         -    }else{
  1306         -      for(kk=1; isalnum(zTable[kk]); kk++){}
  1307         -    }
  1308         -    zTmp = appendText(zTmp, zTable, zTable[kk] ? '"' : 0);
         1307  +    /* Always quote the table name, even if it appears to be pure ascii,
         1308  +    ** in case it is a keyword. Ex:  INSERT INTO "table" ... */
         1309  +    zTmp = appendText(zTmp, zTable, '"');
  1309   1310       if( zTmp ){
  1310   1311         zSelect = appendText(zSelect, zTmp, '\'');
  1311   1312       }
  1312   1313       zSelect = appendText(zSelect, " || ' VALUES(' || ", 0);
  1313   1314       rc = sqlite3_step(pTableInfo);
  1314   1315       while( rc==SQLITE_ROW ){
  1315   1316         const char *zText = (const char *)sqlite3_column_text(pTableInfo, 1);
................................................................................
  1996   1997     if( c=='n' && strncmp(azArg[0], "nullvalue", n)==0 && nArg==2 ) {
  1997   1998       sqlite3_snprintf(sizeof(p->nullvalue), p->nullvalue,
  1998   1999                        "%.*s", (int)ArraySize(p->nullvalue)-1, azArg[1]);
  1999   2000     }else
  2000   2001   
  2001   2002     if( c=='o' && strncmp(azArg[0], "output", n)==0 && nArg==2 ){
  2002   2003       if( p->out!=stdout ){
  2003         -      fclose(p->out);
         2004  +      if( p->outfile[0]=='|' ){
         2005  +        pclose(p->out);
         2006  +      }else{
         2007  +        fclose(p->out);
         2008  +      }
  2004   2009       }
  2005   2010       if( strcmp(azArg[1],"stdout")==0 ){
  2006   2011         p->out = stdout;
  2007   2012         sqlite3_snprintf(sizeof(p->outfile), p->outfile, "stdout");
         2013  +    }else if( azArg[1][0]=='|' ){
         2014  +      p->out = popen(&azArg[1][1], "w");
         2015  +      if( p->out==0 ){
         2016  +        fprintf(stderr,"Error: cannot open pipe \"%s\"\n", &azArg[1][1]);
         2017  +        p->out = stdout;
         2018  +        rc = 1;
         2019  +      }else{
         2020  +        sqlite3_snprintf(sizeof(p->outfile), p->outfile, "%s", azArg[1]);
         2021  +      }
  2008   2022       }else{
  2009   2023         p->out = fopen(azArg[1], "wb");
  2010   2024         if( p->out==0 ){
  2011   2025           fprintf(stderr,"Error: cannot write to \"%s\"\n", azArg[1]);
  2012   2026           p->out = stdout;
  2013   2027           rc = 1;
  2014   2028         } else {

Changes to src/sqlite.h.in.

  6033   6033   ** </dd>
  6034   6034   **
  6035   6035   ** [[SQLITE_DBSTATUS_CACHE_MISS]] ^(<dt>SQLITE_DBSTATUS_CACHE_MISS</dt>
  6036   6036   ** <dd>This parameter returns the number of pager cache misses that have
  6037   6037   ** occurred.)^ ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_MISS 
  6038   6038   ** is always 0.
  6039   6039   ** </dd>
         6040  +**
         6041  +** [[SQLITE_DBSTATUS_CACHE_WRITE]] ^(<dt>SQLITE_DBSTATUS_CACHE_WRITE</dt>
         6042  +** <dd>This parameter returns the number of dirty cache entries that have
         6043  +** been written to disk. Specifically, the number of pages written to the
         6044  +** wal file in wal mode databases, or the number of pages written to the
         6045  +** database file in rollback mode databases. Any pages written as part of
         6046  +** transaction rollback or database recovery operations are not included.
         6047  +** If an IO or other error occurs while writing a page to disk, the effect
         6048  +** on subsequent SQLITE_DBSTATUS_CACHE_WRITE requests is undefined). ^The
         6049  +** highwater mark associated with SQLITE_DBSTATUS_CACHE_WRITE is always 0.
         6050  +** </dd>
  6040   6051   ** </dl>
  6041   6052   */
  6042   6053   #define SQLITE_DBSTATUS_LOOKASIDE_USED       0
  6043   6054   #define SQLITE_DBSTATUS_CACHE_USED           1
  6044   6055   #define SQLITE_DBSTATUS_SCHEMA_USED          2
  6045   6056   #define SQLITE_DBSTATUS_STMT_USED            3
  6046   6057   #define SQLITE_DBSTATUS_LOOKASIDE_HIT        4
  6047   6058   #define SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE  5
  6048   6059   #define SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL  6
  6049   6060   #define SQLITE_DBSTATUS_CACHE_HIT            7
  6050   6061   #define SQLITE_DBSTATUS_CACHE_MISS           8
  6051         -#define SQLITE_DBSTATUS_MAX                  8   /* Largest defined DBSTATUS */
         6062  +#define SQLITE_DBSTATUS_CACHE_WRITE          9
         6063  +#define SQLITE_DBSTATUS_MAX                  9   /* Largest defined DBSTATUS */
  6052   6064   
  6053   6065   
  6054   6066   /*
  6055   6067   ** CAPI3REF: Prepared Statement Status
  6056   6068   **
  6057   6069   ** ^(Each prepared statement maintains various
  6058   6070   ** [SQLITE_STMTSTATUS counters] that measure the number

Changes to src/sqliteInt.h.

  1005   1005   struct FuncDestructor {
  1006   1006     int nRef;
  1007   1007     void (*xDestroy)(void *);
  1008   1008     void *pUserData;
  1009   1009   };
  1010   1010   
  1011   1011   /*
  1012         -** Possible values for FuncDef.flags
         1012  +** Possible values for FuncDef.flags.  Note that the _LENGTH and _TYPEOF
         1013  +** values must correspond to OPFLAG_LENGTHARG and OPFLAG_TYPEOFARG.  There
         1014  +** are assert() statements in the code to verify this.
  1013   1015   */
  1014   1016   #define SQLITE_FUNC_LIKE     0x01 /* Candidate for the LIKE optimization */
  1015   1017   #define SQLITE_FUNC_CASE     0x02 /* Case-sensitive LIKE-type function */
  1016   1018   #define SQLITE_FUNC_EPHEM    0x04 /* Ephemeral.  Delete with VDBE */
  1017   1019   #define SQLITE_FUNC_NEEDCOLL 0x08 /* sqlite3GetFuncCollSeq() might be called */
  1018         -#define SQLITE_FUNC_COUNT    0x20 /* Built-in count(*) aggregate */
  1019         -#define SQLITE_FUNC_COALESCE 0x40 /* Built-in coalesce() or ifnull() function */
         1020  +#define SQLITE_FUNC_COUNT    0x10 /* Built-in count(*) aggregate */
         1021  +#define SQLITE_FUNC_COALESCE 0x20 /* Built-in coalesce() or ifnull() function */
         1022  +#define SQLITE_FUNC_LENGTH   0x40 /* Built-in length() function */
         1023  +#define SQLITE_FUNC_TYPEOF   0x80 /* Built-in typeof() function */
  1020   1024   
  1021   1025   /*
  1022   1026   ** The following three macros, FUNCTION(), LIKEFUNC() and AGGREGATE() are
  1023   1027   ** used to create the initializers for the FuncDef structures.
  1024   1028   **
  1025   1029   **   FUNCTION(zName, nArg, iArg, bNC, xFunc)
  1026   1030   **     Used to create a scalar function definition of a function zName 
................................................................................
  1040   1044   **     that accepts nArg arguments and is implemented by a call to C 
  1041   1045   **     function likeFunc. Argument pArg is cast to a (void *) and made
  1042   1046   **     available as the function user-data (sqlite3_user_data()). The
  1043   1047   **     FuncDef.flags variable is set to the value passed as the flags
  1044   1048   **     parameter.
  1045   1049   */
  1046   1050   #define FUNCTION(zName, nArg, iArg, bNC, xFunc) \
  1047         -  {nArg, SQLITE_UTF8, bNC*SQLITE_FUNC_NEEDCOLL, \
         1051  +  {nArg, SQLITE_UTF8, (bNC*SQLITE_FUNC_NEEDCOLL), \
         1052  +   SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0}
         1053  +#define FUNCTION2(zName, nArg, iArg, bNC, xFunc, extraFlags) \
         1054  +  {nArg, SQLITE_UTF8, (bNC*SQLITE_FUNC_NEEDCOLL)|extraFlags, \
  1048   1055      SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0}
  1049   1056   #define STR_FUNCTION(zName, nArg, pArg, bNC, xFunc) \
  1050   1057     {nArg, SQLITE_UTF8, bNC*SQLITE_FUNC_NEEDCOLL, \
  1051   1058      pArg, 0, xFunc, 0, 0, #zName, 0, 0}
  1052   1059   #define LIKEFUNC(zName, nArg, arg, flags) \
  1053   1060     {nArg, SQLITE_UTF8, flags, (void *)arg, 0, likeFunc, 0, 0, #zName, 0, 0}
  1054   1061   #define AGGREGATE(zName, nArg, arg, nc, xStep, xFinal) \
................................................................................
  1663   1670                            ** TK_TRIGGER: 1 -> new, 0 -> old */
  1664   1671     ynVar iColumn;         /* TK_COLUMN: column index.  -1 for rowid.
  1665   1672                            ** TK_VARIABLE: variable number (always >= 1). */
  1666   1673     i16 iAgg;              /* Which entry in pAggInfo->aCol[] or ->aFunc[] */
  1667   1674     i16 iRightJoinTable;   /* If EP_FromJoin, the right table of the join */
  1668   1675     u8 flags2;             /* Second set of flags.  EP2_... */
  1669   1676     u8 op2;                /* If a TK_REGISTER, the original value of Expr.op */
         1677  +                         /* If TK_COLUMN, the value of p5 for OP_Column */
  1670   1678     AggInfo *pAggInfo;     /* Used by TK_AGG_COLUMN and TK_AGG_FUNCTION */
  1671   1679     Table *pTab;           /* Table for TK_COLUMN expressions. */
  1672   1680   #if SQLITE_MAX_EXPR_DEPTH>0
  1673   1681     int nHeight;           /* Height of the tree headed by this node */
  1674   1682   #endif
  1675   1683   };
  1676   1684   
................................................................................
  1685   1693   #define EP_VarSelect  0x0020  /* pSelect is correlated, not constant */
  1686   1694   #define EP_DblQuoted  0x0040  /* token.z was originally in "..." */
  1687   1695   #define EP_InfixFunc  0x0080  /* True for an infix function: LIKE, GLOB, etc */
  1688   1696   #define EP_ExpCollate 0x0100  /* Collating sequence specified explicitly */
  1689   1697   #define EP_FixedDest  0x0200  /* Result needed in a specific register */
  1690   1698   #define EP_IntValue   0x0400  /* Integer value contained in u.iValue */
  1691   1699   #define EP_xIsSelect  0x0800  /* x.pSelect is valid (otherwise x.pList is) */
  1692         -#define EP_Hint       0x1000  /* Optimizer hint. Not required for correctness */
         1700  +#define EP_Hint       0x1000  /* Not used */
  1693   1701   #define EP_Reduced    0x2000  /* Expr struct is EXPR_REDUCEDSIZE bytes only */
  1694   1702   #define EP_TokenOnly  0x4000  /* Expr struct is EXPR_TOKENONLYSIZE bytes only */
  1695   1703   #define EP_Static     0x8000  /* Held in memory not obtained from malloc() */
  1696   1704   
  1697   1705   /*
  1698   1706   ** The following are the meanings of bits in the Expr.flags2 field.
  1699   1707   */
................................................................................
  2272   2280   */
  2273   2281   struct AuthContext {
  2274   2282     const char *zAuthContext;   /* Put saved Parse.zAuthContext here */
  2275   2283     Parse *pParse;              /* The Parse structure */
  2276   2284   };
  2277   2285   
  2278   2286   /*
  2279         -** Bitfield flags for P5 value in OP_Insert and OP_Delete
         2287  +** Bitfield flags for P5 value in various opcodes.
  2280   2288   */
  2281   2289   #define OPFLAG_NCHANGE       0x01    /* Set to update db->nChange */
  2282   2290   #define OPFLAG_LASTROWID     0x02    /* Set to update db->lastRowid */
  2283   2291   #define OPFLAG_ISUPDATE      0x04    /* This OP_Insert is an sql UPDATE */
  2284   2292   #define OPFLAG_APPEND        0x08    /* This is likely to be an append */
  2285   2293   #define OPFLAG_USESEEKRESULT 0x10    /* Try to avoid a seek in BtreeInsert() */
  2286   2294   #define OPFLAG_CLEARCACHE    0x20    /* Clear pseudo-table cache in OP_Column */
         2295  +#define OPFLAG_LENGTHARG     0x40    /* OP_Column only used for length() */
         2296  +#define OPFLAG_TYPEOFARG     0x80    /* OP_Column only used for typeof() */
  2287   2297   
  2288   2298   /*
  2289   2299    * Each trigger present in the database schema is stored as an instance of
  2290   2300    * struct Trigger. 
  2291   2301    *
  2292   2302    * Pointers to instances of struct Trigger are stored in two ways.
  2293   2303    * 1. In the "trigHash" hash table (part of the sqlite3* that represents the 
................................................................................
  2763   2773   #if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY)
  2764   2774   Expr *sqlite3LimitWhere(Parse *, SrcList *, Expr *, ExprList *, Expr *, Expr *, char *);
  2765   2775   #endif
  2766   2776   void sqlite3DeleteFrom(Parse*, SrcList*, Expr*);
  2767   2777   void sqlite3Update(Parse*, SrcList*, ExprList*, Expr*, int);
  2768   2778   WhereInfo *sqlite3WhereBegin(Parse*, SrcList*, Expr*, ExprList**,ExprList*,u16);
  2769   2779   void sqlite3WhereEnd(WhereInfo*);
  2770         -int sqlite3ExprCodeGetColumn(Parse*, Table*, int, int, int);
         2780  +int sqlite3ExprCodeGetColumn(Parse*, Table*, int, int, int, u8);
  2771   2781   void sqlite3ExprCodeGetColumnOfTable(Vdbe*, Table*, int, int, int);
  2772   2782   void sqlite3ExprCodeMove(Parse*, int, int, int);
  2773   2783   void sqlite3ExprCodeCopy(Parse*, int, int, int);
  2774   2784   void sqlite3ExprCacheStore(Parse*, int, int, int);
  2775   2785   void sqlite3ExprCachePush(Parse*);
  2776   2786   void sqlite3ExprCachePop(Parse*, int);
  2777   2787   void sqlite3ExprCacheRemove(Parse*, int, int);

Changes to src/status.c.

   220    220   
   221    221       /*
   222    222       ** Set *pCurrent to the total cache hits or misses encountered by all
   223    223       ** pagers the database handle is connected to. *pHighwater is always set 
   224    224       ** to zero.
   225    225       */
   226    226       case SQLITE_DBSTATUS_CACHE_HIT:
   227         -    case SQLITE_DBSTATUS_CACHE_MISS: {
          227  +    case SQLITE_DBSTATUS_CACHE_MISS:
          228  +    case SQLITE_DBSTATUS_CACHE_WRITE:{
   228    229         int i;
   229    230         int nRet = 0;
   230    231         assert( SQLITE_DBSTATUS_CACHE_MISS==SQLITE_DBSTATUS_CACHE_HIT+1 );
          232  +      assert( SQLITE_DBSTATUS_CACHE_WRITE==SQLITE_DBSTATUS_CACHE_HIT+2 );
   231    233   
   232    234         for(i=0; i<db->nDb; i++){
   233    235           if( db->aDb[i].pBt ){
   234    236             Pager *pPager = sqlite3BtreePager(db->aDb[i].pBt);
   235    237             sqlite3PagerCacheStat(pPager, op, resetFlag, &nRet);
   236    238           }
   237    239         }

Changes to src/tclsqlite.c.

  3104   3104     */
  3105   3105     Tcl_CreateObjCommand(interp, "sqlite", (Tcl_ObjCmdProc*)DbMain, 0, 0);
  3106   3106   #endif
  3107   3107   
  3108   3108     return TCL_OK;
  3109   3109   }
  3110   3110   EXTERN int Tclsqlite3_Init(Tcl_Interp *interp){ return Sqlite3_Init(interp); }
  3111         -EXTERN int Sqlite3_SafeInit(Tcl_Interp *interp){ return TCL_OK; }
  3112         -EXTERN int Tclsqlite3_SafeInit(Tcl_Interp *interp){ return TCL_OK; }
  3113   3111   EXTERN int Sqlite3_Unload(Tcl_Interp *interp, int flags){ return TCL_OK; }
  3114   3112   EXTERN int Tclsqlite3_Unload(Tcl_Interp *interp, int flags){ return TCL_OK; }
  3115         -EXTERN int Sqlite3_SafeUnload(Tcl_Interp *interp, int flags){ return TCL_OK; }
  3116         -EXTERN int Tclsqlite3_SafeUnload(Tcl_Interp *interp, int flags){ return TCL_OK;}
  3117   3113   
         3114  +/* Because it accesses the file-system and uses persistent state, SQLite
         3115  +** is not considered appropriate for safe interpreters.  Hence, we deliberately
         3116  +** omit the _SafeInit() interfaces.
         3117  +*/
  3118   3118   
  3119   3119   #ifndef SQLITE_3_SUFFIX_ONLY
  3120   3120   int Sqlite_Init(Tcl_Interp *interp){ return Sqlite3_Init(interp); }
  3121   3121   int Tclsqlite_Init(Tcl_Interp *interp){ return Sqlite3_Init(interp); }
  3122         -int Sqlite_SafeInit(Tcl_Interp *interp){ return TCL_OK; }
  3123         -int Tclsqlite_SafeInit(Tcl_Interp *interp){ return TCL_OK; }
  3124   3122   int Sqlite_Unload(Tcl_Interp *interp, int flags){ return TCL_OK; }
  3125   3123   int Tclsqlite_Unload(Tcl_Interp *interp, int flags){ return TCL_OK; }
  3126         -int Sqlite_SafeUnload(Tcl_Interp *interp, int flags){ return TCL_OK; }
  3127         -int Tclsqlite_SafeUnload(Tcl_Interp *interp, int flags){ return TCL_OK;}
  3128   3124   #endif
  3129   3125   
  3130   3126   #ifdef TCLSH
  3131   3127   /*****************************************************************************
  3132   3128   ** All of the code that follows is used to build standalone TCL interpreters
  3133   3129   ** that are statically linked with SQLite.  Enable these by compiling
  3134   3130   ** with -DTCLSH=n where n can be 1 or 2.  An n of 1 generates a standard

Changes to src/test8.c.

   827    827   
   828    828       pConstraint = &pIdxInfo->aConstraint[ii];
   829    829       pUsage = &pIdxInfo->aConstraintUsage[ii];
   830    830   
   831    831       if( !isIgnoreUsable && !pConstraint->usable ) continue;
   832    832   
   833    833       iCol = pConstraint->iColumn;
   834         -    if( pVtab->aIndex[iCol] || iCol<0 ){
   835         -      char *zCol = pVtab->aCol[iCol];
          834  +    if( iCol<0 || pVtab->aIndex[iCol] ){
          835  +      char *zCol = iCol>=0 ? pVtab->aCol[iCol] : "rowid";
   836    836         char *zOp = 0;
   837    837         useIdx = 1;
   838         -      if( iCol<0 ){
   839         -        zCol = "rowid";
   840         -      }
   841    838         switch( pConstraint->op ){
   842    839           case SQLITE_INDEX_CONSTRAINT_EQ:
   843    840             zOp = "="; break;
   844    841           case SQLITE_INDEX_CONSTRAINT_LT:
   845    842             zOp = "<"; break;
   846    843           case SQLITE_INDEX_CONSTRAINT_GT:
   847    844             zOp = ">"; break;
................................................................................
   866    863       }
   867    864     }
   868    865   
   869    866     /* If there is only one term in the ORDER BY clause, and it is
   870    867     ** on a column that this virtual table has an index for, then consume 
   871    868     ** the ORDER BY clause.
   872    869     */
   873         -  if( pIdxInfo->nOrderBy==1 && pVtab->aIndex[pIdxInfo->aOrderBy->iColumn] ){
          870  +  if( pIdxInfo->nOrderBy==1 && (
          871  +        pIdxInfo->aOrderBy->iColumn<0 ||
          872  +        pVtab->aIndex[pIdxInfo->aOrderBy->iColumn]) ){
   874    873       int iCol = pIdxInfo->aOrderBy->iColumn;
   875         -    char *zCol = pVtab->aCol[iCol];
          874  +    char *zCol = iCol>=0 ? pVtab->aCol[iCol] : "rowid";
   876    875       char *zDir = pIdxInfo->aOrderBy->desc?"DESC":"ASC";
   877         -    if( iCol<0 ){
   878         -      zCol = "rowid";
   879         -    }
   880    876       zNew = sqlite3_mprintf(" ORDER BY %s %s", zCol, zDir);
   881    877       string_concat(&zQuery, zNew, 1, &rc);
   882    878       pIdxInfo->orderByConsumed = 1;
   883    879     }
   884    880   
   885    881     appendToEchoModule(pVtab->interp, "xBestIndex");;
   886    882     appendToEchoModule(pVtab->interp, zQuery);

Changes to src/test_malloc.c.

  1319   1319       { "CACHE_USED",          SQLITE_DBSTATUS_CACHE_USED          },
  1320   1320       { "SCHEMA_USED",         SQLITE_DBSTATUS_SCHEMA_USED         },
  1321   1321       { "STMT_USED",           SQLITE_DBSTATUS_STMT_USED           },
  1322   1322       { "LOOKASIDE_HIT",       SQLITE_DBSTATUS_LOOKASIDE_HIT       },
  1323   1323       { "LOOKASIDE_MISS_SIZE", SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE },
  1324   1324       { "LOOKASIDE_MISS_FULL", SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL },
  1325   1325       { "CACHE_HIT",           SQLITE_DBSTATUS_CACHE_HIT           },
  1326         -    { "CACHE_MISS",          SQLITE_DBSTATUS_CACHE_MISS          }
         1326  +    { "CACHE_MISS",          SQLITE_DBSTATUS_CACHE_MISS          },
         1327  +    { "CACHE_WRITE",         SQLITE_DBSTATUS_CACHE_WRITE         }
  1327   1328     };
  1328   1329     Tcl_Obj *pResult;
  1329   1330     if( objc!=4 ){
  1330   1331       Tcl_WrongNumArgs(interp, 1, objv, "DB PARAMETER RESETFLAG");
  1331   1332       return TCL_ERROR;
  1332   1333     }
  1333   1334     if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR;

Changes to src/vdbe.c.

  2123   2123   ** if the P4 argument is a P4_MEM use the value of the P4 argument as
  2124   2124   ** the result.
  2125   2125   **
  2126   2126   ** If the OPFLAG_CLEARCACHE bit is set on P5 and P1 is a pseudo-table cursor,
  2127   2127   ** then the cache of the cursor is reset prior to extracting the column.
  2128   2128   ** The first OP_Column against a pseudo-table after the value of the content
  2129   2129   ** register has changed should have this bit set.
         2130  +**
         2131  +** If the OPFLAG_LENGTHARG and OPFLAG_TYPEOFARG bits are set on P5 when
         2132  +** the result is guaranteed to only be used as the argument of a length()
         2133  +** or typeof() function, respectively.  The loading of large blobs can be
         2134  +** skipped for length() and all content loading can be skipped for typeof().
  2130   2135   */
  2131   2136   case OP_Column: {
  2132   2137     u32 payloadSize;   /* Number of bytes in the record */
  2133   2138     i64 payloadSize64; /* Number of bytes in the record */
  2134   2139     int p1;            /* P1 value of the opcode */
  2135   2140     int p2;            /* column number to retrieve */
  2136   2141     VdbeCursor *pC;    /* The VDBE cursor */
................................................................................
  2263   2268         if( payloadSize <= (u32)avail ){
  2264   2269           zRec = zData;
  2265   2270           pC->aRow = (u8*)zData;
  2266   2271         }else{
  2267   2272           pC->aRow = 0;
  2268   2273         }
  2269   2274       }
  2270         -    /* The following assert is true in all cases accept when
         2275  +    /* The following assert is true in all cases except when
  2271   2276       ** the database file has been corrupted externally.
  2272   2277       **    assert( zRec!=0 || avail>=payloadSize || avail>=9 ); */
  2273   2278       szHdr = getVarint32((u8*)zData, offset);
  2274   2279   
  2275   2280       /* Make sure a corrupt database has not given us an oversize header.
  2276   2281       ** Do this now to avoid an oversize memory allocation.
  2277   2282       **
................................................................................
  2338   2343           szField = sqlite3VdbeSerialTypeLen(t);
  2339   2344           offset += szField;
  2340   2345           if( offset<szField ){  /* True if offset overflows */
  2341   2346             zIdx = &zEndHdr[1];  /* Forces SQLITE_CORRUPT return below */
  2342   2347             break;
  2343   2348           }
  2344   2349         }else{
  2345         -        /* If i is less that nField, then there are less fields in this
         2350  +        /* If i is less that nField, then there are fewer fields in this
  2346   2351           ** record than SetNumColumns indicated there are columns in the
  2347   2352           ** table. Set the offset for any extra columns not present in
  2348         -        ** the record to 0. This tells code below to store a NULL
  2349         -        ** instead of deserializing a value from the record.
         2353  +        ** the record to 0. This tells code below to store the default value
         2354  +        ** for the column instead of deserializing a value from the record.
  2350   2355           */
  2351   2356           aOffset[i] = 0;
  2352   2357         }
  2353   2358       }
  2354   2359       sqlite3VdbeMemRelease(&sMem);
  2355   2360       sMem.flags = MEM_Null;
  2356   2361   
................................................................................
  2372   2377     ** then there are not enough fields in the record to satisfy the
  2373   2378     ** request.  In this case, set the value NULL or to P4 if P4 is
  2374   2379     ** a pointer to a Mem object.
  2375   2380     */
  2376   2381     if( aOffset[p2] ){
  2377   2382       assert( rc==SQLITE_OK );
  2378   2383       if( zRec ){
         2384  +      /* This is the common case where the whole row fits on a single page */
  2379   2385         VdbeMemRelease(pDest);
  2380   2386         sqlite3VdbeSerialGet((u8 *)&zRec[aOffset[p2]], aType[p2], pDest);
  2381   2387       }else{
  2382         -      len = sqlite3VdbeSerialTypeLen(aType[p2]);
  2383         -      sqlite3VdbeMemMove(&sMem, pDest);
  2384         -      rc = sqlite3VdbeMemFromBtree(pCrsr, aOffset[p2], len, pC->isIndex, &sMem);
  2385         -      if( rc!=SQLITE_OK ){
  2386         -        goto op_column_out;
         2388  +      /* This branch happens only when the row overflows onto multiple pages */
         2389  +      t = aType[p2];
         2390  +      if( (pOp->p5 & (OPFLAG_LENGTHARG|OPFLAG_TYPEOFARG))!=0
         2391  +       && ((t>=12 && (t&1)==0) || (pOp->p5 & OPFLAG_TYPEOFARG)!=0)
         2392  +      ){
         2393  +        /* Content is irrelevant for the typeof() function and for
         2394  +        ** the length(X) function if X is a blob.  So we might as well use
         2395  +        ** bogus content rather than reading content from disk.  NULL works
         2396  +        ** for text and blob and whatever is in the payloadSize64 variable
         2397  +        ** will work for everything else. */
         2398  +        zData = t<12 ? (char*)&payloadSize64 : 0;
         2399  +      }else{
         2400  +        len = sqlite3VdbeSerialTypeLen(t);
         2401  +        sqlite3VdbeMemMove(&sMem, pDest);
         2402  +        rc = sqlite3VdbeMemFromBtree(pCrsr, aOffset[p2], len,  pC->isIndex,
         2403  +                                     &sMem);
         2404  +        if( rc!=SQLITE_OK ){
         2405  +          goto op_column_out;
         2406  +        }
         2407  +        zData = sMem.z;
  2387   2408         }
  2388         -      zData = sMem.z;
  2389         -      sqlite3VdbeSerialGet((u8*)zData, aType[p2], pDest);
         2409  +      sqlite3VdbeSerialGet((u8*)zData, t, pDest);
  2390   2410       }
  2391   2411       pDest->enc = encoding;
  2392   2412     }else{
  2393   2413       if( pOp->p4type==P4_MEM ){
  2394   2414         sqlite3VdbeMemShallowCopy(pDest, pOp->p4.pMem, MEM_Static);
  2395   2415       }else{
  2396   2416         MemSetTypeFlag(pDest, MEM_Null);

Changes to src/where.c.

   682    682       return 0;
   683    683     }
   684    684   #ifdef SQLITE_EBCDIC
   685    685     if( *pnoCase ) return 0;
   686    686   #endif
   687    687     pList = pExpr->x.pList;
   688    688     pLeft = pList->a[1].pExpr;
   689         -  if( pLeft->op!=TK_COLUMN || sqlite3ExprAffinity(pLeft)!=SQLITE_AFF_TEXT ){
          689  +  if( pLeft->op!=TK_COLUMN 
          690  +   || sqlite3ExprAffinity(pLeft)!=SQLITE_AFF_TEXT 
          691  +   || IsVirtual(pLeft->pTab)
          692  +  ){
   690    693       /* IMP: R-02065-49465 The left-hand side of the LIKE or GLOB operator must
   691    694       ** be the name of an indexed column with TEXT affinity. */
   692    695       return 0;
   693    696     }
   694    697     assert( pLeft->iColumn!=(-1) ); /* Because IPK never has AFF_TEXT */
   695    698   
   696    699     pRight = pList->a[0].pExpr;
................................................................................
  4379   4382             explainOneScan(
  4380   4383                 pParse, pOrTab, &pSubWInfo->a[0], iLevel, pLevel->iFrom, 0
  4381   4384             );
  4382   4385             if( (wctrlFlags & WHERE_DUPLICATES_OK)==0 ){
  4383   4386               int iSet = ((ii==pOrWc->nTerm-1)?-1:ii);
  4384   4387               int r;
  4385   4388               r = sqlite3ExprCodeGetColumn(pParse, pTabItem->pTab, -1, iCur, 
  4386         -                                         regRowid);
         4389  +                                         regRowid, 0);
  4387   4390               sqlite3VdbeAddOp4Int(v, OP_RowSetTest, regRowset,
  4388   4391                                    sqlite3VdbeCurrentAddr(v)+2, r, iSet);
  4389   4392             }
  4390   4393             sqlite3VdbeAddOp2(v, OP_Gosub, regReturn, iLoopBody);
  4391   4394   
  4392   4395             /* The pSubWInfo->untestedTerms flag means that this OR term
  4393   4396             ** contained one or more AND term from a notReady table.  The

Changes to test/backcompat.test.

    23     23   # for documentation of the available commands.
    24     24   #
    25     25   
    26     26   set testdir [file dirname $argv0]
    27     27   source $testdir/tester.tcl
    28     28   source $testdir/lock_common.tcl
    29     29   source $testdir/malloc_common.tcl
           30  +source $testdir/bc_common.tcl
    30     31   db close
    31     32   
    32         -# Search for binaries to test against. Any executable files that match
    33         -# our naming convention are assumed to be testfixture binaries to test
    34         -# against.
    35         -#
    36         -set binaries [list]
    37         -set pattern "[file tail [info nameofexec]]?*"
    38         -if {$tcl_platform(platform)=="windows"} {
    39         -  set pattern [string map {\.exe {}} $pattern]
    40         -}
    41         -foreach file [glob -nocomplain $pattern] {
    42         -  if {[file executable $file] && [file isfile $file]} {lappend binaries $file}
    43         -}
    44         -if {[llength $binaries]==0} {
    45         -  puts "WARNING: No historical binaries to test against."
    46         -  puts "WARNING: No backwards-compatibility tests have been run."
           33  +if {"" == [bc_find_binaries backcompat.test]} {
    47     34     finish_test
    48     35     return
    49     36   }
    50         -proc get_version {binary} {
    51         -  set chan [launch_testfixture $binary]
    52         -  set v [testfixture $chan { sqlite3 -version }]
    53         -  close $chan
    54         -  set v
    55         -}
    56         -foreach bin $binaries {
    57         -  puts -nonewline "Testing against $bin - "
    58         -  flush stdout
    59         -  puts "version [get_version $bin]"
    60         -}
    61     37   
    62     38   proc do_backcompat_test {rv bin1 bin2 script} {
    63     39   
    64     40     forcedelete test.db
    65     41   
    66     42     if {$bin1 != ""} { set ::bc_chan1 [launch_testfixture $bin1] }
    67     43     set ::bc_chan2 [launch_testfixture $bin2]
................................................................................
    89     65     catch { close $::bc_chan2 }
    90     66     catch { close $::bc_chan1 }
    91     67   }
    92     68   
    93     69   array set ::incompatible [list]
    94     70   proc do_allbackcompat_test {script} {
    95     71   
    96         -  foreach bin $::binaries {
           72  +  foreach bin $::BC(binaries) {
    97     73       set nErr [set_test_counter errors]
    98     74       foreach dir {0 1} {
    99     75   
   100     76         set bintag [string map {testfixture {}} $bin]
   101     77         set bintag [string map {\.exe {}} $bintag]
   102     78         if {$bintag == ""} {set bintag self}
   103     79         set ::bcname ".$bintag.$dir."

Added test/bc_common.tcl.

            1  +
            2  +
            3  +
            4  +proc bc_find_binaries {zCaption} {
            5  +  # Search for binaries to test against. Any executable files that match
            6  +  # our naming convention are assumed to be testfixture binaries to test
            7  +  # against.
            8  +  #
            9  +  set binaries [list]
           10  +  set pattern "[file tail [info nameofexec]]?*"
           11  +  if {$::tcl_platform(platform)=="windows"} {
           12  +    set pattern [string map {\.exe {}} $pattern]
           13  +  }
           14  +  foreach file [glob -nocomplain $pattern] {
           15  +    if {[file executable $file] && [file isfile $file]} {lappend binaries $file}
           16  +  }
           17  +
           18  +  if {[llength $binaries]==0} {
           19  +    puts "WARNING: No historical binaries to test against."
           20  +    puts "WARNING: Omitting backwards-compatibility tests"
           21  +  }
           22  +
           23  +  foreach bin $binaries {
           24  +    puts -nonewline "Testing against $bin - "
           25  +    flush stdout
           26  +    puts "version [get_version $bin]"
           27  +  }
           28  +
           29  +  set ::BC(binaries) $binaries
           30  +  return $binaries
           31  +}
           32  +
           33  +proc get_version {binary} {
           34  +  set chan [launch_testfixture $binary]
           35  +  set v [testfixture $chan { sqlite3 -version }]
           36  +  close $chan
           37  +  set v
           38  +}
           39  +
           40  +proc do_bc_test {bin script} {
           41  +
           42  +  forcedelete test.db
           43  +  set ::bc_chan [launch_testfixture $bin]
           44  +
           45  +  proc code1 {tcl} { uplevel #0 $tcl }
           46  +  proc code2 {tcl} { testfixture $::bc_chan $tcl }
           47  +  proc sql1 sql { code1 [list db eval $sql] }
           48  +  proc sql2 sql { code2 [list db eval $sql] }
           49  +
           50  +  code1 { sqlite3 db test.db }
           51  +  code2 { sqlite3 db test.db }
           52  +
           53  +  set bintag [string map {testfixture {}} $bin]
           54  +  set bintag [string map {\.exe {}} $bintag]
           55  +  if {$bintag == ""} {set bintag self}
           56  +  set saved_prefix $::testprefix
           57  +  append ::testprefix ".$bintag"
           58  +
           59  +  uplevel $script
           60  +
           61  +  set ::testprefix $saved_prefix
           62  +
           63  +  catch { code1 { db close } }
           64  +  catch { code2 { db close } }
           65  +  catch { close $::bc_chan }
           66  +}
           67  +
           68  +proc do_all_bc_test {script} {
           69  +  foreach bin $::BC(binaries) {
           70  +    uplevel [list do_bc_test $bin $script]
           71  +  }
           72  +}

Changes to test/dbstatus2.test.

     5      5   #
     6      6   #    May you do good and not evil.
     7      7   #    May you find forgiveness for yourself and forgive others.
     8      8   #    May you share freely, never taking more than you give.
     9      9   #
    10     10   #***********************************************************************
    11     11   #
    12         -# Tests for the sqlite3_stmt_status() function
           12  +# Tests for the sqlite3_db_status() function
    13     13   #
    14     14   
    15     15   set testdir [file dirname $argv0]
    16     16   source $testdir/tester.tcl
    17     17   
    18     18   set ::testprefix dbstatus2
    19     19   
................................................................................
    28     28   }
    29     29   
    30     30   proc db_hit_miss {db {reset 0}} {
    31     31     set nHit  [sqlite3_db_status $db CACHE_HIT $reset]
    32     32     set nMiss [sqlite3_db_status $db CACHE_MISS $reset]
    33     33     list $nHit $nMiss
    34     34   }
           35  +
           36  +proc db_write {db {reset 0}} {
           37  +  sqlite3_db_status $db CACHE_WRITE $reset
           38  +}
    35     39   
    36     40   do_test 1.1 {
    37     41     db close
    38     42     sqlite3 db test.db
    39     43     expr {[file size test.db] / 1024}
    40     44   } 6
    41     45   
................................................................................
    68     72     set len [string length [read $fd]]
    69     73     close $fd
    70     74     set len
    71     75   } 600
    72     76   do_test 1.8 { sqlite3_db_status db CACHE_HIT  0 } {0 2 0}
    73     77   do_test 1.9 { sqlite3_db_status db CACHE_MISS 0 } {0 1 0}
    74     78   
           79  +do_test 2.1 { db_write db } {0 0 0}
           80  +do_test 2.2 { 
           81  +  execsql { INSERT INTO t1 VALUES(4, randomblob(600)) }
           82  +  db_write db
           83  +} {0 4 0}
           84  +do_test 2.3 { db_write db 1 } {0 4 0}
           85  +do_test 2.4 { db_write db 0 } {0 0 0}
           86  +do_test 2.5 { db_write db 1 } {0 0 0}
           87  +
           88  +do_test 2.6 { 
           89  +  execsql { PRAGMA journal_mode = WAL }
           90  +  db_write db 1
           91  +} {0 1 0}
           92  +do_test 2.7 { 
           93  +  execsql { INSERT INTO t1 VALUES(5, randomblob(600)) }
           94  +  db_write db
           95  +} {0 4 0}
           96  +do_test 2.8 { db_write db 1 } {0 4 0}
           97  +do_test 2.9 { db_write db 0 } {0 0 0}
    75     98    
    76     99   finish_test

Changes to test/fts3_common.tcl.

     9      9   #
    10     10   #***********************************************************************
    11     11   #
    12     12   # This file contains common code used the fts3 tests. At one point
    13     13   # equivalent functionality was implemented in C code. But it is easier
    14     14   # to use Tcl.
    15     15   #
           16  +
           17  +#-------------------------------------------------------------------------
           18  +# INSTRUCTIONS
           19  +#
           20  +# The following commands are available:
           21  +#
           22  +#   fts3_build_db_1 N
           23  +#     Using database handle [db] create an FTS4 table named t1 and populate
           24  +#     it with N rows of data. N must be less than 10,000. Refer to the
           25  +#     header comments above the proc implementation below for details.
           26  +#
           27  +#   fts3_build_db_2 N
           28  +#     Using database handle [db] create an FTS4 table named t2 and populate
           29  +#     it with N rows of data. N must be less than 100,000. Refer to the
           30  +#     header comments above the proc implementation below for details.
           31  +#
           32  +#   fts3_integrity_check TBL
           33  +#     TBL must be an FTS table in the database currently opened by handle
           34  +#     [db]. This proc loads and tokenizes all documents within the table,
           35  +#     then checks that the current contents of the FTS index matches the
           36  +#     results.
           37  +#
           38  +#   fts3_terms TBL WHERE
           39  +#     Todo.
           40  +#
           41  +#   fts3_doclist TBL TERM WHERE
           42  +#     Todo.
           43  +#
           44  +#
           45  +#
           46  +
           47  +#-------------------------------------------------------------------------
           48  +# USAGE: fts3_build_db_1 SWITCHES N
           49  +#
           50  +# Build a sample FTS table in the database opened by database connection 
           51  +# [db]. The name of the new table is "t1".
           52  +#
           53  +proc fts3_build_db_1 {args} {
           54  +
           55  +  set default(-module) fts4
           56  +
           57  +  set nArg [llength $args]
           58  +  if {($nArg%2)==0} {
           59  +    error "wrong # args: should be \"fts3_build_db_1 ?switches? n\""
           60  +  }
           61  +
           62  +  set n [lindex $args [expr $nArg-1]]
           63  +  array set opts [array get default]
           64  +  array set opts [lrange $args 0 [expr $nArg-2]]
           65  +  foreach k [array names opts] {
           66  +    if {0==[info exists default($k)]} { error "unknown option: $k" }
           67  +  }
           68  +
           69  +  if {$n > 10000} {error "n must be <= 10000"}
           70  +  db eval "CREATE VIRTUAL TABLE t1 USING $opts(-module) (x, y)"
           71  +
           72  +  set xwords [list zero one two three four five six seven eight nine ten]
           73  +  set ywords [list alpha beta gamma delta epsilon zeta eta theta iota kappa]
           74  +
           75  +  for {set i 0} {$i < $n} {incr i} {
           76  +    set x ""
           77  +    set y ""
           78  +
           79  +    set x [list]
           80  +    lappend x [lindex $xwords [expr ($i / 1000) % 10]]
           81  +    lappend x [lindex $xwords [expr ($i / 100)  % 10]]
           82  +    lappend x [lindex $xwords [expr ($i / 10)   % 10]]
           83  +    lappend x [lindex $xwords [expr ($i / 1)   % 10]]
           84  +
           85  +    set y [list]
           86  +    lappend y [lindex $ywords [expr ($i / 1000) % 10]]
           87  +    lappend y [lindex $ywords [expr ($i / 100)  % 10]]
           88  +    lappend y [lindex $ywords [expr ($i / 10)   % 10]]
           89  +    lappend y [lindex $ywords [expr ($i / 1)   % 10]]
           90  +
           91  +    db eval { INSERT INTO t1(docid, x, y) VALUES($i, $x, $y) }
           92  +  }
           93  +}
           94  +
           95  +#-------------------------------------------------------------------------
           96  +# USAGE: fts3_build_db_2 N ARGS
           97  +#
           98  +# Build a sample FTS table in the database opened by database connection 
           99  +# [db]. The name of the new table is "t2".
          100  +#
          101  +proc fts3_build_db_2 {args} {
          102  +
          103  +  set default(-module) fts4
          104  +  set default(-extra)   ""
          105  +
          106  +  set nArg [llength $args]
          107  +  if {($nArg%2)==0} {
          108  +    error "wrong # args: should be \"fts3_build_db_1 ?switches? n\""
          109  +  }
          110  +
          111  +  set n [lindex $args [expr $nArg-1]]
          112  +  array set opts [array get default]
          113  +  array set opts [lrange $args 0 [expr $nArg-2]]
          114  +  foreach k [array names opts] {
          115  +    if {0==[info exists default($k)]} { error "unknown option: $k" }
          116  +  }
          117  +
          118  +  if {$n > 100000} {error "n must be <= 100000"}
          119  +
          120  +  set sql "CREATE VIRTUAL TABLE t2 USING $opts(-module) (content"
          121  +  if {$opts(-extra) != ""} {
          122  +    append sql ", " $opts(-extra)
          123  +  }
          124  +  append sql ")"
          125  +  db eval $sql
          126  +
          127  +  set chars [list a b c d e f g h  i j k l m n o p  q r s t u v w x  y z ""]
          128  +
          129  +  for {set i 0} {$i < $n} {incr i} {
          130  +    set word ""
          131  +    set nChar [llength $chars]
          132  +    append word [lindex $chars [expr {($i / 1)   % $nChar}]]
          133  +    append word [lindex $chars [expr {($i / $nChar)  % $nChar}]]
          134  +    append word [lindex $chars [expr {($i / ($nChar*$nChar)) % $nChar}]]
          135  +
          136  +    db eval { INSERT INTO t2(docid, content) VALUES($i, $word) }
          137  +  }
          138  +}
    16    139   
    17    140   #-------------------------------------------------------------------------
    18    141   # USAGE: fts3_integrity_check TBL
    19    142   #
    20    143   # This proc is used to verify that the full-text index is consistent with
    21    144   # the contents of the fts3 table. In other words, it checks that the
    22    145   # data in the %_contents table matches that in the %_segdir and %_segments 
................................................................................
    42    165   #      
    43    166   #
    44    167   proc fts3_integrity_check {tbl} {
    45    168   
    46    169     fts3_read2 $tbl 1 A
    47    170   
    48    171     foreach zTerm [array names A] {
          172  +    #puts $zTerm
    49    173       foreach doclist $A($zTerm) {
    50    174         set docid 0
    51    175         while {[string length $doclist]>0} {
    52    176           set iCol 0
    53    177           set iPos 0
    54    178           set lPos [list]
    55    179           set lCol [list]
................................................................................
    93    217         set sql {SELECT fts3_tokenizer_test('simple', $c)}
    94    218   
    95    219         foreach {pos term dummy} [db one $sql] {
    96    220           if {![info exists C($iDoc,$iCol,$pos)]} {
    97    221             set es "Error at docid=$iDoc col=$iCol pos=$pos. Index is missing"
    98    222             lappend errors $es
    99    223           } else {
   100         -          if {$C($iDoc,$iCol,$pos) != "$term"} {
          224  +          if {[string compare $C($iDoc,$iCol,$pos) $term]} {
   101    225               set    es "Error at docid=$iDoc col=$iCol pos=$pos. Index "
   102    226               append es "has \"$C($iDoc,$iCol,$pos)\", document has \"$term\""
   103    227               lappend errors $es
   104    228             }
   105    229             unset C($iDoc,$iCol,$pos)
   106    230           }
   107    231         }
................................................................................
   229    353   
   230    354     while {[string length $blob] > 0} {
   231    355       set nPrefix [gobble_varint blob]
   232    356       set nSuffix [gobble_varint blob]
   233    357   
   234    358       set zTerm [string range $zPrev 0 [expr $nPrefix-1]]
   235    359       append zTerm [gobble_string blob $nSuffix]
   236         -    set doclist [gobble_string blob [gobble_varint blob]]
          360  +    set nDoclist [gobble_varint blob]
          361  +    set doclist [gobble_string blob $nDoclist]
   237    362   
   238    363       lappend terms $zTerm $doclist
   239    364       set zPrev $zTerm
   240    365     }
   241    366   
   242    367     return $terms
   243    368   }
................................................................................
   245    370   proc fts3_read2 {tbl where varname} {
   246    371     upvar $varname a
   247    372     array unset a
   248    373     db eval " SELECT start_block, leaves_end_block, root 
   249    374               FROM ${tbl}_segdir WHERE $where
   250    375               ORDER BY level ASC, idx DESC
   251    376     " {
   252         -    if {$start_block == 0} {
          377  +    set c 0
          378  +    binary scan $root c c
          379  +    if {$c==0} {
   253    380         foreach {t d} [fts3_readleaf $root] { lappend a($t) $d }
   254    381       } else {
   255    382         db eval " SELECT block 
   256    383                   FROM ${tbl}_segments 
   257    384                   WHERE blockid>=$start_block AND blockid<=$leaves_end_block
   258    385                   ORDER BY blockid
   259    386         " {
   260    387           foreach {t d} [fts3_readleaf $block] { lappend a($t) $d }
   261         -
   262    388         }
   263    389       }
   264    390     }
   265    391   }
   266    392   
   267    393   proc fts3_read {tbl where varname} {
   268    394     upvar $varname a

Added test/fts4check.test.

            1  +# 2012 March 26
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +# This file implements regression tests for SQLite library. The
           12  +# focus of this script is testing the FTS 'integrity-check' function,
           13  +# used to check if the current FTS index accurately reflects the content
           14  +# of the table.
           15  +#
           16  +
           17  +set testdir [file dirname $argv0]
           18  +source $testdir/tester.tcl
           19  +source $testdir/fts3_common.tcl
           20  +set ::testprefix fts4check
           21  +
           22  +# If SQLITE_ENABLE_FTS3 is defined, omit this file.
           23  +ifcapable !fts3 {
           24  +  finish_test
           25  +  return
           26  +}
           27  +
           28  +# Run the integrity-check on FTS table $tbl using database handle $db. If
           29  +# the integrity-check passes, return "ok". Otherwise, throw an exception.
           30  +#
           31  +proc fts_integrity {db tbl} {
           32  +  $db eval "INSERT INTO $tbl ($tbl) VALUES('integrity-check')"
           33  +  return "ok"
           34  +}
           35  +
           36  +#-------------------------------------------------------------------------
           37  +# Test cases 1.*
           38  +#
           39  +#   1.0: Build a reasonably sized FTS table (5000 rows).
           40  +#
           41  +#   1.1: Run the integrity check code to check it passes.
           42  +#
           43  +#   1.2: Make a series of minor changes to the underlying FTS data structures
           44  +#        (e.g. delete or insert a row from the %_content table). Check that
           45  +#        this causes the integrity-check code to fail.
           46  +#
           47  +
           48  +# Build an FTS table and check the integrity-check passes.
           49  +#
           50  +do_test 1.0 { fts3_build_db_1 5000 } {}
           51  +do_test 1.1 { fts_integrity db t1 } {ok}
           52  +
           53  +# Mess around with the underlying tables. Check that this causes the
           54  +# integrity-check test to fail.
           55  +#
           56  +foreach {tn disruption} {
           57  +  1 {
           58  +    INSERT INTO t1_content(docid, c0x, c1y) VALUES(NULL, 'a', 'b');
           59  +  }
           60  +  2 {
           61  +    DELETE FROM t1_content WHERE docid = (SELECT max(docid) FROM t1_content);
           62  +  }
           63  +  3 {
           64  +    DELETE FROM t1_segdir WHERE level=0 AND idx=(
           65  +      SELECT max(idx) FROM t1_segdir WHERE level=0
           66  +    );
           67  +  }
           68  +} {
           69  +  do_execsql_test  1.2.1.$tn "BEGIN; $disruption"
           70  +  do_catchsql_test 1.2.2.$tn {
           71  +    INSERT INTO t1 (t1) VALUES('integrity-check')
           72  +  } {1 {database disk image is malformed}}
           73  +  do_execsql_test  1.2.3.$tn "ROLLBACK"
           74  +}
           75  +
           76  +do_test 1.3 { fts_integrity db t1 } {ok}
           77  +
           78  +#-------------------------------------------------------------------------
           79  +# Test cases 2.*
           80  +#
           81  +#   2.0: Build a reasonably sized FTS table (20000 rows) that includes
           82  +#        prefix indexes.
           83  +#
           84  +#   2.1: Run the integrity check code to check it passes.
           85  +#
           86  +#   2.2: Make a series of minor changes to the underlying FTS data structures
           87  +#        (e.g. delete or insert a row from the %_content table). Check that
           88  +#        this causes the integrity-check code to fail.
           89  +#
           90  +
           91  +do_test 2.0 { fts3_build_db_2 -extra {prefix="3,1"} 20000 } {}
           92  +do_test 2.1 { fts_integrity db t2 } {ok}
           93  +foreach {tn disruption} {
           94  +  1 {
           95  +    INSERT INTO t2_content VALUES(NULL, 'xyz')
           96  +  }
           97  +  3 {
           98  +    DELETE FROM t2_segdir WHERE level=0 AND idx=(
           99  +      SELECT max(idx) FROM t2_segdir WHERE level=1024
          100  +    );
          101  +  }
          102  +} {
          103  +  do_execsql_test  2.2.1.$tn "BEGIN; $disruption"
          104  +  do_catchsql_test 2.2.2.$tn {
          105  +    INSERT INTO t2 (t2) VALUES('integrity-check')
          106  +  } {1 {database disk image is malformed}}
          107  +  do_execsql_test  2.2.3.$tn "ROLLBACK"
          108  +}
          109  +
          110  +
          111  +#-------------------------------------------------------------------------
          112  +# Test cases 3.*
          113  +#
          114  +#   3.0: Build a reasonably sized FTS table (5000 rows) that includes
          115  +#        prefix indexes and uses the languageid= feature.
          116  +#
          117  +#   3.1: Run the integrity check code to check it passes.
          118  +#
          119  +#   3.2: Make a series of minor changes to the underlying FTS data structures
          120  +#        (e.g. delete or insert a row from the %_content table). Check that
          121  +#        this causes the integrity-check code to fail.
          122  +#
          123  +do_test 3.0 {
          124  +  reset_db
          125  +  fts3_build_db_1 5000
          126  +  execsql {
          127  +    CREATE VIRTUAL TABLE t3 USING fts4(x, y, prefix="2,3", languageid=langid);
          128  +  }
          129  +  foreach docid [execsql {SELECT docid FROM t1 ORDER BY 1 ASC}] {
          130  +    execsql {
          131  +      INSERT INTO t3(x, y, langid) 
          132  +      SELECT x, y, (docid%9)*4 FROM t1 WHERE docid=$docid;
          133  +    }
          134  +  }
          135  +} {}
          136  +do_test 3.1 { fts_integrity db t3 } {ok}
          137  +
          138  +foreach {tn disruption} {
          139  +  1 {
          140  +    INSERT INTO t3_content(c0x, c1y, langid) VALUES(NULL, 'a', 0);
          141  +  }
          142  +  2 {
          143  +    UPDATE t3_content SET langid=langid+1 WHERE rowid = (
          144  +      SELECT max(rowid) FROM t3_content
          145  +    )
          146  +  }
          147  +} {
          148  +  do_execsql_test  3.2.1.$tn "BEGIN; $disruption"
          149  +  do_catchsql_test 3.2.2.$tn {
          150  +    INSERT INTO t3 (t3) VALUES('integrity-check')
          151  +  } {1 {database disk image is malformed}}
          152  +  do_execsql_test  3.2.3.$tn "ROLLBACK"
          153  +}
          154  +
          155  +finish_test

Changes to test/fts4langid.test.

   468    468   
   469    469     do_execsql_test 5.4.$lid.3 {
   470    470       SELECT count(*) FROM t6_segdir;
   471    471       SELECT count(*) FROM t6_segments;
   472    472     } {8 0}
   473    473   
   474    474     do_execsql_test 5.4.$lid.4 {
   475         -    INSERT INTO t6(t6) VALUES('optimize');
          475  +    INSERT INTO t6(t6) VALUES('merge=100,3');
          476  +    INSERT INTO t6(t6) VALUES('merge=100,3');
   476    477       SELECT docid FROM t6 WHERE t6 MATCH '"zero zero"' AND lid=$lid;
   477    478     } {1 2 5}
   478    479   
   479    480     do_execsql_test 5.4.$lid.5 {
   480    481       SELECT count(*) FROM t6_segdir;
   481    482       SELECT count(*) FROM t6_segments;
   482         -  } {1 0}
          483  +  } {4 4}
   483    484   }
   484         -
   485         -
   486    485   finish_test

Added test/fts4merge.test.

            1  +# 2012 March 06
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this script is testing the incremental merge function.
           13  +#
           14  +
           15  +set testdir [file dirname $argv0]
           16  +source $testdir/tester.tcl
           17  +source $testdir/fts3_common.tcl
           18  +
           19  +# If SQLITE_ENABLE_FTS3 is defined, omit this file.
           20  +ifcapable !fts3 {
           21  +  finish_test
           22  +  return
           23  +}
           24  +
           25  +proc fts3_integrity_check {tbl} {
           26  +  db eval "INSERT INTO $tbl ($tbl) VALUES('integrity-check')"
           27  +  return "ok"
           28  +}
           29  +
           30  +foreach mod {fts3 fts4} {
           31  +  set ::testprefix fts4merge-$mod
           32  +  reset_db
           33  +
           34  +  #-------------------------------------------------------------------------
           35  +  # Test cases 1.*
           36  +  #
           37  +  do_test 1.0 { fts3_build_db_1 -module $mod 1004 } {}
           38  +  do_test 1.1 { fts3_integrity_check t1 } {ok}
           39  +  do_execsql_test 1.1 { 
           40  +    SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level 
           41  +  } {
           42  +    0 {0 1 2 3 4 5 6 7 8 9 10 11} 
           43  +    1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13}
           44  +    2 {0 1 2}
           45  +  }
           46  +  
           47  +  for {set i 0} {$i<20} {incr i} {
           48  +    do_execsql_test 1.2.$i.1 { INSERT INTO t1(t1) VALUES('merge=1') }
           49  +    do_test 1.2.$i.2 { fts3_integrity_check t1 } ok
           50  +    do_execsql_test 1.2.$i.3 { 
           51  +      SELECT docid FROM t1 WHERE t1 MATCH 'zero one two three'
           52  +    } {123 132 213 231 312 321}
           53  +  }
           54  +  
           55  +  do_execsql_test 1.3 { 
           56  +    SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level 
           57  +  } {
           58  +    0 {0 1 2 3} 
           59  +    1 {0 1 2 3 4 5 6} 
           60  +    2 {0 1 2 3}
           61  +  }
           62  +  
           63  +  for {set i 0} {$i<100} {incr i} {
           64  +    do_execsql_test 1.4.$i { INSERT INTO t1(t1) VALUES('merge=1,4') }
           65  +    do_test 1.4.$i.2 { fts3_integrity_check t1 } ok
           66  +    do_execsql_test 1.4.$i.3 { 
           67  +      SELECT docid FROM t1 WHERE t1 MATCH 'zero one two three'
           68  +    } {123 132 213 231 312 321}
           69  +  }
           70  +  
           71  +  do_execsql_test 1.5 { 
           72  +    SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level 
           73  +  } {
           74  +    2 {0 1}
           75  +    3 0
           76  +  }
           77  +  
           78  +  #-------------------------------------------------------------------------
           79  +  # Test cases 2.* test that errors in the xxx part of the 'merge=xxx' are
           80  +  # handled correctly.
           81  +  #
           82  +  do_execsql_test 2.0 "CREATE VIRTUAL TABLE t2 USING $mod"
           83  +  
           84  +  foreach {tn arg} {
           85  +    1   {merge=abc}
           86  +    2   {merge=%%%}
           87  +    3   {merge=,}
           88  +    4   {merge=5,}
           89  +    5   {merge=6,%}
           90  +    6   {merge=6,six}
           91  +    7   {merge=6,1}
           92  +    8   {merge=6,0}
           93  +  } {
           94  +    do_catchsql_test 2.$tn { 
           95  +      INSERT INTO t2(t2) VALUES($arg);
           96  +    } {1 {SQL logic error or missing database}}
           97  +  }
           98  +  
           99  +  #-------------------------------------------------------------------------
          100  +  # Test cases 3.*
          101  +  #
          102  +  do_test 3.0 { 
          103  +    reset_db
          104  +    execsql { PRAGMA page_size = 512 }
          105  +    fts3_build_db_2 -module $mod 30040 
          106  +  } {}
          107  +  do_test 3.1 { fts3_integrity_check t2 } {ok}
          108  +  
          109  +  do_execsql_test 3.2 { 
          110  +    SELECT level, group_concat(idx, ' ') FROM t2_segdir GROUP BY level 
          111  +  } {
          112  +    0 {0 1 2 3 4 5 6} 
          113  +    1 {0 1 2 3 4} 
          114  +    2 {0 1 2 3 4} 
          115  +    3 {0 1 2 3 4 5 6}
          116  +  }
          117  +  
          118  +  do_execsql_test 3.3 { 
          119  +    INSERT INTO t2(t2) VALUES('merge=1000000,2');
          120  +    SELECT level, group_concat(idx, ' ') FROM t2_segdir GROUP BY level 
          121  +  } {
          122  +    0 0 
          123  +    2 0
          124  +    3 0 
          125  +    4 0
          126  +    6 0
          127  +  }
          128  +  
          129  +  #-------------------------------------------------------------------------
          130  +  # Test cases 4.*
          131  +  #
          132  +  reset_db
          133  +  do_execsql_test 4.1 "
          134  +    PRAGMA page_size = 512;
          135  +    CREATE VIRTUAL TABLE t4 USING $mod;
          136  +    PRAGMA main.page_size;
          137  +  " {512}
          138  +  
          139  +  do_test 4.2 {
          140  +    foreach x {a c b d e f g h i j k l m n o p} {
          141  +      execsql "INSERT INTO t4 VALUES('[string repeat $x 600]')"
          142  +    }
          143  +    execsql {SELECT level, group_concat(idx, ' ') FROM t4_segdir GROUP BY level}
          144  +  } {0 {0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15}}
          145  +  
          146  +  foreach {tn expect} {
          147  +    1  "0 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} 1 0"
          148  +    2  "0 {0 1 2 3 4 5 6 7 8 9 10 11 12}    1 0"
          149  +    3  "0 {0 1 2 3 4 5 6 7 8 9 10 11}       1 0"
          150  +    4  "0 {0 1 2 3 4 5 6 7 8 9 10}          1 0"
          151  +    5  "0 {0 1 2 3 4 5 6 7 8 9}             1 0"
          152  +    6  "0 {0 1 2 3 4 5 6 7 8}               1 0"
          153  +    7  "0 {0 1 2 3 4 5 6 7}                 1 0"
          154  +    8  "0 {0 1 2 3 4 5 6}                   1 0"
          155  +    9  "0 {0 1 2 3 4 5}                     1 0"
          156  +  } {
          157  +    do_execsql_test 4.3.$tn {
          158  +      INSERT INTO t4(t4) VALUES('merge=1,16');
          159  +      SELECT level, group_concat(idx, ' ') FROM t4_segdir GROUP BY level;
          160  +    } $expect
          161  +  }
          162  +  
          163  +  do_execsql_test 4.4.1 {
          164  +    SELECT quote(value) FROM t4_stat WHERE rowid=1
          165  +  } {X'0006'}
          166  +  
          167  +  do_execsql_test 4.4.2 {
          168  +    DELETE FROM t4_stat WHERE rowid=1;
          169  +    INSERT INTO t4(t4) VALUES('merge=1,12');
          170  +    SELECT level, group_concat(idx, ' ') FROM t4_segdir GROUP BY level;
          171  +  } "0 {0 1 2 3 4 5}                     1 0"
          172  +  
          173  +  
          174  +  #-------------------------------------------------------------------------
          175  +  # Test cases 5.*
          176  +  #
          177  +  # Test that if a crisis-merge occurs that disrupts an ongoing incremental
          178  +  # merge, the next call to "merge=A,B" identifies this and starts a new
          179  +  # incremental merge. There are two scenarios:
          180  +  #
          181  +  #   * There are less segments on the input level that the disrupted
          182  +  #     incremental merge operated on, or
          183  +  #   
          184  +  #   * Sufficient segments exist on the input level but the segments 
          185  +  #     contain keys smaller than the largest key in the potential output 
          186  +  #     segment.
          187  +  # 
          188  +  do_test 5.1 {
          189  +    reset_db
          190  +    fts3_build_db_1 -module $mod 1000
          191  +  } {}
          192  +  
          193  +  do_execsql_test 5.2 {
          194  +    SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level;
          195  +  } {
          196  +    0 {0 1 2 3 4 5 6 7} 
          197  +    1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} 
          198  +    2 {0 1 2}
          199  +  }
          200  +  
          201  +  do_execsql_test 5.3 {
          202  +    INSERT INTO t1(t1) VALUES('merge=1,5');
          203  +    INSERT INTO t1(t1) VALUES('merge=1,5');
          204  +    SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level;
          205  +  } {
          206  +    0 {0 1 2}
          207  +    1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13 14} 
          208  +    2 {0 1 2 3}
          209  +  }
          210  +  
          211  +  do_execsql_test 5.4 {SELECT quote(value) from t1_stat WHERE rowid=1} {X'0105'}
          212  +  do_test 5.5 {
          213  +    foreach docid [execsql {SELECT docid FROM t1}] {
          214  +      execsql {INSERT INTO t1 SELECT * FROM t1 WHERE docid=$docid}
          215  +    }
          216  +  } {}
          217  +  
          218  +  do_execsql_test 5.6 {SELECT quote(value) from t1_stat WHERE rowid=1} {X'0105'}
          219  +  
          220  +  do_execsql_test 5.7 {
          221  +    SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level;
          222  +    SELECT quote(value) from t1_stat WHERE rowid=1;
          223  +  } {
          224  +    0 {0 1 2 3 4 5 6 7 8 9 10} 
          225  +    1 {0 1 2 3 4 5 6 7 8 9 10 11 12} 
          226  +    2 {0 1 2 3 4 5 6 7}
          227  +    X'0105'
          228  +  }
          229  +  
          230  +  do_execsql_test 5.8 {
          231  +    INSERT INTO t1(t1) VALUES('merge=1,6');
          232  +    INSERT INTO t1(t1) VALUES('merge=1,6');
          233  +    SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level;
          234  +    SELECT quote(value) from t1_stat WHERE rowid=1;
          235  +  } {
          236  +    0 {0 1 2 3 4} 
          237  +    1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} 
          238  +    2 {0 1 2 3 4 5 6 7 8} X'0106'
          239  +  }
          240  +  
          241  +  do_test 5.8.1 { fts3_integrity_check t1 } ok
          242  +  
          243  +  do_test 5.9 {
          244  +    set L [expr 16*16*7 + 16*3 + 12]
          245  +    foreach docid [execsql {
          246  +        SELECT docid FROM t1 UNION ALL SELECT docid FROM t1 LIMIT $L
          247  +    }] {
          248  +      execsql {INSERT INTO t1 SELECT * FROM t1 WHERE docid=$docid}
          249  +    }
          250  +  } {}
          251  +  
          252  +  do_execsql_test 5.10 {
          253  +    SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level;
          254  +    SELECT quote(value) from t1_stat WHERE rowid=1;
          255  +  } {
          256  +    0 0 1 {0 1} 2 0 3 0 X'0106'
          257  +  }
          258  +  
          259  +  do_execsql_test 5.11 {
          260  +    INSERT INTO t1(t1) VALUES('merge=1,6');
          261  +    SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level;
          262  +    SELECT quote(value) from t1_stat WHERE rowid=1;
          263  +  } {
          264  +    0 0 1 {0 1} 2 0 3 0 X''
          265  +  }
          266  +  
          267  +  #-------------------------------------------------------------------------
          268  +  # Test cases 6.*
          269  +  #
          270  +  # At one point the following test caused an assert() to fail (because the
          271  +  # second 'merge=1,2' operation below actually "merges" a single input
          272  +  # segment, which was unexpected).
          273  +  #
          274  +  do_test 6.1 {
          275  +    reset_db
          276  +    set a [string repeat a 900]
          277  +    set b [string repeat b 900]
          278  +    set c [string repeat c 900]
          279  +    set d [string repeat d 900]
          280  +
          281  +    execsql "CREATE VIRTUAL TABLE t1 USING $mod"
          282  +    execsql {
          283  +      BEGIN;
          284  +        INSERT INTO t1 VALUES($a);
          285  +        INSERT INTO t1 VALUES($b);
          286  +      COMMIT;
          287  +      BEGIN;
          288  +        INSERT INTO t1 VALUES($c);
          289  +        INSERT INTO t1 VALUES($d);
          290  +      COMMIT;
          291  +    }
          292  +  
          293  +    execsql {
          294  +      INSERT INTO t1(t1) VALUES('merge=1,2');
          295  +      INSERT INTO t1(t1) VALUES('merge=1,2');
          296  +    }
          297  +  } {}
          298  +  
          299  +  #-------------------------------------------------------------------------
          300  +  # Test cases 7.*
          301  +  #
          302  +  # Test that the value returned by sqlite3_total_changes() increases by
          303  +  # 1 following a no-op "merge=A,B", or by more than 1 if actual work is
          304  +  # performed.
          305  +  #
          306  +  do_test 7.0 {
          307  +    reset_db
          308  +    fts3_build_db_1 -module $mod 1000
          309  +  } {}
          310  +  
          311  +  do_execsql_test 7.1 {
          312  +    SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level
          313  +  } {
          314  +    0 {0 1 2 3 4 5 6 7} 
          315  +    1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} 
          316  +    2 {0 1 2}
          317  +  }
          318  +  do_test 7.2 {
          319  +    set x [db total_changes]
          320  +    execsql { INSERT INTO t1(t1) VALUES('merge=2,10') }
          321  +    expr { ([db total_changes] - $x)>1 }
          322  +  } {1}
          323  +  do_test 7.3 {
          324  +    set x [db total_changes]
          325  +    execsql { INSERT INTO t1(t1) VALUES('merge=200,10') }
          326  +    expr { ([db total_changes] - $x)>1 }
          327  +  } {1}
          328  +  do_test 7.4 {
          329  +    set x [db total_changes]
          330  +    execsql { INSERT INTO t1(t1) VALUES('merge=200,10') }
          331  +    expr { ([db total_changes] - $x)>1 }
          332  +  } {0}
          333  +  do_test 7.5 {
          334  +    set x [db total_changes]
          335  +    execsql { INSERT INTO t1(t1) VALUES('merge=200,10') }
          336  +    expr { ([db total_changes] - $x)>1 }
          337  +  } {0}
          338  +
          339  +}
          340  +
          341  +finish_test

Added test/fts4merge2.test.

            1  +
            2  +
            3  +set testdir [file dirname $argv0]
            4  +source $testdir/tester.tcl
            5  +source $testdir/fts3_common.tcl
            6  +source $testdir/malloc_common.tcl
            7  +set ::testprefix fts4merge2
            8  +
            9  +# If SQLITE_ENABLE_FTS3 is defined, omit this file.
           10  +ifcapable !fts3 {
           11  +  finish_test
           12  +  return
           13  +}
           14  +
           15  +do_test 1.0 {
           16  +  fts3_build_db_1 1000
           17  +  faultsim_save_and_close
           18  +} {}
           19  +
           20  +do_faultsim_test 1.1 -faults oom-* -prep {
           21  +  faultsim_restore_and_reopen
           22  +} -body {
           23  +  execsql { INSERT INTO t1(t1) VALUES('merge=32,4') }
           24  +} -test {
           25  +  faultsim_test_result {0 {}} 
           26  +}
           27  +
           28  +do_faultsim_test 1.2 -faults oom-t* -prep {
           29  +  if {$iFail<100} {set iFail 803}
           30  +  faultsim_restore_and_reopen
           31  +} -body {
           32  +  execsql { INSERT INTO t1(t1) VALUES('merge=1,2') }
           33  +  execsql { INSERT INTO t1(t1) VALUES('merge=1,2') }
           34  +} -test {
           35  +  faultsim_test_result {0 {}} 
           36  +}
           37  +
           38  +finish_test

Added test/fts4merge3.test.

            1  +# 2012 March 06
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this script is testing the incremental merge function.
           13  +#
           14  +
           15  +set testdir [file dirname $argv0]
           16  +source $testdir/tester.tcl
           17  +source $testdir/fts3_common.tcl
           18  +source $testdir/lock_common.tcl
           19  +source $testdir/bc_common.tcl
           20  +
           21  +set ::testprefix fts4merge3
           22  +
           23  +if {"" == [bc_find_binaries backcompat.test]} {
           24  +  finish_test
           25  +  return
           26  +}
           27  +
           28  +db close
           29  +do_all_bc_test {
           30  +
           31  +  sql2 { PRAGMA page_size = 512 }
           32  +  if { 0==[catch { sql2 { CREATE VIRTUAL TABLE x USING fts4 } } ] } {
           33  +
           34  +    # Build a large database.
           35  +    set msg "this takes around 12 seconds"
           36  +    do_test "1.1 ($msg)" { fts3_build_db_2 20000 } {}
           37  +
           38  +    # Run some queries on it, using the old and new versions.
           39  +    do_test 1.2 { sql1 "SELECT docid FROM t2 WHERE t2 MATCH 'abc'" } {1485}
           40  +    do_test 1.3 { sql2 "SELECT docid FROM t2 WHERE t2 MATCH 'abc'" } {1485}
           41  +
           42  +    do_test 1.4 { sql2 "PRAGMA page_count" } {1286}
           43  +    do_test 1.5 { sql2 { 
           44  +      SELECT level, count(*) FROM t2_segdir GROUP BY level ORDER BY 1
           45  +    } } [list 0 15    1 1     2 14    3 4]
           46  +
           47  +    # Run some incr-merge operations on the db.
           48  +    for {set i 0} {$i<10} {incr i} {
           49  +      do_test 1.6.$i.1 { sql1 { INSERT INTO t2(t2) VALUES('merge=2,2') } } {}
           50  +      do_test 1.6.$i.2 { 
           51  +        sql2 "SELECT docid FROM t2 WHERE t2 MATCH 'abc'" 
           52  +      } {1485}
           53  +    }
           54  +
           55  +    do_test 1.7 { sql2 { 
           56  +      SELECT level, count(*) FROM t2_segdir GROUP BY level ORDER BY 1
           57  +    } } [list  0 1  2 18  3 5]
           58  +
           59  +    # Using the old connection, insert many rows. 
           60  +    do_test 1.8 {
           61  +      for {set i 0} {$i < 1500} {incr i} {
           62  +        sql2 "INSERT INTO t2 SELECT content FROM t2 WHERE docid = $i"
           63  +      }
           64  +    } {}
           65  +
           66  +    do_test 1.9 { sql2 { 
           67  +      SELECT level, count(*) FROM t2_segdir GROUP BY level ORDER BY 1
           68  +    } } [list  0 13  1 13  2 5  3 6]
           69  +
           70  +    # Run a big incr-merge operation on the db.
           71  +    do_test 1.10 { sql1 { INSERT INTO t2(t2) VALUES('merge=2000,2') } } {}
           72  +    do_test 1.11 { 
           73  +      sql2 "SELECT docid FROM t2 WHERE t2 MATCH 'abc'" 
           74  +    } {1485 21485}
           75  +
           76  +    do_test 1.12 {
           77  +      for {set i 0} {$i < 1500} {incr i} {
           78  +        sql2 "INSERT INTO t2 SELECT content FROM t2 WHERE docid = $i"
           79  +      }
           80  +    } {}
           81  +    do_test 1.13 { 
           82  +      sql2 "SELECT docid FROM t2 WHERE t2 MATCH 'abc'" 
           83  +    } {1485 21485 22985}
           84  +
           85  +    do_test 1.14 { 
           86  +      sql2 "INSERT INTO t2(t2) VALUES('optimize')"
           87  +      sql2 "SELECT docid FROM t2 WHERE t2 MATCH 'abc'" 
           88  +    } {1485 21485 22985}
           89  +
           90  +    do_test 1.15 { sql2 { 
           91  +      SELECT level, count(*) FROM t2_segdir GROUP BY level ORDER BY 1
           92  +    } } {6 1}
           93  +  }
           94  +}
           95  +
           96  +
           97  +finish_test

Changes to test/func.test.

  1242   1242     db eval {
  1243   1243       CREATE TABLE t28(x, y DEFAULT(nosuchfunc(1)));
  1244   1244     }
  1245   1245     catchsql {
  1246   1246       INSERT INTO t28(x) VALUES(1);
  1247   1247     }
  1248   1248   } {1 {unknown function: nosuchfunc()}}
         1249  +
         1250  +# Verify that the length() and typeof() functions do not actually load
         1251  +# the content of their argument.
         1252  +#
         1253  +do_test func-29.1 {
         1254  +  db eval {
         1255  +    CREATE TABLE t29(id INTEGER PRIMARY KEY, x, y);
         1256  +    INSERT INTO t29 VALUES(1, 2, 3), (2, NULL, 4), (3, 4.5, 5);
         1257  +    INSERT INTO t29 VALUES(4, randomblob(1000000), 6);
         1258  +    INSERT INTO t29 VALUES(5, "hello", 7);
         1259  +  }
         1260  +  db close
         1261  +  sqlite3 db test.db
         1262  +  sqlite3_db_status db CACHE_MISS 1
         1263  +  db eval {SELECT typeof(x), length(x), typeof(y) FROM t29 ORDER BY id}
         1264  +} {integer 1 integer null {} integer real 3 integer blob 1000000 integer text 5 integer}
         1265  +do_test func-29.2 {
         1266  +  set x [lindex [sqlite3_db_status db CACHE_MISS 1] 1]
         1267  +  if {$x<5} {set x 1}
         1268  +  set x
         1269  +} {1}
         1270  +do_test func-29.3 {
         1271  +  db close
         1272  +  sqlite3 db test.db
         1273  +  sqlite3_db_status db CACHE_MISS 1
         1274  +  db eval {SELECT typeof(+x) FROM t29 ORDER BY id}
         1275  +} {integer null real blob text}
         1276  +do_test func-29.4 {
         1277  +  set x [lindex [sqlite3_db_status db CACHE_MISS 1] 1]
         1278  +  if {$x>100} {set x many}
         1279  +  set x
         1280  +} {many}
         1281  +do_test func-29.5 {
         1282  +  db close
         1283  +  sqlite3 db test.db
         1284  +  sqlite3_db_status db CACHE_MISS 1
         1285  +  db eval {SELECT sum(length(x)) FROM t29}
         1286  +} {1000009}
         1287  +do_test func-29.6 {
         1288  +  set x [lindex [sqlite3_db_status db CACHE_MISS 1] 1]
         1289  +  if {$x<5} {set x 1}
         1290  +  set x
         1291  +} {1}
         1292  +  
  1249   1293   
  1250   1294   finish_test

Added test/incrblob4.test.

            1  +# 2012 March 23
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +#
           12  +#
           13  +set testdir [file dirname $argv0]
           14  +source $testdir/tester.tcl
           15  +ifcapable {!incrblob} { finish_test ; return }
           16  +set testprefix incrblob4
           17  +
           18  +proc create_t1 {} {
           19  +  execsql {
           20  +    PRAGMA page_size = 1024;
           21  +    CREATE TABLE t1(k INTEGER PRIMARY KEY, v);
           22  +  }
           23  +}
           24  +
           25  +proc populate_t1 {} {
           26  +  set data [list a b c d e f g h i j k l m n o p q r s t u v w x y z]
           27  +  foreach d $data {
           28  +    set blob [string repeat $d 900]
           29  +    execsql { INSERT INTO t1(v) VALUES($blob) }
           30  +  }
           31  +}
           32  +
           33  +
           34  +do_test 1.1 { 
           35  +  create_t1
           36  +  populate_t1 
           37  +} {}
           38  +
           39  +do_test 1.2 {
           40  +  set blob [db incrblob t1 v 5]
           41  +  read $blob 10
           42  +} {eeeeeeeeee}
           43  +
           44  +do_test 1.3 {
           45  +  execsql { DELETE FROM t1 }
           46  +  populate_t1
           47  +} {}
           48  +
           49  +
           50  +
           51  +do_test 2.1 { 
           52  +  reset_db
           53  +  create_t1
           54  +  populate_t1 
           55  +} {}
           56  +
           57  +do_test 2.2 {
           58  +  set blob [db incrblob t1 v 10]
           59  +  read $blob 10
           60  +} {jjjjjjjjjj}
           61  +
           62  +do_test 2.3 {
           63  +  set new [string repeat % 900]
           64  +  execsql { DELETE FROM t1 WHERE k=10 }
           65  +  execsql { DELETE FROM t1 WHERE k=9 }
           66  +  execsql { INSERT INTO t1(v) VALUES($new) }
           67  +} {}
           68  +
           69  +
           70  +
           71  +do_test 3.1 {
           72  +  reset_db
           73  +  create_t1
           74  +  populate_t1 
           75  +} {}
           76  +
           77  +do_test 3.2 {
           78  +  set blob [db incrblob t1 v 20]
           79  +  read $blob 10
           80  +} {tttttttttt}
           81  +
           82  +do_test 3.3 {
           83  +  set new [string repeat % 900]
           84  +  execsql { UPDATE t1 SET v = $new WHERE k = 20 }
           85  +  execsql { DELETE FROM t1 WHERE k=19 }
           86  +  execsql { INSERT INTO t1(v) VALUES($new) }
           87  +} {}
           88  +
           89  +finish_test
           90  +

Changes to test/pager1.test.

  1791   1791       PRAGMA writable_schema = 1;
  1792   1792       UPDATE sqlite_master SET rootpage = $lockingpage;
  1793   1793     }
  1794   1794     sqlite3 db2 test.db
  1795   1795     catchsql { SELECT count(*) FROM t1 } db2
  1796   1796   } {1 {database disk image is malformed}}
  1797   1797   db2 close
  1798         -do_test pager1-18.3 {
         1798  +do_test pager1-18.3.1 {
  1799   1799     execsql {
  1800   1800       CREATE TABLE t2(x);
  1801   1801       INSERT INTO t2 VALUES(a_string(5000));
  1802   1802     }
  1803   1803     set pgno [expr ([file size test.db] / 1024)-2]
  1804   1804     hexio_write test.db [expr ($pgno-1)*1024] 00000000
  1805   1805     sqlite3 db2 test.db
  1806         -  catchsql { SELECT length(x) FROM t2 } db2
         1806  +  # even though x is malformed, because typeof() does
         1807  +  # not load the content of x, the error is not noticed.
         1808  +  catchsql { SELECT typeof(x) FROM t2 } db2
         1809  +} {0 text}
         1810  +do_test pager1-18.3.2 {
         1811  +  # in this case, the value of x is loaded and so the error is
         1812  +  # detected
         1813  +  catchsql { SELECT length(x||'') FROM t2 } db2
         1814  +} {1 {database disk image is malformed}}
         1815  +db2 close
         1816  +do_test pager1-18.3.3 {
         1817  +  execsql {
         1818  +    DELETE FROM t2;
         1819  +    INSERT INTO t2 VALUES(randomblob(5000));
         1820  +  }
         1821  +  set pgno [expr ([file size test.db] / 1024)-2]
         1822  +  hexio_write test.db [expr ($pgno-1)*1024] 00000000
         1823  +  sqlite3 db2 test.db
         1824  +  # even though x is malformed, because length() and typeof() do
         1825  +  # not load the content of x, the error is not noticed.
         1826  +  catchsql { SELECT length(x), typeof(x) FROM t2 } db2
         1827  +} {0 {5000 blob}}
         1828  +do_test pager1-18.3.4 {
         1829  +  # in this case, the value of x is loaded and so the error is
         1830  +  # detected
         1831  +  catchsql { SELECT length(x||'') FROM t2 } db2
  1807   1832   } {1 {database disk image is malformed}}
  1808   1833   db2 close
  1809   1834   do_test pager1-18.4 {
  1810   1835     hexio_write test.db [expr ($pgno-1)*1024] 90000000
  1811   1836     sqlite3 db2 test.db
  1812         -  catchsql { SELECT length(x) FROM t2 } db2
         1837  +  catchsql { SELECT length(x||'') FROM t2 } db2
  1813   1838   } {1 {database disk image is malformed}}
  1814   1839   db2 close
  1815   1840   do_test pager1-18.5 {
  1816   1841     sqlite3 db ""
  1817   1842     execsql {
  1818   1843       CREATE TABLE t1(a, b);
  1819   1844       CREATE TABLE t2(a, b);

Changes to test/permutations.test.

   180    180     fts3defer.test fts3defer2.test fts3e.test fts3expr.test fts3expr2.test 
   181    181     fts3near.test fts3query.test fts3shared.test fts3snippet.test 
   182    182     fts3sort.test
   183    183     fts3fault.test fts3malloc.test fts3matchinfo.test
   184    184     fts3aux1.test fts3comp1.test fts3auto.test
   185    185     fts4aa.test fts4content.test
   186    186     fts3conf.test fts3prefix.test fts3fault2.test fts3corrupt.test
   187         -  fts3corrupt2.test fts3first.test fts4langid.test
          187  +  fts3corrupt2.test fts3first.test fts4langid.test fts4merge.test
          188  +  fts4check.test
   188    189   }
   189    190   
   190    191   
   191    192   lappend ::testsuitelist xxx
   192    193   #-------------------------------------------------------------------------
   193    194   # Define the coverage related test suites:
   194    195   #

Changes to test/trace2.test.

   126    126   
   127    127     do_trace_test 2.2 {
   128    128       INSERT INTO x1 VALUES('North northwest wind between 8 and 14 mph');
   129    129     } {
   130    130       "INSERT INTO x1 VALUES('North northwest wind between 8 and 14 mph');" 
   131    131       "-- INSERT INTO 'main'.'x1_content' VALUES(?,(?))" 
   132    132       "-- REPLACE INTO 'main'.'x1_docsize' VALUES(?,?)" 
   133         -    "-- SELECT value FROM 'main'.'x1_stat' WHERE id=0" 
   134         -    "-- REPLACE INTO 'main'.'x1_stat' VALUES(0,?)" 
          133  +    "-- SELECT value FROM 'main'.'x1_stat' WHERE id=?" 
          134  +    "-- REPLACE INTO 'main'.'x1_stat' VALUES(?,?)" 
   135    135       "-- SELECT (SELECT max(idx) FROM 'main'.'x1_segdir' WHERE level = ?) + 1" 
   136    136       "-- SELECT coalesce((SELECT max(blockid) FROM 'main'.'x1_segments') + 1, 1)"
   137         -    "-- INSERT INTO 'main'.'x1_segdir' VALUES(?,?,?,?,?,?)"
          137  +    "-- REPLACE INTO 'main'.'x1_segdir' VALUES(?,?,?,?,?,?)"
   138    138     }
   139    139   
   140    140     do_trace_test 2.3 {
   141    141       INSERT INTO x1(x1) VALUES('optimize');
   142    142     } {
   143    143       "INSERT INTO x1(x1) VALUES('optimize');"
   144    144       "-- SELECT DISTINCT level / (1024 * ?) FROM 'main'.'x1_segdir'"
   145    145       "-- SELECT idx, start_block, leaves_end_block, end_block, root FROM 'main'.'x1_segdir' WHERE level BETWEEN ? AND ?ORDER BY level DESC, idx ASC"
   146    146       "-- SELECT max(level) FROM 'main'.'x1_segdir' WHERE level BETWEEN ? AND ?"
   147    147       "-- SELECT coalesce((SELECT max(blockid) FROM 'main'.'x1_segments') + 1, 1)"
   148    148       "-- DELETE FROM 'main'.'x1_segdir' WHERE level BETWEEN ? AND ?"
   149         -    "-- INSERT INTO 'main'.'x1_segdir' VALUES(?,?,?,?,?,?)"
          149  +    "-- REPLACE INTO 'main'.'x1_segdir' VALUES(?,?,?,?,?,?)"
   150    150     }
   151    151   }
   152    152   
   153    153   finish_test

Changes to test/vtab1.test.

    11     11   # This file implements regression tests for SQLite library.  The
    12     12   # focus of this file is creating and dropping virtual tables.
    13     13   #
    14     14   # $Id: vtab1.test,v 1.57 2008/08/01 17:51:47 danielk1977 Exp $
    15     15   
    16     16   set testdir [file dirname $argv0]
    17     17   source $testdir/tester.tcl
           18  +set testprefix vtab1
    18     19   
    19     20   ifcapable !vtab||!schema_pragmas {
    20     21     finish_test
    21     22     return
    22     23   }
    23     24   
    24     25   #----------------------------------------------------------------------
................................................................................
    38     39   # in that file for the special behaviour of the Tcl $echo_module variable.
    39     40   #
    40     41   # TODO: 
    41     42   #   * How to test the sqlite3_index_constraint_usage.omit field?
    42     43   #   * vtab1-5.*
    43     44   #
    44     45   # vtab1-14.*: Test 'IN' constraints - i.e. "SELECT * FROM t1 WHERE id IN(...)"
           46  +#
           47  +# vtab1-18.*: Check that the LIKE optimization is not applied when the lhs
           48  +#             is a virtual table column.
    45     49   #
    46     50   
    47     51   
    48     52   #----------------------------------------------------------------------
    49     53   # Test cases vtab1.1.*
    50     54   #
    51     55   
................................................................................
  1214   1218         DROP TABLE e5;
  1215   1219         SAVEPOINT one;
  1216   1220         ROLLBACK TO one;
  1217   1221       COMMIT;
  1218   1222     }
  1219   1223   } {}
  1220   1224   
         1225  +#-------------------------------------------------------------------------
         1226  +# The following tests - vtab1-18.* - test that the optimization of LIKE
         1227  +# constraints in where.c plays well with virtual tables.
         1228  +#
         1229  +#   18.1.*: Case-insensitive LIKE.
         1230  +#   18.2.*: Case-sensitive LIKE.
         1231  +#
  1221   1232   unset -nocomplain echo_module_begin_fail
         1233  +
         1234  +do_execsql_test 18.1.0 {
         1235  +  CREATE TABLE t6(a, b TEXT);
         1236  +  CREATE INDEX i6 ON t6(b, a);
         1237  +  INSERT INTO t6 VALUES(1, 'Peter');
         1238  +  INSERT INTO t6 VALUES(2, 'Andrew');
         1239  +  INSERT INTO t6 VALUES(3, 'James');
         1240  +  INSERT INTO t6 VALUES(4, 'John');
         1241  +  INSERT INTO t6 VALUES(5, 'Phillip');
         1242  +  INSERT INTO t6 VALUES(6, 'Bartholomew');
         1243  +  CREATE VIRTUAL TABLE e6 USING echo(t6);
         1244  +}
         1245  +
         1246  +foreach {tn sql res filter} {
         1247  +  1.1 "SELECT a FROM e6 WHERE b>'James'" {4 1 5}
         1248  +    {xFilter {SELECT rowid, * FROM 't6' WHERE b > ?} James}
         1249  +
         1250  +  1.2 "SELECT a FROM e6 WHERE b>='J' AND b<'K'" {3 4}
         1251  +    {xFilter {SELECT rowid, * FROM 't6' WHERE b >= ? AND b < ?} J K}
         1252  +
         1253  +  1.3 "SELECT a FROM e6 WHERE b LIKE 'J%'" {3 4}
         1254  +    {xFilter {SELECT rowid, * FROM 't6'}}
         1255  +
         1256  +  1.4 "SELECT a FROM e6 WHERE b LIKE 'j%'" {3 4}
         1257  +    {xFilter {SELECT rowid, * FROM 't6'}}
         1258  +} {
         1259  +  set echo_module {}
         1260  +  do_execsql_test 18.$tn.1 $sql $res
         1261  +  do_test         18.$tn.2 { lrange $::echo_module 2 end } $filter
         1262  +}
         1263  +
         1264  +do_execsql_test 18.2.0 {  PRAGMA case_sensitive_like = ON }
         1265  +foreach {tn sql res filter} {
         1266  +  2.1 "SELECT a FROM e6 WHERE b LIKE 'J%'" {3 4}
         1267  +    {xFilter {SELECT rowid, * FROM 't6'}}
         1268  +
         1269  +  2.2 "SELECT a FROM e6 WHERE b LIKE 'j%'" {}
         1270  +    {xFilter {SELECT rowid, * FROM 't6'}}
         1271  +} {
         1272  +  set echo_module {}
         1273  +  do_execsql_test 18.$tn.1 $sql $res
         1274  +  do_test         18.$tn.2 { lrange $::echo_module 2 end } $filter
         1275  +}
         1276  +do_execsql_test 18.2.x {  PRAGMA case_sensitive_like = OFF }
         1277  +
  1222   1278   finish_test