/ Check-in [a75f1800]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fixes to prevent an FTS index from growing indefinitely as the corresponding table is updated. Change the FTS 'automerge' option to allow the user to specify the number of segments that should be merged simultaneously by auto-merges.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: a75f1800021d30b712053373440041b7b355a36a
User & Date: dan 2014-05-16 16:16:59
Context
2014-05-16
23:15
Work around compilation issue with MSVC. check-in: 9623a29c user: mistachkin tags: trunk
16:16
Fixes to prevent an FTS index from growing indefinitely as the corresponding table is updated. Change the FTS 'automerge' option to allow the user to specify the number of segments that should be merged simultaneously by auto-merges. check-in: a75f1800 user: dan tags: trunk
15:48
Add extra test to backcompat.test to ensure that old and new versions of FTS may work together on the same incremental merge operation. Closed-Leaf check-in: 3997d47b user: dan tags: fts4-experimental
14:17
Repurpose the SQLITE_TESTCTRL_FAULT_INSTALL test-control to register a callback to be invoked by sqlite3FaultSim(). That test-control has been unused since 2008-06-20 and was never used in any official release. check-in: 0d43a7ad user: drh tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts3/fts3.c.

  1329   1329     p->azColumn = (char **)&p[1];
  1330   1330     p->pTokenizer = pTokenizer;
  1331   1331     p->nMaxPendingData = FTS3_MAX_PENDING_DATA;
  1332   1332     p->bHasDocsize = (isFts4 && bNoDocsize==0);
  1333   1333     p->bHasStat = isFts4;
  1334   1334     p->bFts4 = isFts4;
  1335   1335     p->bDescIdx = bDescIdx;
  1336         -  p->bAutoincrmerge = 0xff;   /* 0xff means setting unknown */
         1336  +  p->nAutoincrmerge = 0xff;   /* 0xff means setting unknown */
  1337   1337     p->zContentTbl = zContent;
  1338   1338     p->zLanguageid = zLanguageid;
  1339   1339     zContent = 0;
  1340   1340     zLanguageid = 0;
  1341   1341     TESTONLY( p->inTransaction = -1 );
  1342   1342     TESTONLY( p->mxSavepoint = -1 );
  1343   1343   
................................................................................
  3298   3298     ** segments.
  3299   3299     */
  3300   3300     const u32 nMinMerge = 64;       /* Minimum amount of incr-merge work to do */
  3301   3301   
  3302   3302     Fts3Table *p = (Fts3Table*)pVtab;
  3303   3303     int rc = sqlite3Fts3PendingTermsFlush(p);
  3304   3304   
  3305         -  if( rc==SQLITE_OK && p->bAutoincrmerge==1 && p->nLeafAdd>(nMinMerge/16) ){
         3305  +  if( rc==SQLITE_OK 
         3306  +   && p->nLeafAdd>(nMinMerge/16) 
         3307  +   && p->nAutoincrmerge && p->nAutoincrmerge!=0xff
         3308  +  ){
  3306   3309       int mxLevel = 0;              /* Maximum relative level value in db */
  3307   3310       int A;                        /* Incr-merge parameter A */
  3308   3311   
  3309   3312       rc = sqlite3Fts3MaxLevel(p, &mxLevel);
  3310   3313       assert( rc==SQLITE_OK || mxLevel==0 );
  3311   3314       A = p->nLeafAdd * mxLevel;
  3312   3315       A += (A/2);
  3313         -    if( A>(int)nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, 8);
         3316  +    if( A>(int)nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, p->nAutoincrmerge);
  3314   3317     }
  3315   3318     sqlite3Fts3SegmentsClose(p);
  3316   3319     return rc;
  3317   3320   }
  3318   3321   
  3319   3322   /*
  3320   3323   ** If it is currently unknown whether or not the FTS table has an %_stat

Changes to ext/fts3/fts3Int.h.

   206    206     const char *zName;              /* virtual table name */
   207    207     int nColumn;                    /* number of named columns in virtual table */
   208    208     char **azColumn;                /* column names.  malloced */
   209    209     u8 *abNotindexed;               /* True for 'notindexed' columns */
   210    210     sqlite3_tokenizer *pTokenizer;  /* tokenizer for inserts and queries */
   211    211     char *zContentTbl;              /* content=xxx option, or NULL */
   212    212     char *zLanguageid;              /* languageid=xxx option, or NULL */
   213         -  u8 bAutoincrmerge;              /* True if automerge=1 */
          213  +  int nAutoincrmerge;             /* Value configured by 'automerge' */
   214    214     u32 nLeafAdd;                   /* Number of leaf blocks added this trans */
   215    215   
   216    216     /* Precompiled statements used by the implementation. Each of these 
   217    217     ** statements is run and reset within a single virtual table API call. 
   218    218     */
   219         -  sqlite3_stmt *aStmt[37];
          219  +  sqlite3_stmt *aStmt[40];
   220    220   
   221    221     char *zReadExprlist;
   222    222     char *zWriteExprlist;
   223    223   
   224    224     int nNodeSize;                  /* Soft limit for node size */
   225    225     u8 bFts4;                       /* True for FTS4, false for FTS3 */
   226    226     u8 bHasStat;                    /* True if %_stat table exists (2==unknown) */

Changes to ext/fts3/fts3_write.c.

   189    189     char *zTerm;                    /* Pointer to previous term buffer */
   190    190     int nTerm;                      /* Number of bytes in zTerm */
   191    191     int nMalloc;                    /* Size of malloc'd buffer at zMalloc */
   192    192     char *zMalloc;                  /* Malloc'd space (possibly) used for zTerm */
   193    193     int nSize;                      /* Size of allocation at aData */
   194    194     int nData;                      /* Bytes of data in aData */
   195    195     char *aData;                    /* Pointer to block from malloc() */
          196  +  i64 nLeafData;                  /* Number of bytes of leaf data written */
   196    197   };
   197    198   
   198    199   /*
   199    200   ** Type SegmentNode is used by the following three functions to create
   200    201   ** the interior part of the segment b+-tree structures (everything except
   201    202   ** the leaf nodes). These functions and type are only ever used by code
   202    203   ** within the fts3SegWriterXXX() family of functions described above.
................................................................................
   263    264   #define SQL_DELETE_SEGDIR_ENTRY       30
   264    265   #define SQL_SHIFT_SEGDIR_ENTRY        31
   265    266   #define SQL_SELECT_SEGDIR             32
   266    267   #define SQL_CHOMP_SEGDIR              33
   267    268   #define SQL_SEGMENT_IS_APPENDABLE     34
   268    269   #define SQL_SELECT_INDEXES            35
   269    270   #define SQL_SELECT_MXLEVEL            36
          271  +
          272  +#define SQL_SELECT_LEVEL_RANGE2       37
          273  +#define SQL_UPDATE_LEVEL_IDX          38
          274  +#define SQL_UPDATE_LEVEL              39
   270    275   
   271    276   /*
   272    277   ** This function is used to obtain an SQLite prepared statement handle
   273    278   ** for the statement identified by the second argument. If successful,
   274    279   ** *pp is set to the requested statement handle and SQLITE_OK returned.
   275    280   ** Otherwise, an SQLite error code is returned and *pp is set to 0.
   276    281   **
................................................................................
   365    370   
   366    371   /* SQL_SELECT_INDEXES
   367    372   **   Return the list of valid segment indexes for absolute level ?  */
   368    373   /* 35 */  "SELECT idx FROM %Q.'%q_segdir' WHERE level=? ORDER BY 1 ASC",
   369    374   
   370    375   /* SQL_SELECT_MXLEVEL
   371    376   **   Return the largest relative level in the FTS index or indexes.  */
   372         -/* 36 */  "SELECT max( level %% 1024 ) FROM %Q.'%q_segdir'"
          377  +/* 36 */  "SELECT max( level %% 1024 ) FROM %Q.'%q_segdir'",
          378  +
          379  +          /* Return segments in order from oldest to newest.*/ 
          380  +/* 37 */  "SELECT level, idx, end_block "
          381  +            "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? "
          382  +            "ORDER BY level DESC, idx ASC",
          383  +
          384  +          /* Update statements used while promoting segments */
          385  +/* 38 */  "UPDATE OR FAIL %Q.'%q_segdir' SET level=-1,idx=? "
          386  +            "WHERE level=? AND idx=?",
          387  +/* 39 */  "UPDATE OR FAIL %Q.'%q_segdir' SET level=? WHERE level=-1"
          388  +
   373    389     };
   374    390     int rc = SQLITE_OK;
   375    391     sqlite3_stmt *pStmt;
   376    392   
   377    393     assert( SizeofArray(azSql)==SizeofArray(p->aStmt) );
   378    394     assert( eStmt<SizeofArray(azSql) && eStmt>=0 );
   379    395     
................................................................................
  1906   1922   static int fts3WriteSegdir(
  1907   1923     Fts3Table *p,                   /* Virtual table handle */
  1908   1924     sqlite3_int64 iLevel,           /* Value for "level" field (absolute level) */
  1909   1925     int iIdx,                       /* Value for "idx" field */
  1910   1926     sqlite3_int64 iStartBlock,      /* Value for "start_block" field */
  1911   1927     sqlite3_int64 iLeafEndBlock,    /* Value for "leaves_end_block" field */
  1912   1928     sqlite3_int64 iEndBlock,        /* Value for "end_block" field */
         1929  +  sqlite3_int64 nLeafData,        /* Bytes of leaf data in segment */
  1913   1930     char *zRoot,                    /* Blob value for "root" field */
  1914   1931     int nRoot                       /* Number of bytes in buffer zRoot */
  1915   1932   ){
  1916   1933     sqlite3_stmt *pStmt;
  1917   1934     int rc = fts3SqlStmt(p, SQL_INSERT_SEGDIR, &pStmt, 0);
  1918   1935     if( rc==SQLITE_OK ){
  1919   1936       sqlite3_bind_int64(pStmt, 1, iLevel);
  1920   1937       sqlite3_bind_int(pStmt, 2, iIdx);
  1921   1938       sqlite3_bind_int64(pStmt, 3, iStartBlock);
  1922   1939       sqlite3_bind_int64(pStmt, 4, iLeafEndBlock);
  1923         -    sqlite3_bind_int64(pStmt, 5, iEndBlock);
         1940  +    if( nLeafData==0 ){
         1941  +      sqlite3_bind_int64(pStmt, 5, iEndBlock);
         1942  +    }else{
         1943  +      char *zEnd = sqlite3_mprintf("%lld %lld", iEndBlock, nLeafData);
         1944  +      if( !zEnd ) return SQLITE_NOMEM;
         1945  +      sqlite3_bind_text(pStmt, 5, zEnd, -1, sqlite3_free);
         1946  +    }
  1924   1947       sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC);
  1925   1948       sqlite3_step(pStmt);
  1926   1949       rc = sqlite3_reset(pStmt);
  1927   1950     }
  1928   1951     return rc;
  1929   1952   }
  1930   1953   
................................................................................
  2241   2264       nSuffix = nTerm;
  2242   2265       nReq = 1 +                              /* varint containing prefix size */
  2243   2266         sqlite3Fts3VarintLen(nTerm) +         /* varint containing suffix size */
  2244   2267         nTerm +                               /* Term suffix */
  2245   2268         sqlite3Fts3VarintLen(nDoclist) +      /* Size of doclist */
  2246   2269         nDoclist;                             /* Doclist data */
  2247   2270     }
         2271  +
         2272  +  /* Increase the total number of bytes written to account for the new entry. */
         2273  +  pWriter->nLeafData += nReq;
  2248   2274   
  2249   2275     /* If the buffer currently allocated is too small for this entry, realloc
  2250   2276     ** the buffer to make it large enough.
  2251   2277     */
  2252   2278     if( nReq>pWriter->nSize ){
  2253   2279       char *aNew = sqlite3_realloc(pWriter->aData, nReq);
  2254   2280       if( !aNew ) return SQLITE_NOMEM;
................................................................................
  2313   2339       iLastLeaf = pWriter->iFree;
  2314   2340       rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, pWriter->nData);
  2315   2341       if( rc==SQLITE_OK ){
  2316   2342         rc = fts3NodeWrite(p, pWriter->pTree, 1,
  2317   2343             pWriter->iFirst, pWriter->iFree, &iLast, &zRoot, &nRoot);
  2318   2344       }
  2319   2345       if( rc==SQLITE_OK ){
  2320         -      rc = fts3WriteSegdir(
  2321         -          p, iLevel, iIdx, pWriter->iFirst, iLastLeaf, iLast, zRoot, nRoot);
         2346  +      rc = fts3WriteSegdir(p, iLevel, iIdx, 
         2347  +          pWriter->iFirst, iLastLeaf, iLast, pWriter->nLeafData, zRoot, nRoot);
  2322   2348       }
  2323   2349     }else{
  2324   2350       /* The entire tree fits on the root node. Write it to the segdir table. */
  2325         -    rc = fts3WriteSegdir(
  2326         -        p, iLevel, iIdx, 0, 0, 0, pWriter->aData, pWriter->nData);
         2351  +    rc = fts3WriteSegdir(p, iLevel, iIdx, 
         2352  +        0, 0, 0, pWriter->nLeafData, pWriter->aData, pWriter->nData);
  2327   2353     }
  2328   2354     p->nLeafAdd++;
  2329   2355     return rc;
  2330   2356   }
  2331   2357   
  2332   2358   /*
  2333   2359   ** Release all memory held by the SegmentWriter object passed as the 
................................................................................
  2402   2428         getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1)
  2403   2429     );
  2404   2430     if( SQLITE_ROW==sqlite3_step(pStmt) ){
  2405   2431       *pnMax = sqlite3_column_int64(pStmt, 0);
  2406   2432     }
  2407   2433     return sqlite3_reset(pStmt);
  2408   2434   }
         2435  +
         2436  +/*
         2437  +** iAbsLevel is an absolute level that may be assumed to exist within
         2438  +** the database. This function checks if it is the largest level number
         2439  +** within its index. Assuming no error occurs, *pbMax is set to 1 if
         2440  +** iAbsLevel is indeed the largest level, or 0 otherwise, and SQLITE_OK
         2441  +** is returned. If an error occurs, an error code is returned and the
         2442  +** final value of *pbMax is undefined.
         2443  +*/
         2444  +static int fts3SegmentIsMaxLevel(Fts3Table *p, i64 iAbsLevel, int *pbMax){
         2445  +
         2446  +  /* Set pStmt to the compiled version of:
         2447  +  **
         2448  +  **   SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?
         2449  +  **
         2450  +  ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR).
         2451  +  */
         2452  +  sqlite3_stmt *pStmt;
         2453  +  int rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0);
         2454  +  if( rc!=SQLITE_OK ) return rc;
         2455  +  sqlite3_bind_int64(pStmt, 1, iAbsLevel+1);
         2456  +  sqlite3_bind_int64(pStmt, 2, 
         2457  +      ((iAbsLevel/FTS3_SEGDIR_MAXLEVEL)+1) * FTS3_SEGDIR_MAXLEVEL
         2458  +  );
         2459  +
         2460  +  *pbMax = 0;
         2461  +  if( SQLITE_ROW==sqlite3_step(pStmt) ){
         2462  +    *pbMax = sqlite3_column_type(pStmt, 0)==SQLITE_NULL;
         2463  +  }
         2464  +  return sqlite3_reset(pStmt);
         2465  +}
  2409   2466   
  2410   2467   /*
  2411   2468   ** Delete all entries in the %_segments table associated with the segment
  2412   2469   ** opened with seg-reader pSeg. This function does not affect the contents
  2413   2470   ** of the %_segdir table.
  2414   2471   */
  2415   2472   static int fts3DeleteSegment(
................................................................................
  2937   2994       sqlite3_free(pCsr->aBuffer);
  2938   2995   
  2939   2996       pCsr->nSegment = 0;
  2940   2997       pCsr->apSegment = 0;
  2941   2998       pCsr->aBuffer = 0;
  2942   2999     }
  2943   3000   }
         3001  +
         3002  +/*
         3003  +** Decode the "end_block" field, selected by column iCol of the SELECT 
         3004  +** statement passed as the first argument. 
         3005  +**
         3006  +** The "end_block" field may contain either an integer, or a text field
         3007  +** containing the text representation of two non-negative integers separated 
         3008  +** by one or more space (0x20) characters. In the first case, set *piEndBlock 
         3009  +** to the integer value and *pnByte to zero before returning. In the second, 
         3010  +** set *piEndBlock to the first value and *pnByte to the second.
         3011  +*/
         3012  +static void fts3ReadEndBlockField(
         3013  +  sqlite3_stmt *pStmt, 
         3014  +  int iCol, 
         3015  +  i64 *piEndBlock,
         3016  +  i64 *pnByte
         3017  +){
         3018  +  const unsigned char *zText = sqlite3_column_text(pStmt, iCol);
         3019  +  if( zText ){
         3020  +    int i;
         3021  +    int iMul = 1;
         3022  +    i64 iVal = 0;
         3023  +    for(i=0; zText[i]>='0' && zText[i]<='9'; i++){
         3024  +      iVal = iVal*10 + (zText[i] - '0');
         3025  +    }
         3026  +    *piEndBlock = iVal;
         3027  +    while( zText[i]==' ' ) i++;
         3028  +    iVal = 0;
         3029  +    if( zText[i]=='-' ){
         3030  +      i++;
         3031  +      iMul = -1;
         3032  +    }
         3033  +    for(/* no-op */; zText[i]>='0' && zText[i]<='9'; i++){
         3034  +      iVal = iVal*10 + (zText[i] - '0');
         3035  +    }
         3036  +    *pnByte = (iVal * (i64)iMul);
         3037  +  }
         3038  +}
         3039  +
         3040  +
         3041  +/*
         3042  +** A segment of size nByte bytes has just been written to absolute level
         3043  +** iAbsLevel. Promote any segments that should be promoted as a result.
         3044  +*/
         3045  +static int fts3PromoteSegments(
         3046  +  Fts3Table *p,                   /* FTS table handle */
         3047  +  int iAbsLevel,                  /* Absolute level just updated */
         3048  +  sqlite3_int64 nByte             /* Size of new segment at iAbsLevel */
         3049  +){
         3050  +  int rc = SQLITE_OK;
         3051  +  sqlite3_stmt *pRange;
         3052  +
         3053  +  rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE2, &pRange, 0);
         3054  +
         3055  +  if( rc==SQLITE_OK ){
         3056  +    int bOk = 0;
         3057  +    int iLast = (iAbsLevel/FTS3_SEGDIR_MAXLEVEL + 1) * FTS3_SEGDIR_MAXLEVEL - 1;
         3058  +    i64 nLimit = (nByte*3)/2;
         3059  +
         3060  +    /* Loop through all entries in the %_segdir table corresponding to 
         3061  +    ** segments in this index on levels greater than iAbsLevel. If there is
         3062  +    ** at least one such segment, and it is possible to determine that all 
         3063  +    ** such segments are smaller than nLimit bytes in size, they will be 
         3064  +    ** promoted to level iAbsLevel.  */
         3065  +    sqlite3_bind_int(pRange, 1, iAbsLevel+1);
         3066  +    sqlite3_bind_int(pRange, 2, iLast);
         3067  +    while( SQLITE_ROW==sqlite3_step(pRange) ){
         3068  +      i64 nSize, dummy;
         3069  +      fts3ReadEndBlockField(pRange, 2, &dummy, &nSize);
         3070  +      if( nSize<=0 || nSize>nLimit ){
         3071  +        /* If nSize==0, then the %_segdir.end_block field does not not 
         3072  +        ** contain a size value. This happens if it was written by an
         3073  +        ** old version of FTS. In this case it is not possible to determine
         3074  +        ** the size of the segment, and so segment promotion does not
         3075  +        ** take place.  */
         3076  +        bOk = 0;
         3077  +        break;
         3078  +      }
         3079  +      bOk = 1;
         3080  +    }
         3081  +    rc = sqlite3_reset(pRange);
         3082  +
         3083  +    if( bOk ){
         3084  +      int iIdx = 0;
         3085  +      sqlite3_stmt *pUpdate1;
         3086  +      sqlite3_stmt *pUpdate2;
         3087  +
         3088  +      if( rc==SQLITE_OK ){
         3089  +        rc = fts3SqlStmt(p, SQL_UPDATE_LEVEL_IDX, &pUpdate1, 0);
         3090  +      }
         3091  +      if( rc==SQLITE_OK ){
         3092  +        rc = fts3SqlStmt(p, SQL_UPDATE_LEVEL, &pUpdate2, 0);
         3093  +      }
         3094  +
         3095  +      if( rc==SQLITE_OK ){
         3096  +
         3097  +        /* Loop through all %_segdir entries for segments in this index with
         3098  +        ** levels equal to or greater than iAbsLevel. As each entry is visited,
         3099  +        ** updated it to set (level = -1) and (idx = N), where N is 0 for the
         3100  +        ** oldest segment in the range, 1 for the next oldest, and so on.
         3101  +        **
         3102  +        ** In other words, move all segments being promoted to level -1,
         3103  +        ** setting the "idx" fields as appropriate to keep them in the same
         3104  +        ** order. The contents of level -1 (which is never used, except
         3105  +        ** transiently here), will be moved back to level iAbsLevel below.  */
         3106  +        sqlite3_bind_int(pRange, 1, iAbsLevel);
         3107  +        while( SQLITE_ROW==sqlite3_step(pRange) ){
         3108  +          sqlite3_bind_int(pUpdate1, 1, iIdx++);
         3109  +          sqlite3_bind_int(pUpdate1, 2, sqlite3_column_int(pRange, 0));
         3110  +          sqlite3_bind_int(pUpdate1, 3, sqlite3_column_int(pRange, 1));
         3111  +          sqlite3_step(pUpdate1);
         3112  +          rc = sqlite3_reset(pUpdate1);
         3113  +          if( rc!=SQLITE_OK ){
         3114  +            sqlite3_reset(pRange);
         3115  +            break;
         3116  +          }
         3117  +        }
         3118  +      }
         3119  +      if( rc==SQLITE_OK ){
         3120  +        rc = sqlite3_reset(pRange);
         3121  +      }
         3122  +
         3123  +      /* Move level -1 to level iAbsLevel */
         3124  +      if( rc==SQLITE_OK ){
         3125  +        sqlite3_bind_int(pUpdate2, 1, iAbsLevel);
         3126  +        sqlite3_step(pUpdate2);
         3127  +        rc = sqlite3_reset(pUpdate2);
         3128  +      }
         3129  +    }
         3130  +  }
         3131  +
         3132  +
         3133  +  return rc;
         3134  +}
  2944   3135   
  2945   3136   /*
  2946   3137   ** Merge all level iLevel segments in the database into a single 
  2947   3138   ** iLevel+1 segment. Or, if iLevel<0, merge all segments into a
  2948   3139   ** single segment with a level equal to the numerically largest level 
  2949   3140   ** currently present in the database.
  2950   3141   **
................................................................................
  2962   3153     int rc;                         /* Return code */
  2963   3154     int iIdx = 0;                   /* Index of new segment */
  2964   3155     sqlite3_int64 iNewLevel = 0;    /* Level/index to create new segment at */
  2965   3156     SegmentWriter *pWriter = 0;     /* Used to write the new, merged, segment */
  2966   3157     Fts3SegFilter filter;           /* Segment term filter condition */
  2967   3158     Fts3MultiSegReader csr;         /* Cursor to iterate through level(s) */
  2968   3159     int bIgnoreEmpty = 0;           /* True to ignore empty segments */
         3160  +  i64 iMaxLevel = 0;              /* Max level number for this index/langid */
  2969   3161   
  2970   3162     assert( iLevel==FTS3_SEGCURSOR_ALL
  2971   3163          || iLevel==FTS3_SEGCURSOR_PENDING
  2972   3164          || iLevel>=0
  2973   3165     );
  2974   3166     assert( iLevel<FTS3_SEGDIR_MAXLEVEL );
  2975   3167     assert( iIndex>=0 && iIndex<p->nIndex );
  2976   3168   
  2977   3169     rc = sqlite3Fts3SegReaderCursor(p, iLangid, iIndex, iLevel, 0, 0, 1, 0, &csr);
  2978   3170     if( rc!=SQLITE_OK || csr.nSegment==0 ) goto finished;
         3171  +
         3172  +  if( iLevel!=FTS3_SEGCURSOR_PENDING ){
         3173  +    rc = fts3SegmentMaxLevel(p, iLangid, iIndex, &iMaxLevel);
         3174  +    if( rc!=SQLITE_OK ) goto finished;
         3175  +  }
  2979   3176   
  2980   3177     if( iLevel==FTS3_SEGCURSOR_ALL ){
  2981   3178       /* This call is to merge all segments in the database to a single
  2982   3179       ** segment. The level of the new segment is equal to the numerically
  2983   3180       ** greatest segment level currently present in the database for this
  2984   3181       ** index. The idx of the new segment is always 0.  */
  2985   3182       if( csr.nSegment==1 ){
  2986   3183         rc = SQLITE_DONE;
  2987   3184         goto finished;
  2988   3185       }
  2989         -    rc = fts3SegmentMaxLevel(p, iLangid, iIndex, &iNewLevel);
         3186  +    iNewLevel = iMaxLevel;
  2990   3187       bIgnoreEmpty = 1;
  2991   3188   
  2992         -  }else if( iLevel==FTS3_SEGCURSOR_PENDING ){
  2993         -    iNewLevel = getAbsoluteLevel(p, iLangid, iIndex, 0);
  2994         -    rc = fts3AllocateSegdirIdx(p, iLangid, iIndex, 0, &iIdx);
  2995   3189     }else{
  2996   3190       /* This call is to merge all segments at level iLevel. find the next
  2997   3191       ** available segment index at level iLevel+1. The call to
  2998   3192       ** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to 
  2999   3193       ** a single iLevel+2 segment if necessary.  */
  3000         -    rc = fts3AllocateSegdirIdx(p, iLangid, iIndex, iLevel+1, &iIdx);
         3194  +    assert( FTS3_SEGCURSOR_PENDING==-1 );
  3001   3195       iNewLevel = getAbsoluteLevel(p, iLangid, iIndex, iLevel+1);
         3196  +    rc = fts3AllocateSegdirIdx(p, iLangid, iIndex, iLevel+1, &iIdx);
         3197  +    bIgnoreEmpty = (iLevel!=FTS3_SEGCURSOR_PENDING) && (iNewLevel>iMaxLevel);
  3002   3198     }
  3003   3199     if( rc!=SQLITE_OK ) goto finished;
         3200  +
  3004   3201     assert( csr.nSegment>0 );
  3005   3202     assert( iNewLevel>=getAbsoluteLevel(p, iLangid, iIndex, 0) );
  3006   3203     assert( iNewLevel<getAbsoluteLevel(p, iLangid, iIndex,FTS3_SEGDIR_MAXLEVEL) );
  3007   3204   
  3008   3205     memset(&filter, 0, sizeof(Fts3SegFilter));
  3009   3206     filter.flags = FTS3_SEGMENT_REQUIRE_POS;
  3010   3207     filter.flags |= (bIgnoreEmpty ? FTS3_SEGMENT_IGNORE_EMPTY : 0);
................................................................................
  3013   3210     while( SQLITE_OK==rc ){
  3014   3211       rc = sqlite3Fts3SegReaderStep(p, &csr);
  3015   3212       if( rc!=SQLITE_ROW ) break;
  3016   3213       rc = fts3SegWriterAdd(p, &pWriter, 1, 
  3017   3214           csr.zTerm, csr.nTerm, csr.aDoclist, csr.nDoclist);
  3018   3215     }
  3019   3216     if( rc!=SQLITE_OK ) goto finished;
  3020         -  assert( pWriter );
         3217  +  assert( pWriter || bIgnoreEmpty );
  3021   3218   
  3022   3219     if( iLevel!=FTS3_SEGCURSOR_PENDING ){
  3023   3220       rc = fts3DeleteSegdir(
  3024   3221           p, iLangid, iIndex, iLevel, csr.apSegment, csr.nSegment
  3025   3222       );
  3026   3223       if( rc!=SQLITE_OK ) goto finished;
  3027   3224     }
  3028         -  rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx);
         3225  +  if( pWriter ){
         3226  +    rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx);
         3227  +    if( rc==SQLITE_OK ){
         3228  +      if( iLevel==FTS3_SEGCURSOR_PENDING || iNewLevel<iMaxLevel ){
         3229  +        rc = fts3PromoteSegments(p, iNewLevel, pWriter->nLeafData);
         3230  +      }
         3231  +    }
         3232  +  }
  3029   3233   
  3030   3234    finished:
  3031   3235     fts3SegWriterFree(pWriter);
  3032   3236     sqlite3Fts3SegReaderFinish(&csr);
  3033   3237     return rc;
  3034   3238   }
  3035   3239   
  3036   3240   
  3037   3241   /* 
  3038         -** Flush the contents of pendingTerms to level 0 segments.
         3242  +** Flush the contents of pendingTerms to level 0 segments. 
  3039   3243   */
  3040   3244   int sqlite3Fts3PendingTermsFlush(Fts3Table *p){
  3041   3245     int rc = SQLITE_OK;
  3042   3246     int i;
  3043   3247           
  3044   3248     for(i=0; rc==SQLITE_OK && i<p->nIndex; i++){
  3045   3249       rc = fts3SegmentMerge(p, p->iPrevLangid, i, FTS3_SEGCURSOR_PENDING);
................................................................................
  3047   3251     }
  3048   3252     sqlite3Fts3PendingTermsClear(p);
  3049   3253   
  3050   3254     /* Determine the auto-incr-merge setting if unknown.  If enabled,
  3051   3255     ** estimate the number of leaf blocks of content to be written
  3052   3256     */
  3053   3257     if( rc==SQLITE_OK && p->bHasStat
  3054         -   && p->bAutoincrmerge==0xff && p->nLeafAdd>0
         3258  +   && p->nAutoincrmerge==0xff && p->nLeafAdd>0
  3055   3259     ){
  3056   3260       sqlite3_stmt *pStmt = 0;
  3057   3261       rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pStmt, 0);
  3058   3262       if( rc==SQLITE_OK ){
  3059   3263         sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE);
  3060   3264         rc = sqlite3_step(pStmt);
  3061         -      p->bAutoincrmerge = (rc==SQLITE_ROW && sqlite3_column_int(pStmt, 0));
         3265  +      if( rc==SQLITE_ROW ){
         3266  +        p->nAutoincrmerge = sqlite3_column_int(pStmt, 0);
         3267  +        if( p->nAutoincrmerge==1 ) p->nAutoincrmerge = 8;
         3268  +      }else if( rc==SQLITE_DONE ){
         3269  +        p->nAutoincrmerge = 0;
         3270  +      }
  3062   3271         rc = sqlite3_reset(pStmt);
  3063   3272       }
  3064   3273     }
  3065   3274     return rc;
  3066   3275   }
  3067   3276   
  3068   3277   /*
................................................................................
  3422   3631   struct IncrmergeWriter {
  3423   3632     int nLeafEst;                   /* Space allocated for leaf blocks */
  3424   3633     int nWork;                      /* Number of leaf pages flushed */
  3425   3634     sqlite3_int64 iAbsLevel;        /* Absolute level of input segments */
  3426   3635     int iIdx;                       /* Index of *output* segment in iAbsLevel+1 */
  3427   3636     sqlite3_int64 iStart;           /* Block number of first allocated block */
  3428   3637     sqlite3_int64 iEnd;             /* Block number of last allocated block */
         3638  +  sqlite3_int64 nLeafData;        /* Bytes of leaf page data so far */
         3639  +  u8 bNoLeafData;                 /* If true, store 0 for segment size */
  3429   3640     NodeWriter aNodeWriter[FTS_MAX_APPENDABLE_HEIGHT];
  3430   3641   };
  3431   3642   
  3432   3643   /*
  3433   3644   ** An object of the following type is used to read data from a single
  3434   3645   ** FTS segment node. See the following functions:
  3435   3646   **
................................................................................
  3760   3971   
  3761   3972       nSuffix = nTerm;
  3762   3973       nSpace  = 1;
  3763   3974       nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix;
  3764   3975       nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist;
  3765   3976     }
  3766   3977   
         3978  +  pWriter->nLeafData += nSpace;
  3767   3979     blobGrowBuffer(&pLeaf->block, pLeaf->block.n + nSpace, &rc);
  3768         -
  3769   3980     if( rc==SQLITE_OK ){
  3770   3981       if( pLeaf->block.n==0 ){
  3771   3982         pLeaf->block.n = 1;
  3772   3983         pLeaf->block.a[0] = '\0';
  3773   3984       }
  3774   3985       rc = fts3AppendToNode(
  3775   3986           &pLeaf->block, &pLeaf->key, zTerm, nTerm, aDoclist, nDoclist
................................................................................
  3860   4071     if( rc==SQLITE_OK ){
  3861   4072       rc = fts3WriteSegdir(p, 
  3862   4073           pWriter->iAbsLevel+1,               /* level */
  3863   4074           pWriter->iIdx,                      /* idx */
  3864   4075           pWriter->iStart,                    /* start_block */
  3865   4076           pWriter->aNodeWriter[0].iBlock,     /* leaves_end_block */
  3866   4077           pWriter->iEnd,                      /* end_block */
         4078  +        (pWriter->bNoLeafData==0 ? pWriter->nLeafData : 0),   /* end_block */
  3867   4079           pRoot->block.a, pRoot->block.n      /* root */
  3868   4080       );
  3869   4081     }
  3870   4082     sqlite3_free(pRoot->block.a);
  3871   4083     sqlite3_free(pRoot->key.a);
  3872   4084   
  3873   4085     *pRc = rc;
................................................................................
  3961   4173   
  3962   4174       /* Read the %_segdir entry for index iIdx absolute level (iAbsLevel+1) */
  3963   4175       sqlite3_bind_int64(pSelect, 1, iAbsLevel+1);
  3964   4176       sqlite3_bind_int(pSelect, 2, iIdx);
  3965   4177       if( sqlite3_step(pSelect)==SQLITE_ROW ){
  3966   4178         iStart = sqlite3_column_int64(pSelect, 1);
  3967   4179         iLeafEnd = sqlite3_column_int64(pSelect, 2);
  3968         -      iEnd = sqlite3_column_int64(pSelect, 3);
         4180  +      fts3ReadEndBlockField(pSelect, 3, &iEnd, &pWriter->nLeafData);
         4181  +      if( pWriter->nLeafData<0 ){
         4182  +        pWriter->nLeafData = pWriter->nLeafData * -1;
         4183  +      }
         4184  +      pWriter->bNoLeafData = (pWriter->nLeafData==0);
  3969   4185         nRoot = sqlite3_column_bytes(pSelect, 4);
  3970   4186         aRoot = sqlite3_column_blob(pSelect, 4);
  3971   4187       }else{
  3972   4188         return sqlite3_reset(pSelect);
  3973   4189       }
  3974   4190   
  3975   4191       /* Check for the zero-length marker in the %_segments table */
................................................................................
  4562   4778     return SQLITE_OK;
  4563   4779   }
  4564   4780   
  4565   4781   
  4566   4782   /*
  4567   4783   ** Attempt an incremental merge that writes nMerge leaf blocks.
  4568   4784   **
  4569         -** Incremental merges happen nMin segments at a time. The two
  4570         -** segments to be merged are the nMin oldest segments (the ones with
  4571         -** the smallest indexes) in the highest level that contains at least
  4572         -** nMin segments. Multiple merges might occur in an attempt to write the 
  4573         -** quota of nMerge leaf blocks.
         4785  +** Incremental merges happen nMin segments at a time. The segments 
         4786  +** to be merged are the nMin oldest segments (the ones with the smallest 
         4787  +** values for the _segdir.idx field) in the highest level that contains 
         4788  +** at least nMin segments. Multiple merges might occur in an attempt to 
         4789  +** write the quota of nMerge leaf blocks.
  4574   4790   */
  4575   4791   int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){
  4576   4792     int rc;                         /* Return code */
  4577   4793     int nRem = nMerge;              /* Number of leaf pages yet to  be written */
  4578   4794     Fts3MultiSegReader *pCsr;       /* Cursor used to read input data */
  4579   4795     Fts3SegFilter *pFilter;         /* Filter used with cursor pCsr */
  4580   4796     IncrmergeWriter *pWriter;       /* Writer object */
................................................................................
  4591   4807     pCsr = (Fts3MultiSegReader *)&pFilter[1];
  4592   4808   
  4593   4809     rc = fts3IncrmergeHintLoad(p, &hint);
  4594   4810     while( rc==SQLITE_OK && nRem>0 ){
  4595   4811       const i64 nMod = FTS3_SEGDIR_MAXLEVEL * p->nIndex;
  4596   4812       sqlite3_stmt *pFindLevel = 0; /* SQL used to determine iAbsLevel */
  4597   4813       int bUseHint = 0;             /* True if attempting to append */
         4814  +    int iIdx = 0;                 /* Largest idx in level (iAbsLevel+1) */
  4598   4815   
  4599   4816       /* Search the %_segdir table for the absolute level with the smallest
  4600   4817       ** relative level number that contains at least nMin segments, if any.
  4601   4818       ** If one is found, set iAbsLevel to the absolute level number and
  4602   4819       ** nSeg to nMin. If no level with at least nMin segments can be found, 
  4603   4820       ** set nSeg to -1.
  4604   4821       */
................................................................................
  4644   4861       ** indexes of absolute level iAbsLevel. If this cursor is opened using 
  4645   4862       ** the 'hint' parameters, it is possible that there are less than nSeg
  4646   4863       ** segments available in level iAbsLevel. In this case, no work is
  4647   4864       ** done on iAbsLevel - fall through to the next iteration of the loop 
  4648   4865       ** to start work on some other level.  */
  4649   4866       memset(pWriter, 0, nAlloc);
  4650   4867       pFilter->flags = FTS3_SEGMENT_REQUIRE_POS;
         4868  +
         4869  +    if( rc==SQLITE_OK ){
         4870  +      rc = fts3IncrmergeOutputIdx(p, iAbsLevel, &iIdx);
         4871  +      assert( bUseHint==1 || bUseHint==0 );
         4872  +      if( iIdx==0 || (bUseHint && iIdx==1) ){
         4873  +        int bIgnore;
         4874  +        rc = fts3SegmentIsMaxLevel(p, iAbsLevel+1, &bIgnore);
         4875  +        if( bIgnore ){
         4876  +          pFilter->flags |= FTS3_SEGMENT_IGNORE_EMPTY;
         4877  +        }
         4878  +      }
         4879  +    }
         4880  +
  4651   4881       if( rc==SQLITE_OK ){
  4652   4882         rc = fts3IncrmergeCsr(p, iAbsLevel, nSeg, pCsr);
  4653   4883       }
  4654   4884       if( SQLITE_OK==rc && pCsr->nSegment==nSeg
  4655   4885        && SQLITE_OK==(rc = sqlite3Fts3SegReaderStart(p, pCsr, pFilter))
  4656   4886        && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pCsr))
  4657   4887       ){
  4658         -      int iIdx = 0;               /* Largest idx in level (iAbsLevel+1) */
  4659         -      rc = fts3IncrmergeOutputIdx(p, iAbsLevel, &iIdx);
  4660         -      if( rc==SQLITE_OK ){
  4661         -        if( bUseHint && iIdx>0 ){
  4662         -          const char *zKey = pCsr->zTerm;
  4663         -          int nKey = pCsr->nTerm;
  4664         -          rc = fts3IncrmergeLoad(p, iAbsLevel, iIdx-1, zKey, nKey, pWriter);
  4665         -        }else{
  4666         -          rc = fts3IncrmergeWriter(p, iAbsLevel, iIdx, pCsr, pWriter);
  4667         -        }
         4888  +      if( bUseHint && iIdx>0 ){
         4889  +        const char *zKey = pCsr->zTerm;
         4890  +        int nKey = pCsr->nTerm;
         4891  +        rc = fts3IncrmergeLoad(p, iAbsLevel, iIdx-1, zKey, nKey, pWriter);
         4892  +      }else{
         4893  +        rc = fts3IncrmergeWriter(p, iAbsLevel, iIdx, pCsr, pWriter);
  4668   4894         }
  4669   4895   
  4670   4896         if( rc==SQLITE_OK && pWriter->nLeafEst ){
  4671   4897           fts3LogMerge(nSeg, iAbsLevel);
  4672   4898           do {
  4673   4899             rc = fts3IncrmergeAppend(p, pWriter, pCsr);
  4674   4900             if( rc==SQLITE_OK ) rc = sqlite3Fts3SegReaderStep(p, pCsr);
................................................................................
  4682   4908             if( nSeg!=0 ){
  4683   4909               bDirtyHint = 1;
  4684   4910               fts3IncrmergeHintPush(&hint, iAbsLevel, nSeg, &rc);
  4685   4911             }
  4686   4912           }
  4687   4913         }
  4688   4914   
         4915  +      if( nSeg!=0 ){
         4916  +        pWriter->nLeafData = pWriter->nLeafData * -1;
         4917  +      }
  4689   4918         fts3IncrmergeRelease(p, pWriter, &rc);
         4919  +      if( nSeg==0 && pWriter->bNoLeafData==0 ){
         4920  +        fts3PromoteSegments(p, iAbsLevel+1, pWriter->nLeafData);
         4921  +      }
  4690   4922       }
  4691   4923   
  4692   4924       sqlite3Fts3SegReaderFinish(pCsr);
  4693   4925     }
  4694   4926   
  4695   4927     /* Write the hint values into the %_stat table for the next incr-merger */
  4696   4928     if( bDirtyHint && rc==SQLITE_OK ){
................................................................................
  4769   5001   */
  4770   5002   static int fts3DoAutoincrmerge(
  4771   5003     Fts3Table *p,                   /* FTS3 table handle */
  4772   5004     const char *zParam              /* Nul-terminated string containing boolean */
  4773   5005   ){
  4774   5006     int rc = SQLITE_OK;
  4775   5007     sqlite3_stmt *pStmt = 0;
  4776         -  p->bAutoincrmerge = fts3Getint(&zParam)!=0;
         5008  +  p->nAutoincrmerge = fts3Getint(&zParam);
         5009  +  if( p->nAutoincrmerge==1 || p->nAutoincrmerge>FTS3_MERGE_COUNT ){
         5010  +    p->nAutoincrmerge = 8;
         5011  +  }
  4777   5012     if( !p->bHasStat ){
  4778   5013       assert( p->bFts4==0 );
  4779   5014       sqlite3Fts3CreateStatTable(&rc, p);
  4780   5015       if( rc ) return rc;
  4781   5016     }
  4782   5017     rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pStmt, 0);
  4783   5018     if( rc ) return rc;
  4784   5019     sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE);
  4785         -  sqlite3_bind_int(pStmt, 2, p->bAutoincrmerge);
         5020  +  sqlite3_bind_int(pStmt, 2, p->nAutoincrmerge);
  4786   5021     sqlite3_step(pStmt);
  4787   5022     rc = sqlite3_reset(pStmt);
  4788   5023     return rc;
  4789   5024   }
  4790   5025   
  4791   5026   /*
  4792   5027   ** Return a 64-bit checksum for the FTS index entry specified by the

Changes to test/backcompat.test.

    53     53     }
    54     54   
    55     55     proc sql1 sql { code1 [list db eval $sql] }
    56     56     proc sql2 sql { code2 [list db eval $sql] }
    57     57   
    58     58     code1 { sqlite3 db test.db }
    59     59     code2 { sqlite3 db test.db }
           60  +
           61  +  foreach c {code1 code2} {
           62  +    $c {
           63  +      set v [split [db version] .]
           64  +      if {[llength $v]==3} {lappend v 0}
           65  +      set ::sqlite_libversion [format \
           66  +        "%d%.2d%.2d%2d" [lindex $v 0] [lindex $v 1] [lindex $v 2] [lindex $v 3]
           67  +      ]
           68  +    }
           69  +  }
    60     70   
    61     71     uplevel $script
    62     72   
    63     73     catch { code1 { db close } }
    64     74     catch { code2 { db close } }
    65     75     catch { close $::bc_chan2 }
    66     76     catch { close $::bc_chan1 }
           77  +
           78  +
    67     79   }
    68     80   
    69     81   array set ::incompatible [list]
    70     82   proc do_allbackcompat_test {script} {
    71     83   
    72     84     foreach bin $::BC(binaries) {
    73     85       set nErr [set_test_counter errors]
................................................................................
   377    389     
   378    390           6    "SELECT offsets(t1) FROM t1 WHERE t1 MATCH 'aa'"
   379    391           7    "SELECT offsets(t1) FROM t1 WHERE t1 MATCH '44'"
   380    392           8    "SELECT offsets(t1) FROM t1 WHERE t1 MATCH 'a*'"
   381    393         } {
   382    394           do_test backcompat-3.7 [list sql1 $q] [sql2 $q]
   383    395         }
          396  +
          397  +      # Now test that an incremental merge can be started by one version
          398  +      # and finished by another. And that the integrity-check still 
          399  +      # passes.
          400  +      do_test backcompat-3.8 {
          401  +        sql1 { 
          402  +          DROP TABLE IF EXISTS t1;
          403  +          DROP TABLE IF EXISTS t2;
          404  +          CREATE TABLE t1(docid, words);
          405  +          CREATE VIRTUAL TABLE t2 USING fts3(words);
          406  +        }
          407  +        code1 [list source $testdir/genesis.tcl]
          408  +        code1 { fts_kjv_genesis }
          409  +        sql1 {
          410  +          INSERT INTO t2 SELECT words FROM t1;
          411  +          INSERT INTO t2 SELECT words FROM t1;
          412  +          INSERT INTO t2 SELECT words FROM t1;
          413  +          INSERT INTO t2 SELECT words FROM t1;
          414  +          INSERT INTO t2 SELECT words FROM t1;
          415  +          INSERT INTO t2 SELECT words FROM t1;
          416  +          SELECT level, group_concat(idx, ' ') FROM t2_segdir GROUP BY level;
          417  +        }
          418  +      } {0 {0 1 2 3 4 5}}
          419  +
          420  +      if {[code1 { set ::sqlite_libversion }] >=3071200 
          421  +       && [code2 { set ::sqlite_libversion }] >=3071200 
          422  +      } {
          423  +        do_test backcompat-3.9 {
          424  +          sql1 { INSERT INTO t2(t2) VALUES('merge=100,4'); }
          425  +          sql2 { INSERT INTO t2(t2) VALUES('merge=100,4'); }
          426  +          sql1 { INSERT INTO t2(t2) VALUES('merge=100,4'); }
          427  +          sql2 { INSERT INTO t2(t2) VALUES('merge=2500,4'); }
          428  +          sql2 {
          429  +            SELECT level, group_concat(idx, ' ') FROM t2_segdir GROUP BY level;
          430  +          }
          431  +        } {0 {0 1} 1 0}
          432  +
          433  +        do_test backcompat-3.10 {
          434  +          sql1 { INSERT INTO t2(t2) VALUES('integrity-check') }
          435  +          sql2 { INSERT INTO t2(t2) VALUES('integrity-check') }
          436  +        } {}
          437  +      }
   384    438       }
   385    439     }
   386    440   }
   387    441   
   388    442   #-------------------------------------------------------------------------
   389    443   # Test that Rtree tables may be read/written by different versions of 
   390    444   # SQLite. 

Changes to test/fts3d.test.

   209    209       SELECT OFFSETS(t1) FROM t1
   210    210        WHERE t1 MATCH 'this OR that OR was OR a OR is OR test' ORDER BY docid;
   211    211     }
   212    212   } [list {0 0 0 4 0 4 5 2 0 3 8 1 0 5 10 4} \
   213    213           {0 1 0 4 0 2 5 3 0 3 9 1 0 5 11 4} \
   214    214           {0 0 0 4 0 4 5 2 0 3 8 1 0 5 10 4}]
   215    215   
   216         -check_terms_all fts3d-4.1      {a four is one test that this three two was}
          216  +puts [db eval {SELECT c FROM t1 } ]
          217  +check_terms_all fts3d-4.1      {a four is test that this was}
   217    218   check_doclist_all fts3d-4.1.1  a {[1 0[2]] [2 0[2]] [3 0[2]]}
   218    219   check_doclist_all fts3d-4.1.2  four {}
   219    220   check_doclist_all fts3d-4.1.3  is {[1 0[1]] [3 0[1]]}
   220         -check_doclist_all fts3d-4.1.4  one {}
          221  +#check_doclist_all fts3d-4.1.4  one {}
   221    222   check_doclist_all fts3d-4.1.5  test {[1 0[3]] [2 0[3]] [3 0[3]]}
   222    223   check_doclist_all fts3d-4.1.6  that {[2 0[0]]}
   223    224   check_doclist_all fts3d-4.1.7  this {[1 0[0]] [3 0[0]]}
   224         -check_doclist_all fts3d-4.1.8  three {}
   225         -check_doclist_all fts3d-4.1.9  two {}
          225  +#check_doclist_all fts3d-4.1.8  three {}
          226  +#check_doclist_all fts3d-4.1.9  two {}
   226    227   check_doclist_all fts3d-4.1.10 was {[2 0[1]]}
   227    228   
   228    229   check_terms fts3d-4.2     0 0 {a four test that was}
   229    230   check_doclist fts3d-4.2.1 0 0 a {[2 0[2]]}
   230    231   check_doclist fts3d-4.2.2 0 0 four {[2]}
   231    232   check_doclist fts3d-4.2.3 0 0 test {[2 0[3]]}
   232    233   check_doclist fts3d-4.2.4 0 0 that {[2 0[0]]}
................................................................................
   235    236   check_terms fts3d-4.3     0 1 {a four is test this}
   236    237   check_doclist fts3d-4.3.1 0 1 a {[3 0[2]]}
   237    238   check_doclist fts3d-4.3.2 0 1 four {[3]}
   238    239   check_doclist fts3d-4.3.3 0 1 is {[3 0[1]]}
   239    240   check_doclist fts3d-4.3.4 0 1 test {[3 0[3]]}
   240    241   check_doclist fts3d-4.3.5 0 1 this {[3 0[0]]}
   241    242   
   242         -check_terms fts3d-4.4      1 0 {a four is one test that this three two was}
          243  +check_terms fts3d-4.4      1 0 {a four is test that this was}
   243    244   check_doclist fts3d-4.4.1  1 0 a {[1 0[2]] [2 0[2]] [3 0[2]]}
   244         -check_doclist fts3d-4.4.2  1 0 four {[1] [2 0[4]] [3 0[4]]}
          245  +check_doclist fts3d-4.4.2  1 0 four {[2 0[4]] [3 0[4]]}
   245    246   check_doclist fts3d-4.4.3  1 0 is {[1 0[1]] [3 0[1]]}
   246         -check_doclist fts3d-4.4.4  1 0 one {[1] [2] [3]}
          247  +#check_doclist fts3d-4.4.4  1 0 one {[1] [2] [3]}
   247    248   check_doclist fts3d-4.4.5  1 0 test {[1 0[3]] [2 0[3]] [3 0[3]]}
   248    249   check_doclist fts3d-4.4.6  1 0 that {[2 0[0]]}
   249    250   check_doclist fts3d-4.4.7  1 0 this {[1 0[0]] [3 0[0]]}
   250         -check_doclist fts3d-4.4.8  1 0 three {[1] [2] [3]}
   251         -check_doclist fts3d-4.4.9  1 0 two {[1] [2] [3]}
          251  +#check_doclist fts3d-4.4.8  1 0 three {[1] [2] [3]}
          252  +#check_doclist fts3d-4.4.9  1 0 two {[1] [2] [3]}
   252    253   check_doclist fts3d-4.4.10 1 0 was {[2 0[1]]}
   253    254   
   254    255   # Optimize should leave the result in the level of the highest-level
   255    256   # prior segment.
   256    257   do_test fts3d-4.5 {
   257    258     execsql {
   258    259       SELECT OPTIMIZE(t1) FROM t1 LIMIT 1;

Added test/fts4growth.test.

            1  +# 2014 May 12
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this script is testing the FTS4 module.
           13  +#
           14  +#
           15  +
           16  +set testdir [file dirname $argv0]
           17  +source $testdir/tester.tcl
           18  +set testprefix fts4growth
           19  +
           20  +# If SQLITE_ENABLE_FTS3 is defined, omit this file.
           21  +ifcapable !fts3 {
           22  +  finish_test
           23  +  return
           24  +}
           25  +
           26  +source $testdir/genesis.tcl
           27  +
           28  +do_execsql_test 1.1 { CREATE VIRTUAL TABLE x1 USING fts3; }
           29  +
           30  +do_test 1.2 {
           31  +  foreach L {
           32  +    {"See here, young man," said Mulga Bill, "from Walgett to the sea,}
           33  +    {From Conroy's Gap to Castlereagh, there's none can ride like me.}
           34  +    {I'm good all round at everything as everybody knows,}
           35  +    {Although I'm not the one to talk -- I hate a man that blows.}
           36  +  } {
           37  +    execsql { INSERT INTO x1 VALUES($L) }
           38  +  }
           39  +  execsql { SELECT end_block, length(root) FROM x1_segdir }
           40  +} {{0 114} 114 {0 118} 118 {0 95} 95 {0 115} 115}
           41  +
           42  +do_execsql_test 1.3 {
           43  +  INSERT INTO x1(x1) VALUES('optimize');
           44  +  SELECT level, end_block, length(root) FROM x1_segdir;
           45  +} {0 {0 394} 394}
           46  +
           47  +do_test 1.4 {
           48  +  foreach L {
           49  +    {But riding is my special gift, my chiefest, sole delight;}
           50  +    {Just ask a wild duck can it swim, a wildcat can it fight.}
           51  +    {There's nothing clothed in hair or hide, or built of flesh or steel,}
           52  +    {There's nothing walks or jumps, or runs, on axle, hoof, or wheel,}
           53  +    {But what I'll sit, while hide will hold and girths and straps are tight:}
           54  +    {I'll ride this here two-wheeled concern right straight away at sight."}
           55  +  } {
           56  +    execsql { INSERT INTO x1 VALUES($L) }
           57  +  }
           58  +  execsql { 
           59  +    INSERT INTO x1(x1) VALUES('merge=4,4');
           60  +    SELECT level, end_block, length(root) FROM x1_segdir;
           61  +  }
           62  +} {0 {0 110} 110 0 {0 132} 132 0 {0 129} 129 1 {128 658} 2}
           63  +
           64  +do_execsql_test 1.5 {
           65  +  SELECT length(block) FROM x1_segments;
           66  +} {658 {}}
           67  +
           68  +do_test 1.6 {
           69  +  foreach L {
           70  +    {'Twas Mulga Bill, from Eaglehawk, that sought his own abode,}
           71  +    {That perched above Dead Man's Creek, beside the mountain road.}
           72  +    {He turned the cycle down the hill and mounted for the fray,}
           73  +    {But 'ere he'd gone a dozen yards it bolted clean away.}
           74  +    {It left the track, and through the trees, just like a silver steak,}
           75  +    {It whistled down the awful slope towards the Dead Man's Creek.}
           76  +    {It shaved a stump by half an inch, it dodged a big white-box:}
           77  +    {The very wallaroos in fright went scrambling up the rocks,}
           78  +    {The wombats hiding in their caves dug deeper underground,}
           79  +    {As Mulga Bill, as white as chalk, sat tight to every bound.}
           80  +    {It struck a stone and gave a spring that cleared a fallen tree,}
           81  +    {It raced beside a precipice as close as close could be;}
           82  +    {And then as Mulga Bill let out one last despairing shriek}
           83  +    {It made a leap of twenty feet into the Dead Man's Creek.}
           84  +  } {
           85  +    execsql { INSERT INTO x1 VALUES($L) }
           86  +  }
           87  +  execsql { 
           88  +    SELECT level, end_block, length(root) FROM x1_segdir;
           89  +  }
           90  +} {1 {128 658} 2 1 {130 1377} 6 0 {0 117} 117}
           91  +
           92  +do_execsql_test 1.7 {
           93  +  SELECT sum(length(block)) FROM x1_segments WHERE blockid IN (129, 130);
           94  +} {1377}
           95  +
           96  +#-------------------------------------------------------------------------
           97  +#
           98  +do_execsql_test 2.1 { 
           99  +  CREATE TABLE t1(docid, words);
          100  +  CREATE VIRTUAL TABLE x2 USING fts4;
          101  +}
          102  +fts_kjv_genesis 
          103  +do_test 2.2 {
          104  +  foreach id [db eval {SELECT docid FROM t1}] {
          105  +    execsql {
          106  +      INSERT INTO x2(docid, content) SELECT $id, words FROM t1 WHERE docid=$id
          107  +    }
          108  +  }
          109  +  foreach id [db eval {SELECT docid FROM t1}] {
          110  +    execsql {
          111  +      INSERT INTO x2(docid, content) SELECT NULL, words FROM t1 WHERE docid=$id
          112  +    }
          113  +    if {[db one {SELECT count(*) FROM x2_segdir WHERE level<2}]==2} break
          114  +  }
          115  +} {}
          116  +
          117  +do_execsql_test 2.3 { 
          118  +  SELECT count(*) FROM x2_segdir WHERE level=2;
          119  +  SELECT count(*) FROM x2_segdir WHERE level=3;
          120  +} {6 0}
          121  +
          122  +do_execsql_test 2.4 { 
          123  +  INSERT INTO x2(x2) VALUES('merge=4,4');
          124  +  SELECT count(*) FROM x2_segdir WHERE level=2;
          125  +  SELECT count(*) FROM x2_segdir WHERE level=3;
          126  +} {6 1}
          127  +
          128  +do_execsql_test 2.5 { 
          129  +  SELECT end_block FROM x2_segdir WHERE level=3;
          130  +  INSERT INTO x2(x2) VALUES('merge=4,4');
          131  +  SELECT end_block FROM x2_segdir WHERE level=3;
          132  +  INSERT INTO x2(x2) VALUES('merge=4,4');
          133  +  SELECT end_block FROM x2_segdir WHERE level=3;
          134  +} {{3828 -3430} {3828 -10191} {3828 -14109}}
          135  +
          136  +do_execsql_test 2.6 {
          137  +  SELECT sum(length(block)) FROM x2_segdir, x2_segments WHERE 
          138  +    blockid BETWEEN start_block AND leaves_end_block
          139  +    AND level=3
          140  +} {14109}
          141  +
          142  +do_execsql_test 2.7 { 
          143  +  INSERT INTO x2(x2) VALUES('merge=1000,4');
          144  +  SELECT end_block FROM x2_segdir WHERE level=3;
          145  +} {{3828 86120}}
          146  +
          147  +do_execsql_test 2.8 {
          148  +  SELECT sum(length(block)) FROM x2_segdir, x2_segments WHERE 
          149  +    blockid BETWEEN start_block AND leaves_end_block
          150  +    AND level=3
          151  +} {86120}
          152  +
          153  +#--------------------------------------------------------------------------
          154  +# Test that delete markers are removed from FTS segments when possible.
          155  +# It is only possible to remove delete markers when the output of the
          156  +# merge operation will become the oldest segment in the index.
          157  +#
          158  +#   3.1 - when the oldest segment is created by an 'optimize'.
          159  +#   3.2 - when the oldest segment is created by an incremental merge.
          160  +#   3.3 - by a crisis merge.
          161  +#
          162  +
          163  +proc insert_doc {args} {
          164  +  foreach iDoc $args {
          165  +    set L [lindex {
          166  +      {In your eagerness to engage the Trojans,}
          167  +      {don’t any of you charge ahead of others,}
          168  +      {trusting in your strength and horsemanship.}
          169  +      {And don’t lag behind. That will hurt our charge.}
          170  +      {Any man whose chariot confronts an enemy’s}
          171  +      {should thrust with his spear at him from there.}
          172  +      {That’s the most effective tactic, the way}
          173  +      {men wiped out city strongholds long ago —}
          174  +      {their chests full of that style and spirit.}
          175  +    } [expr $iDoc%9]]
          176  +    execsql { REPLACE INTO x3(docid, content) VALUES($iDoc, $L) }
          177  +  }
          178  +}
          179  +
          180  +proc delete_doc {args} {
          181  +  foreach iDoc $args {
          182  +    execsql { DELETE FROM x3 WHERE docid = $iDoc }
          183  +  }
          184  +}
          185  +
          186  +proc second {x} { lindex $x 1 }
          187  +db func second second
          188  +
          189  +do_execsql_test 3.0 { CREATE VIRTUAL TABLE x3 USING fts4 }
          190  +
          191  +do_test 3.1.1 {
          192  +  db transaction { insert_doc 1 2 3 4 5 6 }
          193  +  execsql { SELECT level, idx, second(end_block) FROM x3_segdir }
          194  +} {0 0 412}
          195  +do_test 3.1.2 {
          196  +  delete_doc 1 2 3 4 5 6
          197  +  execsql { SELECT count(*) FROM x3_segdir }
          198  +} {0}
          199  +do_test 3.1.3 {
          200  +  db transaction { 
          201  +    insert_doc 1 2 3 4 5 6 7 8 9
          202  +    delete_doc 9 8 7
          203  +  }
          204  +  execsql { SELECT level, idx, second(end_block) FROM x3_segdir }
          205  +} {0 0 591 0 1 65 0 2 72 0 3 76}
          206  +do_test 3.1.4 {
          207  +  execsql { INSERT INTO x3(x3) VALUES('optimize') }
          208  +  execsql { SELECT level, idx, second(end_block) FROM x3_segdir }
          209  +} {0 0 412}
          210  +
          211  +do_test 3.2.1 {
          212  +  execsql { DELETE FROM x3 }
          213  +  insert_doc 8 7 6 5 4 3 2 1
          214  +  delete_doc 7 8
          215  +  execsql { SELECT count(*) FROM x3_segdir }
          216  +} {10}
          217  +do_test 3.2.2 {
          218  +  execsql { INSERT INTO x3(x3) VALUES('merge=500,10') }
          219  +  execsql { SELECT level, idx, second(end_block) FROM x3_segdir }
          220  +} {1 0 412}
          221  +
          222  +# This assumes the crisis merge happens when there are already 16 
          223  +# segments and one more is added.
          224  +#
          225  +do_test 3.3.1 {
          226  +  execsql { DELETE FROM x3 }
          227  +  insert_doc 1 2 3 4 5 6  7 8 9 10 11
          228  +  delete_doc 11 10 9 8 7
          229  +  execsql { SELECT count(*) FROM x3_segdir }
          230  +} {16}
          231  +
          232  +do_test 3.3.2 {
          233  +  insert_doc 12
          234  +  execsql { SELECT level, idx, second(end_block) FROM x3_segdir WHERE level=1 }
          235  +} {1 0 412}
          236  +
          237  +#--------------------------------------------------------------------------
          238  +# Check a theory on a bug in fts4 - that segments with idx==0 were not 
          239  +# being incrementally merged correctly. Theory turned out to be false.
          240  +#
          241  +do_execsql_test 4.1 {
          242  +  DROP TABLE IF EXISTS x4;
          243  +  DROP TABLE IF EXISTS t1;
          244  +  CREATE TABLE t1(docid, words);
          245  +  CREATE VIRTUAL TABLE x4 USING fts4(words);
          246  +}
          247  +do_test 4.2 {
          248  +  fts_kjv_genesis 
          249  +  execsql { INSERT INTO x4 SELECT words FROM t1 }
          250  +  execsql { INSERT INTO x4 SELECT words FROM t1 }
          251  +} {}
          252  +
          253  +do_execsql_test 4.3 {
          254  +  SELECT level, idx, second(end_block) FROM x4_segdir 
          255  +} {0 0 117483 0 1 118006}
          256  +
          257  +do_execsql_test 4.4 {
          258  +  INSERT INTO x4(x4) VALUES('merge=10,2');
          259  +  SELECT count(*) FROM x4_segdir;
          260  +} {3}
          261  +
          262  +do_execsql_test 4.5 {
          263  +  INSERT INTO x4(x4) VALUES('merge=10,2');
          264  +  SELECT count(*) FROM x4_segdir;
          265  +} {3}
          266  +
          267  +do_execsql_test 4.6 {
          268  +  INSERT INTO x4(x4) VALUES('merge=1000,2');
          269  +  SELECT count(*) FROM x4_segdir;
          270  +} {1}
          271  +
          272  +
          273  +
          274  +#--------------------------------------------------------------------------
          275  +# Check that segments are not promoted if the "end_block" field does not
          276  +# contain a size.
          277  +#
          278  +do_execsql_test 5.1 {
          279  +  DROP TABLE IF EXISTS x2;
          280  +  DROP TABLE IF EXISTS t1;
          281  +  CREATE TABLE t1(docid, words);
          282  +  CREATE VIRTUAL TABLE x2 USING fts4;
          283  +}
          284  +fts_kjv_genesis 
          285  +
          286  +proc first {L} {lindex $L 0}
          287  +db func first first
          288  +
          289  +do_test 5.2 {
          290  +  foreach r [db eval { SELECT rowid FROM t1 }] {
          291  +    execsql {
          292  +      INSERT INTO x2(docid, content) SELECT docid, words FROM t1 WHERE rowid=$r
          293  +    }
          294  +  }
          295  +  foreach d [db eval { SELECT docid FROM t1 LIMIT -1 OFFSET 20 }] {
          296  +    execsql { DELETE FROM x2 WHERE docid = $d }
          297  +  }
          298  +
          299  +  execsql {
          300  +    INSERT INTO x2(x2) VALUES('optimize');
          301  +    SELECT level, idx, end_block FROM x2_segdir
          302  +  }
          303  +} {2 0 {752 1926}}
          304  +
          305  +do_execsql_test 5.3 {
          306  +  UPDATE x2_segdir SET end_block = CAST( first(end_block) AS INTEGER );
          307  +  SELECT end_block, typeof(end_block) FROM x2_segdir;
          308  +} {752 integer}
          309  +
          310  +do_execsql_test 5.4 {
          311  +  INSERT INTO x2 SELECT words FROM t1 LIMIT 50;
          312  +  SELECT level, idx, end_block FROM x2_segdir
          313  +} {2 0 752 0 0 {758 5174}}
          314  +
          315  +do_execsql_test 5.5 {
          316  +  UPDATE x2_segdir SET end_block = end_block || ' 1926' WHERE level=2;
          317  +  INSERT INTO x2 SELECT words FROM t1 LIMIT 40;
          318  +  SELECT level, idx, end_block FROM x2_segdir
          319  +} {0 0 {752 1926} 0 1 {758 5174} 0 2 {763 4170}}
          320  +
          321  +proc t1_to_x2 {} {
          322  +  foreach id [db eval {SELECT docid FROM t1 LIMIT 2}] {
          323  +    execsql {
          324  +      DELETE FROM x2 WHERE docid=$id;
          325  +      INSERT INTO x2(docid, content) SELECT $id, words FROM t1 WHERE docid=$id;
          326  +    }
          327  +  }
          328  +}
          329  +
          330  +#--------------------------------------------------------------------------
          331  +# Check that segments created by auto-merge are not promoted until they
          332  +# are completed.
          333  +#
          334  +
          335  +do_execsql_test 6.1 {
          336  +  CREATE VIRTUAL TABLE x5 USING fts4;
          337  +  INSERT INTO x5 SELECT words FROM t1 LIMIT 100 OFFSET 0;
          338  +  INSERT INTO x5 SELECT words FROM t1 LIMIT 100 OFFSET 25;
          339  +  INSERT INTO x5 SELECT words FROM t1 LIMIT 100 OFFSET 50;
          340  +  INSERT INTO x5 SELECT words FROM t1 LIMIT 100 OFFSET 75;
          341  +  SELECT count(*) FROM x5_segdir
          342  +} {4}
          343  +
          344  +do_execsql_test 6.2 {
          345  +  INSERT INTO x5(x5) VALUES('merge=2,4');
          346  +  SELECT level, idx, end_block FROM x5_segdir;
          347  +} {0 0 {10 9216} 0 1 {21 9330} 0 2 {31 8850} 0 3 {40 8689} 1 0 {1320 -3117}}
          348  +
          349  +do_execsql_test 6.3 {
          350  +  INSERT INTO x5 SELECT words FROM t1 LIMIT 100 OFFSET 100;
          351  +  SELECT level, idx, end_block FROM x5_segdir;
          352  +} {
          353  +  0 0 {10 9216} 0 1 {21 9330} 0 2 {31 8850} 
          354  +  0 3 {40 8689} 1 0 {1320 -3117} 0 4 {1329 8297}
          355  +}
          356  +
          357  +do_execsql_test 6.4 {
          358  +  INSERT INTO x5(x5) VALUES('merge=200,4');
          359  +  SELECT level, idx, end_block FROM x5_segdir;
          360  +} {0 0 {1329 8297} 1 0 {1320 28009}}
          361  +
          362  +do_execsql_test 6.5 {
          363  +  INSERT INTO x5 SELECT words FROM t1;
          364  +  SELECT level, idx, end_block FROM x5_segdir;
          365  +} {
          366  +  0 1 {1329 8297} 0 0 {1320 28009} 0 2 {1449 118006}
          367  +}
          368  +
          369  +#--------------------------------------------------------------------------
          370  +# Ensure that if part of an incremental merge is performed by an old
          371  +# version that does not support storing segment sizes in the end_block
          372  +# field, no size is stored in the final segment (as it would be incorrect).
          373  +#
          374  +do_execsql_test 7.1 {
          375  +  CREATE VIRTUAL TABLE x6 USING fts4;
          376  +  INSERT INTO x6 SELECT words FROM t1;
          377  +  INSERT INTO x6 SELECT words FROM t1;
          378  +  INSERT INTO x6 SELECT words FROM t1;
          379  +  INSERT INTO x6 SELECT words FROM t1;
          380  +  INSERT INTO x6 SELECT words FROM t1;
          381  +  INSERT INTO x6 SELECT words FROM t1;
          382  +  SELECT level, idx, end_block FROM x6_segdir;
          383  +} {
          384  +  0 0 {118 117483} 0 1 {238 118006} 0 2 {358 118006} 
          385  +  0 3 {478 118006} 0 4 {598 118006} 0 5 {718 118006}
          386  +}
          387  +
          388  +do_execsql_test 7.2 {
          389  +  INSERT INTO x6(x6) VALUES('merge=25,4');
          390  +  SELECT level, idx, end_block FROM x6_segdir;
          391  +} {
          392  +  0 0 {118 117483} 0 1 {238 118006} 0 2 {358 118006} 
          393  +  0 3 {478 118006} 0 4 {598 118006} 0 5 {718 118006}
          394  +  1 0 {16014 -51226}
          395  +}
          396  +
          397  +do_execsql_test 7.3 {
          398  +  UPDATE x6_segdir SET end_block = first(end_block) WHERE level=1;
          399  +  SELECT level, idx, end_block FROM x6_segdir;
          400  +} {
          401  +  0 0 {118 117483} 0 1 {238 118006} 0 2 {358 118006} 
          402  +  0 3 {478 118006} 0 4 {598 118006} 0 5 {718 118006}
          403  +  1 0 16014
          404  +}
          405  +
          406  +do_execsql_test 7.4 {
          407  +  INSERT INTO x6(x6) VALUES('merge=25,4');
          408  +  SELECT level, idx, end_block FROM x6_segdir;
          409  +} {
          410  +  0 0 {118 117483} 0 1 {238 118006} 0 2 {358 118006} 
          411  +  0 3 {478 118006} 0 4 {598 118006} 0 5 {718 118006}
          412  +  1 0 16014
          413  +}
          414  +
          415  +do_execsql_test 7.5 {
          416  +  INSERT INTO x6(x6) VALUES('merge=2500,4');
          417  +  SELECT level, idx, end_block FROM x6_segdir;
          418  +} {
          419  +  0 0 {598 118006} 0 1 {718 118006} 1 0 16014
          420  +}
          421  +
          422  +do_execsql_test 7.6 {
          423  +  INSERT INTO x6(x6) VALUES('merge=2500,2');
          424  +  SELECT level, idx, start_block, leaves_end_block, end_block FROM x6_segdir;
          425  +} {
          426  +  2 0 23695 24147 {41262 633507}
          427  +}
          428  +
          429  +do_execsql_test 7.7 {
          430  +  SELECT sum(length(block)) FROM x6_segments 
          431  +  WHERE blockid BETWEEN 23695 AND 24147
          432  +} {633507}
          433  +
          434  +
          435  +
          436  +finish_test
          437  +

Added test/fts4growth2.test.

            1  +# 2014 May 12
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this script is testing the FTS4 module.
           13  +#
           14  +#
           15  +
           16  +set testdir [file dirname $argv0]
           17  +source $testdir/tester.tcl
           18  +set testprefix fts4growth2
           19  +
           20  +# If SQLITE_ENABLE_FTS3 is defined, omit this file.
           21  +ifcapable !fts3 {
           22  +  finish_test
           23  +  return
           24  +}
           25  +
           26  +source $testdir/genesis.tcl
           27  +
           28  +do_execsql_test 1.0 { CREATE TABLE t1(docid, words); }
           29  +fts_kjv_genesis 
           30  +
           31  +proc tt {val} {
           32  +  execsql {
           33  +    DELETE FROM x1 
           34  +      WHERE docid IN (SELECT docid FROM t1 WHERE (rowid-1)%4==$val+0);
           35  +    INSERT INTO x1(docid, content) 
           36  +      SELECT docid, words FROM t1 WHERE (rowid%4)==$val+0;
           37  +  }
           38  +  #puts [db eval {SELECT level, idx, end_block FROM x1_segdir}]
           39  +}
           40  +
           41  +do_execsql_test 1.1 {
           42  +  CREATE VIRTUAL TABLE x1 USING fts4;
           43  +  INSERT INTO x1(x1) VALUES('automerge=2');
           44  +}
           45  +
           46  +do_test 1.2 {
           47  +  for {set i 0} {$i < 40} {incr i} {
           48  +    tt 0 ; tt 1 ; tt 2 ; tt 3
           49  +  }
           50  +  execsql { 
           51  +    SELECT max(level) FROM x1_segdir; 
           52  +    SELECT count(*) FROM x1_segdir WHERE level=3;
           53  +  }
           54  +} {3 1}
           55  +
           56  +do_test 1.3 {
           57  +  for {set i 0} {$i < 40} {incr i} {
           58  +    tt 0 ; tt 1 ; tt 2 ; tt 3
           59  +  }
           60  +  execsql { 
           61  +    SELECT max(level) FROM x1_segdir; 
           62  +    SELECT count(*) FROM x1_segdir WHERE level=2;
           63  +  }
           64  +} {2 1}
           65  +
           66  +#-------------------------------------------------------------------------
           67  +#
           68  +do_execsql_test 2.1 {
           69  +  DELETE FROM t1 WHERE rowid>16;
           70  +  DROP TABLE IF EXISTS x1;
           71  +  CREATE VIRTUAL TABLE x1 USING fts4;
           72  +}
           73  +
           74  +db func second second
           75  +proc second {L} {lindex $L 1}
           76  +
           77  +for {set tn 0} {$tn < 40} {incr tn} {
           78  +  do_test 2.2.$tn {
           79  +    for {set i 0} {$i < 100} {incr i} {
           80  +      tt 0 ; tt 1 ; tt 2 ; tt 3
           81  +    }
           82  +    execsql { SELECT max(level) FROM x1_segdir }
           83  +  } {1}
           84  +}
           85  +
           86  +
           87  +finish_test
           88  +

Changes to test/fts4merge4.test.

    49     49       execsql {INSERT INTO t1 VALUES('a b c d e f g h i j k l');}
    50     50     }
    51     51   } {}
    52     52   do_execsql_test 2.2 { SELECT count(*) FROM t1_segdir; } 35
    53     53   do_execsql_test 2.3 { INSERT INTO t1(t1) VALUES('optimize') } {}
    54     54   do_execsql_test 2.4 { SELECT count(*) FROM t1_segdir; } 1
    55     55   
           56  +#-------------------------------------------------------------------------
           57  +# Now test that the automerge=? option appears to work.
           58  +#
           59  +do_execsql_test 2.1 { CREATE VIRTUAL TABLE t2 USING fts4; }
           60  +
           61  +set doc ""
           62  +foreach c1 "a b c d e f g h i j" {
           63  +  foreach c2 "a b c d e f g h i j" {
           64  +    foreach c3 "a b c d e f g h i j" {
           65  +      lappend doc "$c1$c2$c3"
           66  +    }
           67  +  }
           68  +}
           69  +set doc [string repeat $doc 10]
           70  +
           71  +foreach {tn am expected} {
           72  +  1 {automerge=2} {1 1   2 1   4 1   6 1}
           73  +  2 {automerge=4} {1 2   2 1   3 1}
           74  +  3 {automerge=8} {0 4   1 3   2 1}
           75  +  4 {automerge=1} {0 4   1 3   2 1}
           76  +} {
           77  +  foreach {tn2 openclose} {1 {} 2 { db close ; sqlite3 db test.db }} {
           78  +    do_test 2.2.$tn.$tn2 {
           79  +      execsql { DELETE FROM t2 }
           80  +      execsql { INSERT INTO t2(t2) VALUES($am) };
           81  +
           82  +      eval $openclose
           83  +  
           84  +      for {set i 0} {$i < 100} {incr i} {
           85  +        execsql { 
           86  +          BEGIN;
           87  +            INSERT INTO t2 VALUES($doc);
           88  +            INSERT INTO t2 VALUES($doc);
           89  +            INSERT INTO t2 VALUES($doc);
           90  +            INSERT INTO t2 VALUES($doc);
           91  +            INSERT INTO t2 VALUES($doc);
           92  +          COMMIT;
           93  +        }
           94  +      }
           95  +  
           96  +      execsql { SELECT level, count(*) FROM t2_segdir GROUP BY level }
           97  +    } [list {*}$expected]
           98  +  }
           99  +}
    56    100   
    57    101   sqlite3_enable_shared_cache $::enable_shared_cache
    58    102   finish_test

Changes to test/permutations.test.

   108    108     savepoint4.test savepoint6.test select9.test 
   109    109     speed1.test speed1p.test speed2.test speed3.test speed4.test 
   110    110     speed4p.test sqllimits1.test tkt2686.test thread001.test thread002.test
   111    111     thread003.test thread004.test thread005.test trans2.test vacuum3.test 
   112    112     incrvacuum_ioerr.test autovacuum_crash.test btree8.test shared_err.test
   113    113     vtab_err.test walslow.test walcrash.test walcrash3.test
   114    114     walthread.test rtree3.test indexfault.test securedel2.test
          115  +  fts4growth.test fts4growth2.test
   115    116   }]
   116    117   if {[info exists ::env(QUICKTEST_INCLUDE)]} {
   117    118     set allquicktests [concat $allquicktests $::env(QUICKTEST_INCLUDE)]
   118    119   }
   119    120   
   120    121   #############################################################################
   121    122   # Start of tests
................................................................................
   192    193     fts3fault.test fts3malloc.test fts3matchinfo.test
   193    194     fts3aux1.test fts3comp1.test fts3auto.test
   194    195     fts4aa.test fts4content.test
   195    196     fts3conf.test fts3prefix.test fts3fault2.test fts3corrupt.test
   196    197     fts3corrupt2.test fts3first.test fts4langid.test fts4merge.test
   197    198     fts4check.test fts4unicode.test fts4noti.test
   198    199     fts3varint.test
          200  +  fts4growth.test fts4growth2.test
   199    201   }
   200    202   
   201    203   test_suite "nofaultsim" -prefix "" -description {
   202    204     "Very" quick test suite. Runs in less than 5 minutes on a workstation. 
   203    205     This test suite is the same as the "quick" tests, except that some files
   204    206     that test malloc and IO errors are omitted.
   205    207   } -files [

Changes to test/trace2.test.

   132    132       "-- INSERT INTO 'main'.'x1_content' VALUES(?,(?))" 
   133    133       "-- REPLACE INTO 'main'.'x1_docsize' VALUES(?,?)" 
   134    134       "-- SELECT value FROM 'main'.'x1_stat' WHERE id=?" 
   135    135       "-- REPLACE INTO 'main'.'x1_stat' VALUES(?,?)" 
   136    136       "-- SELECT (SELECT max(idx) FROM 'main'.'x1_segdir' WHERE level = ?) + 1" 
   137    137       "-- SELECT coalesce((SELECT max(blockid) FROM 'main'.'x1_segments') + 1, 1)"
   138    138       "-- REPLACE INTO 'main'.'x1_segdir' VALUES(?,?,?,?,?,?)"
          139  +    "-- SELECT level, idx, end_block FROM 'main'.'x1_segdir' WHERE level BETWEEN ? AND ? ORDER BY level DESC, idx ASC"
   139    140     }
   140    141   
   141    142     do_trace_test 2.3 {
   142    143       INSERT INTO x1(x1) VALUES('optimize');
   143    144     } {
   144    145       "INSERT INTO x1(x1) VALUES('optimize');"
   145    146       "-- SELECT DISTINCT level / (1024 * ?) FROM 'main'.'x1_segdir'"