/ Check-in [bb4a37b5]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add a cookie mechanism to ensure that the %_config table is re-read as required.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts5
Files: files | file ages | folders
SHA1: bb4a37b53de60da9ec8b9317eec14afa99690828
User & Date: dan 2014-11-28 20:01:13
Context
2014-12-01
20:05
Add code to parse a rank() function specification. And a tcl interface to add auxiliary functions to fts5. check-in: 9c1697a2 user: dan tags: fts5
2014-11-28
20:01
Add a cookie mechanism to ensure that the %_config table is re-read as required. check-in: bb4a37b5 user: dan tags: fts5
2014-11-27
20:03
Add a %_config table to fts5. check-in: 83491c56 user: dan tags: fts5
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5.c.

   336    336     }
   337    337   
   338    338     /* Call sqlite3_declare_vtab() */
   339    339     if( rc==SQLITE_OK ){
   340    340       rc = sqlite3Fts5ConfigDeclareVtab(pConfig);
   341    341     }
   342    342   
   343         -  /* Load the contents of %_config */
   344         -  if( rc==SQLITE_OK ){
   345         -    rc = sqlite3Fts5ConfigLoad(pConfig);
   346         -  }
   347         -
   348    343     if( rc!=SQLITE_OK ){
   349    344       fts5FreeVtab(pTab, 0);
   350    345       pTab = 0;
   351    346     }else if( bCreate ){
   352    347       fts5CheckTransactionState(pTab, FTS5_BEGIN, 0);
   353    348     }
   354    349     *ppVTab = (sqlite3_vtab*)pTab;

Changes to ext/fts5/fts5Int.h.

    56     56   typedef struct Fts5Config Fts5Config;
    57     57   
    58     58   /*
    59     59   ** An instance of the following structure encodes all information that can
    60     60   ** be gleaned from the CREATE VIRTUAL TABLE statement.
    61     61   **
    62     62   ** And all information loaded from the %_config table.
           63  +**
           64  +** nAutomerge:
           65  +**   The minimum number of segments that an auto-merge operation should
           66  +**   attempt to merge together. A value of 1 sets the object to use the 
           67  +**   compile time default. Zero disables auto-merge altogether.
    63     68   */
    64     69   struct Fts5Config {
    65     70     sqlite3 *db;                    /* Database handle */
    66     71     char *zDb;                      /* Database holding FTS index (e.g. "main") */
    67     72     char *zName;                    /* Name of FTS index */
    68     73     int nCol;                       /* Number of columns */
    69     74     char **azCol;                   /* Column names */
................................................................................
    71     76     int *aPrefix;                   /* Sizes in bytes of nPrefix prefix indexes */
    72     77     Fts5Tokenizer *pTok;
    73     78     fts5_tokenizer *pTokApi;
    74     79   
    75     80     /* Values loaded from the %_config table */
    76     81     int iCookie;                    /* Incremented when %_config is modified */
    77     82     int pgsz;                       /* Approximate page size used in %_data */
           83  +  int nAutomerge;                 /* 'automerge' setting */
    78     84   };
    79     85   
    80     86   int sqlite3Fts5ConfigParse(
    81     87       Fts5Global*, sqlite3*, int, const char **, Fts5Config**, char**
    82     88   );
    83     89   void sqlite3Fts5ConfigFree(Fts5Config*);
    84     90   
................................................................................
    90     96     void *pCtx,                     /* Context passed to xToken() */
    91     97     int (*xToken)(void*, const char*, int, int, int, int)    /* Callback */
    92     98   );
    93     99   
    94    100   void sqlite3Fts5Dequote(char *z);
    95    101   
    96    102   /* Load the contents of the %_config table */
    97         -int sqlite3Fts5ConfigLoad(Fts5Config*);
          103  +int sqlite3Fts5ConfigLoad(Fts5Config*, int);
    98    104   
    99    105   /* Set the value of a single config attribute */
   100    106   int sqlite3Fts5ConfigSetValue(Fts5Config*, const char*, sqlite3_value*, int*);
   101    107   
   102    108   /*
   103    109   ** End of interface to code in fts5_config.c.
   104    110   **************************************************************************/
................................................................................
   122    128   void sqlite3Fts5BufferAppendBlob(int*, Fts5Buffer*, int, const u8*);
   123    129   void sqlite3Fts5BufferAppendString(int *, Fts5Buffer*, const char*);
   124    130   void sqlite3Fts5BufferFree(Fts5Buffer*);
   125    131   void sqlite3Fts5BufferZero(Fts5Buffer*);
   126    132   void sqlite3Fts5BufferSet(int*, Fts5Buffer*, int, const u8*);
   127    133   void sqlite3Fts5BufferAppendPrintf(int *, Fts5Buffer*, char *zFmt, ...);
   128    134   void sqlite3Fts5BufferAppendListElem(int*, Fts5Buffer*, const char*, int);
          135  +void sqlite3Fts5BufferAppend32(int*, Fts5Buffer*, int);
   129    136   
   130    137   #define fts5BufferZero(x)             sqlite3Fts5BufferZero(x)
   131    138   #define fts5BufferGrow(a,b,c)         sqlite3Fts5BufferGrow(a,b,c)
   132    139   #define fts5BufferAppendVarint(a,b,c) sqlite3Fts5BufferAppendVarint(a,b,c)
   133    140   #define fts5BufferFree(a)             sqlite3Fts5BufferFree(a)
   134    141   #define fts5BufferAppendBlob(a,b,c,d) sqlite3Fts5BufferAppendBlob(a,b,c,d)
   135    142   #define fts5BufferSet(a,b,c,d)        sqlite3Fts5BufferSet(a,b,c,d)
          143  +#define fts5BufferAppend32(a,b,c)     sqlite3Fts5BufferAppend32(a,b,c)
          144  +
          145  +/* Write and decode big-endian 32-bit integer values */
          146  +void sqlite3Fts5Put32(u8*, int);
          147  +int sqlite3Fts5Get32(const u8*);
   136    148   
   137    149   typedef struct Fts5PoslistReader Fts5PoslistReader;
   138    150   struct Fts5PoslistReader {
   139    151     /* Variables used only by sqlite3Fts5PoslistIterXXX() functions. */
   140    152     int iCol;                       /* If (iCol>=0), this column only */
   141    153     const u8 *a;                    /* Position list to iterate through */
   142    154     int n;                          /* Size of buffer at a[] in bytes */
................................................................................
   294    306   
   295    307   /* 
   296    308   ** Called during virtual module initialization to register UDF 
   297    309   ** fts5_decode() with SQLite 
   298    310   */
   299    311   int sqlite3Fts5IndexInit(sqlite3*);
   300    312   
   301         -void sqlite3Fts5IndexAutomerge(Fts5Index *p, int nMerge);
          313  +int sqlite3Fts5IndexSetCookie(Fts5Index*, int);
   302    314   
   303    315   /*
   304    316   ** Return the total number of entries read from the %_data table by 
   305    317   ** this connection since it was created.
   306    318   */
   307    319   int sqlite3Fts5IndexReads(Fts5Index *p);
   308    320   

Changes to ext/fts5/fts5_buffer.c.

    41     41   ** Encode value iVal as an SQLite varint and append it to the buffer object
    42     42   ** pBuf. If an OOM error occurs, set the error code in p.
    43     43   */
    44     44   void sqlite3Fts5BufferAppendVarint(int *pRc, Fts5Buffer *pBuf, i64 iVal){
    45     45     if( sqlite3Fts5BufferGrow(pRc, pBuf, 9) ) return;
    46     46     pBuf->n += sqlite3PutVarint(&pBuf->p[pBuf->n], iVal);
    47     47   }
           48  +
           49  +void sqlite3Fts5Put32(u8 *aBuf, int iVal){
           50  +  aBuf[0] = (iVal>>24) & 0x00FF;
           51  +  aBuf[1] = (iVal>>16) & 0x00FF;
           52  +  aBuf[2] = (iVal>> 8) & 0x00FF;
           53  +  aBuf[3] = (iVal>> 0) & 0x00FF;
           54  +}
           55  +
           56  +int sqlite3Fts5Get32(const u8 *aBuf){
           57  +  return (aBuf[0] << 24) + (aBuf[1] << 16) + (aBuf[2] << 8) + aBuf[3];
           58  +}
           59  +
           60  +void sqlite3Fts5BufferAppend32(int *pRc, Fts5Buffer *pBuf, int iVal){
           61  +  char *a;
           62  +  if( sqlite3Fts5BufferGrow(pRc, pBuf, 4) ) return;
           63  +  sqlite3Fts5Put32(&pBuf->p[pBuf->n], iVal);
           64  +  pBuf->n += 4;
           65  +}
    48     66   
    49     67   /*
    50     68   ** Append buffer nData/pData to buffer pBuf. If an OOM error occurs, set 
    51     69   ** the error code in p. If an error has already occurred when this function
    52     70   ** is called, it is a no-op.
    53     71   */
    54     72   void sqlite3Fts5BufferAppendBlob(

Changes to ext/fts5/fts5_config.c.

    12     12   **
    13     13   ** This is an SQLite module implementing full-text search.
    14     14   */
    15     15   
    16     16   #include "fts5Int.h"
    17     17   
    18     18   #define FTS5_DEFAULT_PAGE_SIZE   1000
           19  +#define FTS5_DEFAULT_AUTOMERGE      4
           20  +
           21  +/* Maximum allowed page size */
           22  +#define FTS5_MAX_PAGE_SIZE (128*1024)
    19     23   
    20     24   /*
    21     25   ** Convert an SQL-style quoted string into a normal string by removing
    22     26   ** the quote characters.  The conversion is done in-place.  If the
    23     27   ** input does not begin with a quote character, then this routine
    24     28   ** is a no-op.
    25     29   **
................................................................................
   149    153     int rc = SQLITE_OK;             /* Return code */
   150    154     Fts5Config *pRet;               /* New object to return */
   151    155   
   152    156     *ppOut = pRet = (Fts5Config*)sqlite3_malloc(sizeof(Fts5Config));
   153    157     if( pRet==0 ) return SQLITE_NOMEM;
   154    158     memset(pRet, 0, sizeof(Fts5Config));
   155    159     pRet->db = db;
          160  +  pRet->iCookie = -1;
   156    161   
   157    162     pRet->azCol = (char**)sqlite3_malloc(sizeof(char*) * nArg);
   158    163     pRet->zDb = fts5Strdup(azArg[1]);
   159    164     pRet->zName = fts5Strdup(azArg[2]);
   160    165     if( sqlite3_stricmp(pRet->zName, FTS5_RANK_NAME)==0 ){
   161    166       *pzErr = sqlite3_mprintf("reserved fts5 table name: %s", pRet->zName);
   162    167       rc = SQLITE_ERROR;
................................................................................
   303    308     sqlite3_value *pVal,
   304    309     int *pbBadkey
   305    310   ){
   306    311     int rc = SQLITE_OK;
   307    312     if(      0==sqlite3_stricmp(zKey, "cookie") ){
   308    313       pConfig->iCookie = sqlite3_value_int(pVal);
   309    314     }
          315  +
   310    316     else if( 0==sqlite3_stricmp(zKey, "pgsz") ){
          317  +    int pgsz = 0;
          318  +    if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
          319  +      pgsz = sqlite3_value_int(pVal);
          320  +    }
          321  +    if( pgsz<=0 || pgsz>FTS5_MAX_PAGE_SIZE ){
          322  +      if( pbBadkey ) *pbBadkey = 1;
          323  +    }else{
          324  +      pConfig->pgsz = pgsz;
          325  +    }
          326  +  }
          327  +
          328  +  else if( 0==sqlite3_stricmp(zKey, "automerge") ){
          329  +    int nAutomerge = -1;
   311    330       if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
   312         -      pConfig->pgsz = sqlite3_value_int(pVal);
   313         -    }else{
          331  +      nAutomerge = sqlite3_value_int(pVal);
          332  +    }
          333  +    if( nAutomerge<0 || nAutomerge>64 ){
   314    334         if( pbBadkey ) *pbBadkey = 1;
          335  +    }else{
          336  +      if( nAutomerge==1 ) nAutomerge = FTS5_DEFAULT_AUTOMERGE;
          337  +      pConfig->nAutomerge = nAutomerge;
   315    338       }
   316    339     }
   317         -  else if( 0==sqlite3_stricmp(zKey, "automerge") ){
   318         -    // todo
   319         -  }
          340  +
   320    341     else if( 0==sqlite3_stricmp(zKey, "rank") ){
   321    342       // todo
   322    343     }else{
   323    344       if( pbBadkey ) *pbBadkey = 1;
   324    345     }
   325    346     return rc;
   326    347   }
   327    348   
   328    349   /*
   329    350   ** Load the contents of the %_config table into memory.
   330    351   */
   331         -int sqlite3Fts5ConfigLoad(Fts5Config *pConfig){
          352  +int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){
   332    353     const char *zSelect = "SELECT k, v FROM %Q.'%q_config'";
   333    354     char *zSql;
   334    355     sqlite3_stmt *p = 0;
   335    356     int rc;
   336    357   
   337    358     /* Set default values */
   338    359     pConfig->pgsz = FTS5_DEFAULT_PAGE_SIZE;
   339         -  pConfig->iCookie = 0;
          360  +  pConfig->nAutomerge = FTS5_DEFAULT_AUTOMERGE;
   340    361   
   341    362     zSql = sqlite3_mprintf(zSelect, pConfig->zDb, pConfig->zName);
   342    363     if( zSql==0 ){
   343    364       rc = SQLITE_NOMEM;
   344    365     }else{
   345    366       rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &p, 0);
   346    367       sqlite3_free(zSql);
................................................................................
   352    373         const char *zK = (const char*)sqlite3_column_text(p, 0);
   353    374         sqlite3_value *pVal = sqlite3_column_value(p, 1);
   354    375         sqlite3Fts5ConfigSetValue(pConfig, zK, pVal, 0);
   355    376       }
   356    377       rc = sqlite3_finalize(p);
   357    378     }
   358    379   
          380  +  if( rc==SQLITE_OK ){
          381  +    pConfig->iCookie = iCookie;
          382  +  }
   359    383     return rc;
   360    384   }
   361    385   

Changes to ext/fts5/fts5_index.c.

    38     38   **
    39     39   **   * extra fields in the "structure record" record the state of ongoing
    40     40   **     incremental merge operations.
    41     41   **
    42     42   */
    43     43   
    44     44   #define FTS5_WORK_UNIT      64    /* Number of leaf pages in unit of work */
    45         -#define FTS5_MIN_MERGE       4    /* Minimum number of segments to merge */
    46     45   #define FTS5_CRISIS_MERGE   16    /* Maximum number of segments to merge */
    47     46   
    48     47   #define FTS5_MIN_DLIDX_SIZE  4    /* Add dlidx if this many empty pages */
    49     48   
    50     49   /*
    51     50   ** Details:
    52     51   **
................................................................................
    57     56   ** , contains the following 5 types of records. See the comments surrounding
    58     57   ** the FTS5_*_ROWID macros below for a description of how %_data rowids are 
    59     58   ** assigned to each fo them.
    60     59   **
    61     60   ** 1. Structure Records:
    62     61   **
    63     62   **   The set of segments that make up an index - the index structure - are
    64         -**   recorded in a single record within the %_data table. The record is a list
    65         -**   of SQLite varints. 
           63  +**   recorded in a single record within the %_data table. The record consists
           64  +**   of a single 32-bit configuration cookie value followed by a list of 
           65  +**   SQLite varints. If the FTS table features more than one index (because
           66  +**   there are one or more prefix indexes), it is guaranteed that all share
           67  +**   the same cookie value.
    66     68   **
    67         -**   The record begins with three varints:
           69  +**   Immediately following the configuration cookie, the record begins with
           70  +**   three varints:
    68     71   **
    69     72   **     + number of levels,
    70     73   **     + total number of segments on all levels,
    71     74   **     + value of write counter.
    72     75   **
    73     76   **   Then, for each level from 0 to nMax:
    74     77   **
................................................................................
   284    287   
   285    288   /*
   286    289   ** One object per %_data table.
   287    290   */
   288    291   struct Fts5Index {
   289    292     Fts5Config *pConfig;            /* Virtual table configuration */
   290    293     char *zDataTbl;                 /* Name of %_data table */
   291         -  int nMinMerge;                  /* Minimum input segments in a merge */
   292    294     int nCrisisMerge;               /* Maximum allowed segments per level */
   293    295     int nWorkUnit;                  /* Leaf pages in a "unit" of work */
   294    296   
   295    297     /*
   296    298     ** Variables related to the accumulation of tokens and doclists within the
   297    299     ** in-memory hash tables before they are flushed to disk.
   298    300     */
................................................................................
   956    958   ** If an error occurs, *ppOut is set to NULL and an SQLite error code
   957    959   ** returned. Otherwise, *ppOut is set to point to the new object and
   958    960   ** SQLITE_OK returned.
   959    961   */
   960    962   static int fts5StructureDecode(
   961    963     const u8 *pData,                /* Buffer containing serialized structure */
   962    964     int nData,                      /* Size of buffer pData in bytes */
          965  +  int *piCookie,                  /* Configuration cookie value */
   963    966     Fts5Structure **ppOut           /* OUT: Deserialized object */
   964    967   ){
   965    968     int rc = SQLITE_OK;
   966    969     int i = 0;
   967    970     int iLvl;
   968    971     int nLevel = 0;
   969    972     int nSegment = 0;
   970    973     int nByte;                      /* Bytes of space to allocate at pRet */
   971    974     Fts5Structure *pRet = 0;        /* Structure object to return */
          975  +
          976  +  /* Grab the cookie value */
          977  +  if( piCookie ) *piCookie = sqlite3Fts5Get32(pData);
          978  +  i = 4;
   972    979   
   973    980     /* Read the total number of levels and segments from the start of the
   974    981     ** structure record.  */
   975         -  i = getVarint32(&pData[i], nLevel);
          982  +  i += getVarint32(&pData[i], nLevel);
   976    983     i += getVarint32(&pData[i], nSegment);
   977    984     nByte = (
   978    985         sizeof(Fts5Structure) +                    /* Main structure */
   979    986         sizeof(Fts5StructureLevel) * (nLevel)      /* aLevel[] array */
   980    987     );
   981    988     pRet = (Fts5Structure*)sqlite3Fts5MallocZero(&rc, nByte);
   982    989   
................................................................................
  1079   1086   ** Fts5Index handle. If an error has already occurred when this function
  1080   1087   ** is called, it is a no-op.
  1081   1088   */
  1082   1089   static Fts5Structure *fts5StructureRead(Fts5Index *p, int iIdx){
  1083   1090     Fts5Config *pConfig = p->pConfig;
  1084   1091     Fts5Structure *pRet = 0;        /* Object to return */
  1085   1092     Fts5Data *pData;                /* %_data entry containing structure record */
         1093  +  int iCookie;                    /* Configuration cookie */
  1086   1094   
  1087   1095     assert( iIdx<=pConfig->nPrefix );
  1088   1096     pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID(iIdx));
  1089   1097     if( !pData ) return 0;
  1090         -  p->rc = fts5StructureDecode(pData->p, pData->n, &pRet);
         1098  +  p->rc = fts5StructureDecode(pData->p, pData->n, &iCookie, &pRet);
         1099  +
         1100  +  if( p->rc==SQLITE_OK && p->pConfig->iCookie!=iCookie ){
         1101  +    p->rc = sqlite3Fts5ConfigLoad(p->pConfig, iCookie);
         1102  +  }
  1091   1103   
  1092   1104     fts5DataRelease(pData);
  1093   1105     return pRet;
  1094   1106   }
  1095   1107   
  1096   1108   /*
  1097   1109   ** Release a reference to an Fts5Structure object returned by an earlier 
................................................................................
  1125   1137   ** If an error occurs, leave an error code in the Fts5Index object. If an
  1126   1138   ** error has already occurred, this function is a no-op.
  1127   1139   */
  1128   1140   static void fts5StructureWrite(Fts5Index *p, int iIdx, Fts5Structure *pStruct){
  1129   1141     int nSegment;                   /* Total number of segments */
  1130   1142     Fts5Buffer buf;                 /* Buffer to serialize record into */
  1131   1143     int iLvl;                       /* Used to iterate through levels */
         1144  +  int iCookie;                    /* Cookie value to store */
  1132   1145   
  1133   1146     nSegment = fts5StructureCountSegments(pStruct);
  1134   1147     memset(&buf, 0, sizeof(Fts5Buffer));
         1148  +
         1149  +  /* Append the current configuration cookie */
         1150  +  iCookie = p->pConfig->iCookie;
         1151  +  if( iCookie<0 ) iCookie = 0;
         1152  +  fts5BufferAppend32(&p->rc, &buf, iCookie);
         1153  +
  1135   1154     fts5BufferAppendVarint(&p->rc, &buf, pStruct->nLevel);
  1136   1155     fts5BufferAppendVarint(&p->rc, &buf, nSegment);
  1137   1156     fts5BufferAppendVarint(&p->rc, &buf, (i64)pStruct->nWriteCounter);
  1138   1157   
  1139   1158     for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
  1140   1159       int iSeg;                     /* Used to iterate through segments */
  1141   1160       Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl];
................................................................................
  2821   2840     const u8 *aData, 
  2822   2841     int nData
  2823   2842   ){
  2824   2843     Fts5PageWriter *pPage = &pWriter->aWriter[0];
  2825   2844     const u8 *a = aData;
  2826   2845     int n = nData;
  2827   2846     
         2847  +  assert( p->pConfig->pgsz>0 );
  2828   2848     while( p->rc==SQLITE_OK && (pPage->buf.n + n)>=p->pConfig->pgsz ){
  2829   2849       int nReq = p->pConfig->pgsz - pPage->buf.n;
  2830   2850       int nCopy = 0;
  2831   2851       while( nCopy<nReq ){
  2832   2852         i64 dummy;
  2833   2853         nCopy += getVarint(&a[nCopy], (u64*)&dummy);
  2834   2854       }
................................................................................
  3175   3195       /* If nBest is still 0, then the index must be empty. */
  3176   3196   #ifdef SQLITE_DEBUG
  3177   3197       for(iLvl=0; nBest==0 && iLvl<pStruct->nLevel; iLvl++){
  3178   3198         assert( pStruct->aLevel[iLvl].nSeg==0 );
  3179   3199       }
  3180   3200   #endif
  3181   3201   
  3182         -    if( nBest<p->nMinMerge && pStruct->aLevel[iBestLvl].nMerge==0 ) break;
         3202  +    if( nBest<p->pConfig->nAutomerge 
         3203  +     && pStruct->aLevel[iBestLvl].nMerge==0 
         3204  +    ){
         3205  +      break;
         3206  +    }
  3183   3207       fts5IndexMergeLevel(p, iIdx, &pStruct, iBestLvl, &nRem);
  3184   3208       fts5StructurePromote(p, iBestLvl+1, pStruct);
  3185   3209       assert( nRem==0 || p->rc==SQLITE_OK );
  3186   3210       *ppStruct = pStruct;
  3187   3211     }
  3188   3212   }
  3189   3213   
................................................................................
  3289   3313         pSeg->iSegid = iSegid;
  3290   3314         pSeg->nHeight = nHeight;
  3291   3315         pSeg->pgnoFirst = 1;
  3292   3316         pSeg->pgnoLast = pgnoLast;
  3293   3317       }
  3294   3318     }
  3295   3319   
  3296         -  if( p->nMinMerge>0 ) fts5IndexWork(p, iHash, &pStruct, pgnoLast);
         3320  +  if( p->pConfig->nAutomerge>0 ) fts5IndexWork(p, iHash, &pStruct, pgnoLast);
  3297   3321     fts5IndexCrisisMerge(p, iHash, &pStruct);
  3298   3322     fts5StructureWrite(p, iHash, pStruct);
  3299   3323     fts5StructureRelease(pStruct);
  3300   3324   }
  3301   3325   
  3302   3326   /*
  3303   3327   ** Flush any data stored in the in-memory hash tables to the database.
................................................................................
  3367   3391     Fts5Index *p;                   /* New object */
  3368   3392   
  3369   3393     *pp = p = (Fts5Index*)sqlite3_malloc(sizeof(Fts5Index));
  3370   3394     if( !p ) return SQLITE_NOMEM;
  3371   3395   
  3372   3396     memset(p, 0, sizeof(Fts5Index));
  3373   3397     p->pConfig = pConfig;
  3374         -  p->nMinMerge = FTS5_MIN_MERGE;
  3375   3398     p->nCrisisMerge = FTS5_CRISIS_MERGE;
  3376   3399     p->nWorkUnit = FTS5_WORK_UNIT;
  3377   3400     p->nMaxPendingData = 1024*1024;
  3378   3401     p->zDataTbl = sqlite3_mprintf("%s_data", pConfig->zName);
  3379   3402     if( p->zDataTbl==0 ){
  3380   3403       rc = SQLITE_NOMEM;
  3381   3404     }else if( bCreate ){
................................................................................
  3777   3800       rc = p->rc;
  3778   3801     }
  3779   3802   
  3780   3803     return rc;
  3781   3804   }
  3782   3805   
  3783   3806   /*
         3807  +** This is part of the fts5_decode() debugging aid.
         3808  +**
         3809  +** Arguments pBlob/nBlob contain a serialized Fts5Structure object. This
         3810  +** function appends a human-readable representation of the same object
         3811  +** to the buffer passed as the second argument. 
  3784   3812   */
  3785   3813   static void fts5DecodeStructure(
  3786   3814     int *pRc,                       /* IN/OUT: error code */
  3787   3815     Fts5Buffer *pBuf,
  3788   3816     const u8 *pBlob, int nBlob
  3789   3817   ){
  3790   3818     int rc;                         /* Return code */
  3791   3819     Fts5Structure *p = 0;           /* Decoded structure object */
  3792   3820   
  3793         -  rc = fts5StructureDecode(pBlob, nBlob, &p);
         3821  +  rc = fts5StructureDecode(pBlob, nBlob, 0, &p);
  3794   3822     if( rc!=SQLITE_OK ){
  3795   3823       *pRc = rc;
  3796   3824       return;
  3797   3825     }
  3798   3826   
  3799   3827     fts5DebugStructure(pRc, pBuf, p);
  3800   3828     fts5StructureRelease(p);
................................................................................
  3981   4009   int sqlite3Fts5IndexInit(sqlite3 *db){
  3982   4010     int rc = sqlite3_create_function(
  3983   4011         db, "fts5_decode", 2, SQLITE_UTF8, 0, fts5DecodeFunction, 0, 0
  3984   4012     );
  3985   4013     return rc;
  3986   4014   }
  3987   4015   
  3988         -/*
  3989         -** Set the minimum number of segments that an auto-merge operation should
  3990         -** attempt to merge together. A value of 1 sets the object to use the 
  3991         -** compile time default. Zero or less disables auto-merge altogether.
  3992         -*/
  3993         -void sqlite3Fts5IndexAutomerge(Fts5Index *p, int nMinMerge){
  3994         -  if( nMinMerge==1 ){
  3995         -    p->nMinMerge = FTS5_MIN_MERGE;
  3996         -  }else{
  3997         -    p->nMinMerge = nMinMerge;
  3998         -  }
  3999         -}
  4000         -
  4001   4016   /*
  4002   4017   ** Iterator pMulti currently points to a valid entry (not EOF). This
  4003   4018   ** function appends a copy of the position-list of the entry pMulti 
  4004   4019   ** currently points to to buffer pBuf.
  4005   4020   **
  4006   4021   ** If an error occurs, an error code is left in p->rc. It is assumed
  4007   4022   ** no error has already occurred when this function is called.
................................................................................
  4403   4418   /*
  4404   4419   ** Return the total number of blocks this module has read from the %_data
  4405   4420   ** table since it was created.
  4406   4421   */
  4407   4422   int sqlite3Fts5IndexReads(Fts5Index *p){
  4408   4423     return p->nRead;
  4409   4424   }
         4425  +
         4426  +/*
         4427  +** Set the 32-bit cookie value at the start of all structure records to
         4428  +** the value passed as the second argument.
         4429  +**
         4430  +** Return SQLITE_OK if successful, or an SQLite error code if an error
         4431  +** occurs.
         4432  +*/
         4433  +int sqlite3Fts5IndexSetCookie(Fts5Index *p, int iNew){
         4434  +  int rc = SQLITE_OK;
         4435  +  Fts5Config *pConfig = p->pConfig;
         4436  +  u8 aCookie[4];
         4437  +  int i;
         4438  +
         4439  +  sqlite3Fts5Put32(aCookie, iNew);
         4440  +  for(i=0; rc==SQLITE_OK && i<=pConfig->nPrefix; i++){
         4441  +    sqlite3_blob *pBlob = 0;
         4442  +    i64 iRowid = FTS5_STRUCTURE_ROWID(i);
         4443  +    rc = sqlite3_blob_open(
         4444  +        pConfig->db, pConfig->zDb, p->zDataTbl, "block", iRowid, 1, &pBlob
         4445  +    );
         4446  +    if( rc==SQLITE_OK ){
         4447  +      sqlite3_blob_write(pBlob, aCookie, 4, 0);
         4448  +      rc = sqlite3_blob_close(pBlob);
         4449  +    }
         4450  +  }
         4451  +
         4452  +  return rc;
         4453  +}
  4410   4454   

Changes to ext/fts5/fts5_storage.c.

   764    764     sqlite3_stmt *pReplace = 0;
   765    765     int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_CONFIG, &pReplace);
   766    766     if( rc==SQLITE_OK ){
   767    767       sqlite3_bind_text(pReplace, 1, z, -1, SQLITE_TRANSIENT);
   768    768       sqlite3_bind_value(pReplace, 2, pVal);
   769    769       sqlite3_step(pReplace);
   770    770       rc = sqlite3_reset(pReplace);
          771  +  }
          772  +  if( rc==SQLITE_OK ){
          773  +    int iNew = p->pConfig->iCookie + 1;
          774  +    rc = sqlite3Fts5IndexSetCookie(p->pIndex, iNew);
          775  +    if( rc==SQLITE_OK ){
          776  +      p->pConfig->iCookie = iNew;
          777  +    }
   771    778     }
   772    779     return rc;
   773    780   }
   774    781   
   775    782   

Changes to test/fts5aa.test.

    81     81   }
    82     82   
    83     83   #-------------------------------------------------------------------------
    84     84   #
    85     85   reset_db
    86     86   do_execsql_test 4.0 {
    87     87     CREATE VIRTUAL TABLE t1 USING fts5(x,y);
    88         -  INSERT INTO t1(t1, rowid) VALUES('pgsz', 32);
           88  +  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
    89     89   }
    90     90   foreach {i x y} {
    91     91      1  {g f d b f} {h h e i a}
    92     92      2  {f i g j e} {i j c f f}
    93     93      3  {e e i f a} {e h f d f}
    94     94      4  {h j f j i} {h a c f j}
    95     95      5  {d b j c g} {f e i b e}
................................................................................
   105    105   }
   106    106   
   107    107   #-------------------------------------------------------------------------
   108    108   #
   109    109   reset_db
   110    110   do_execsql_test 5.0 {
   111    111     CREATE VIRTUAL TABLE t1 USING fts5(x,y);
   112         -  INSERT INTO t1(t1, rowid) VALUES('pgsz', 32);
          112  +  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
   113    113   }
   114    114   foreach {i x y} {
   115    115      1  {dd abc abc abc abcde} {aaa dd ddd ddd aab}
   116    116      2  {dd aab d aaa b} {abcde c aaa aaa aaa}
   117    117      3  {abcde dd b b dd} {abc abc d abc ddddd}
   118    118      4  {aaa abcde dddd dddd abcde} {abc b b abcde abc}
   119    119      5  {aab dddd d dddd c} {ddd abcde dddd abcde c}
................................................................................
   130    130   
   131    131   #-------------------------------------------------------------------------
   132    132   #
   133    133   breakpoint
   134    134   reset_db
   135    135   do_execsql_test 6.0 {
   136    136     CREATE VIRTUAL TABLE t1 USING fts5(x,y);
   137         -  INSERT INTO t1(t1, rowid) VALUES('pgsz', 32);
          137  +  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
   138    138   }
   139    139   
   140    140   do_execsql_test 6.1 {
   141    141     INSERT  INTO t1(rowid, x, y) VALUES(22, 'a b c', 'c b a');
   142    142     REPLACE INTO t1(rowid, x, y) VALUES(22, 'd e f', 'f e d');
   143    143   }
   144    144   
................................................................................
   148    148   
   149    149   #-------------------------------------------------------------------------
   150    150   #
   151    151   reset_db
   152    152   expr srand(0)
   153    153   do_execsql_test 7.0 {
   154    154     CREATE VIRTUAL TABLE t1 USING fts5(x,y,z);
   155         -  INSERT INTO t1(t1, rowid) VALUES('pgsz', 32);
          155  +  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
   156    156   }
   157    157   
   158    158   proc doc {} {
   159    159     set v [list aaa aab abc abcde b c d dd ddd dddd ddddd]
   160    160     set ret [list]
   161    161     for {set j 0} {$j < 20} {incr j} {
   162    162       lappend ret [lindex $v [expr int(rand()*[llength $v])]]
................................................................................
   187    187   }
   188    188   
   189    189   #-------------------------------------------------------------------------
   190    190   #
   191    191   reset_db
   192    192   do_execsql_test 8.0 {
   193    193     CREATE VIRTUAL TABLE t1 USING fts5(x, prefix="1,2,3");
   194         -  INSERT INTO t1(t1, rowid) VALUES('pgsz', 32);
          194  +  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
   195    195   }
   196    196   
   197    197   do_execsql_test 8.1 {
   198    198     INSERT INTO t1 VALUES('the quick brown fox');
   199    199     INSERT INTO t1(t1) VALUES('integrity-check');
   200    200   }
   201    201   
................................................................................
   204    204   #
   205    205   reset_db
   206    206   
   207    207   expr srand(0)
   208    208   
   209    209   do_execsql_test 9.0 {
   210    210     CREATE VIRTUAL TABLE t1 USING fts5(x,y,z, prefix="1,2,3");
   211         -  INSERT INTO t1(t1, rowid) VALUES('pgsz', 32);
          211  +  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
   212    212   }
   213    213   
   214    214   proc doc {} {
   215    215     set v [list aaa aab abc abcde b c d dd ddd dddd ddddd]
   216    216     set ret [list]
   217    217     for {set j 0} {$j < 20} {incr j} {
   218    218       lappend ret [lindex $v [expr int(rand()*[llength $v])]]

Changes to test/fts5ab.test.

    55     55   } {}
    56     56   
    57     57   #-------------------------------------------------------------------------
    58     58   
    59     59   reset_db
    60     60   do_execsql_test 2.1 {
    61     61     CREATE VIRTUAL TABLE t1 USING fts5(x);
    62         -  INSERT INTO t1(t1, rowid) VALUES('pgsz', 32);
           62  +  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
    63     63     INSERT INTO t1 VALUES('one');
    64     64     INSERT INTO t1 VALUES('two');
    65     65     INSERT INTO t1 VALUES('three');
    66     66   }
    67     67   
    68     68   do_catchsql_test 2.2 {
    69     69     SELECT rowid, * FROM t1 WHERE t1 MATCH 'AND AND'
................................................................................
    95     95   }
    96     96   
    97     97   #-------------------------------------------------------------------------
    98     98   #
    99     99   reset_db
   100    100   do_execsql_test 3.0 {
   101    101     CREATE VIRTUAL TABLE t1 USING fts5(a,b);
   102         -  INSERT INTO t1(t1, rowid) VALUES('pgsz', 32);
          102  +  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
   103    103   }
   104    104   
   105    105   foreach {tn a b} {
   106    106      1 {abashed abandons abase abash abaft} {abases abased}
   107    107      2 {abasing abases abaft abated abandons} {abases abandoned}
   108    108      3 {abatement abash abash abated abase} {abasements abashing}
   109    109      4 {abaft abasements abase abasement abasing} {abasement abases}

Changes to test/fts5ac.test.

    21     21   ifcapable !fts5 {
    22     22     finish_test
    23     23     return
    24     24   }
    25     25   
    26     26   do_execsql_test 1.0 {
    27     27     CREATE VIRTUAL TABLE xx USING fts5(x,y);
    28         -  INSERT INTO xx(xx, rowid) VALUES('pgsz', 32);
           28  +  INSERT INTO xx(xx, rank) VALUES('pgsz', 32);
    29     29   }
    30     30   
    31     31   set data {
    32     32       0   {p o q e z k z p n f y u z y n y}   {l o o l v v k}
    33     33       1   {p k h h p y l l h i p v n}         {p p l u r i f a j g e r r x w}
    34     34       2   {l s z j k i m p s}                 {l w e j t j e e i t w r o p o}
    35     35       3   {x g y m y m h p}                   {k j j b r e y y a k y}

Changes to test/fts5ad.test.

    51     51       SELECT rowid FROM yy WHERE yy MATCH $match ORDER BY rowid ASC
    52     52     } $res
    53     53   }
    54     54   
    55     55   foreach {T create} {
    56     56     2 {
    57     57       CREATE VIRTUAL TABLE t1 USING fts5(a, b);
    58         -    INSERT INTO t1(t1, rowid) VALUES('pgsz', 32);
           58  +    INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
    59     59     }
    60     60     
    61     61     3 {
    62     62       CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix=1,2,3,4,5);
    63         -    INSERT INTO t1(t1, rowid) VALUES('pgsz', 32);
           63  +    INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
    64     64     }
    65     65   
    66     66   } {
    67     67   
    68     68     do_test $T.1 { 
    69     69       execsql { DROP TABLE IF EXISTS t1 }
    70     70       execsql $create

Changes to test/fts5ae.test.

    21     21   ifcapable !fts5 {
    22     22     finish_test
    23     23     return
    24     24   }
    25     25   
    26     26   do_execsql_test 1.0 {
    27     27     CREATE VIRTUAL TABLE t1 USING fts5(a, b);
    28         -  INSERT INTO t1(t1, rowid) VALUES('pgsz', 32);
           28  +  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
    29     29   }
    30     30   
    31     31   do_execsql_test 1.1 {
    32     32     INSERT INTO t1 VALUES('hello', 'world');
    33     33     SELECT rowid FROM t1 WHERE t1 MATCH 'hello' ORDER BY rowid ASC;
    34     34   } {1}
    35     35   

Changes to test/fts5ah.test.

    24     24   
    25     25   #-------------------------------------------------------------------------
    26     26   # This file contains tests for very large doclists.
    27     27   #
    28     28   
    29     29   do_test 1.0 {
    30     30     execsql { CREATE VIRTUAL TABLE t1 USING fts5(a) }
    31         -  execsql { INSERT INTO t1(t1, rowid) VALUES('pgsz', 128) }
           31  +  execsql { INSERT INTO t1(t1, rank) VALUES('pgsz', 128) }
    32     32     for {set i 1} {$i <= 10000} {incr i} {
    33     33       set v {x x x x x x x x x x x x x x x x x x x x}
    34     34       if {($i % 2139)==0} {lset v 3 Y ; lappend Y $i}
    35     35       if {($i % 1577)==0} {lset v 5 W ; lappend W $i}
    36     36       execsql { INSERT INTO t1 VALUES($v) }
    37     37     }
    38     38   } {}
................................................................................
    67     67   } {1}
    68     68   
    69     69   do_test 1.5 {
    70     70     set fwd [execsql_reads {SELECT rowid FROM t1 WHERE t1 MATCH 'x' }]
    71     71     set bwd [execsql_reads {
    72     72       SELECT rowid FROM t1 WHERE t1 MATCH 'x' ORDER BY 1 ASC 
    73     73     }]
    74         -  expr {$bwd < $fwd + 10}
           74  +  expr {$bwd < $fwd + 12}
    75     75   } {1}
    76     76   
    77     77   foreach {tn q res} "
    78     78     1 { SELECT rowid FROM t1 WHERE t1 MATCH 'w + x'   }  [list $W]
    79     79     2 { SELECT rowid FROM t1 WHERE t1 MATCH 'x + w'   }  [list $W]
    80     80     3 { SELECT rowid FROM t1 WHERE t1 MATCH 'x AND w' }  [list $W]
    81     81     4 { SELECT rowid FROM t1 WHERE t1 MATCH 'y AND x' }  [list $Y]

Changes to test/fts5aj.test.

    42     42     }
    43     43     set res
    44     44   }
    45     45   
    46     46   expr srand(0)
    47     47   do_execsql_test 1.0 {
    48     48     CREATE VIRTUAL TABLE t1 USING fts5(x);
    49         -  INSERT INTO t1(t1, rowid) VALUES('pgsz', 64);
           49  +  INSERT INTO t1(t1, rank) VALUES('pgsz', 64);
    50     50   }
    51     51   
    52     52   for {set iTest 0} {$iTest < 50000} {incr iTest} {
    53     53     if {$iTest > 1000} { execsql { DELETE FROM t1 WHERE rowid=($iTest-1000) } }
    54     54     set new [doc]
    55     55     execsql { INSERT INTO t1 VALUES($new) }
    56     56     if {$iTest==10000} { set sz1 [db one {SELECT count(*) FROM t1_data}] }