/ Check-in [83491c56]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add a %_config table to fts5.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts5
Files: files | file ages | folders
SHA1:83491c56661ca78f96020ba68184bb3fb19e674f
User & Date: dan 2014-11-27 20:03:45
Context
2014-11-28
20:01
Add a cookie mechanism to ensure that the %_config table is re-read as required. check-in: bb4a37b5 user: dan tags: fts5
2014-11-27
20:03
Add a %_config table to fts5. check-in: 83491c56 user: dan tags: fts5
2014-11-24
16:24
Add the auxiliary highlight() function to fts5. check-in: 05909237 user: dan tags: fts5
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5.c.

   335    335       );
   336    336     }
   337    337   
   338    338     /* Call sqlite3_declare_vtab() */
   339    339     if( rc==SQLITE_OK ){
   340    340       rc = sqlite3Fts5ConfigDeclareVtab(pConfig);
   341    341     }
          342  +
          343  +  /* Load the contents of %_config */
          344  +  if( rc==SQLITE_OK ){
          345  +    rc = sqlite3Fts5ConfigLoad(pConfig);
          346  +  }
   342    347   
   343    348     if( rc!=SQLITE_OK ){
   344    349       fts5FreeVtab(pTab, 0);
   345    350       pTab = 0;
   346    351     }else if( bCreate ){
   347    352       fts5CheckTransactionState(pTab, FTS5_BEGIN, 0);
   348    353     }
................................................................................
   883    888     return rc;
   884    889   }
   885    890   
   886    891   /*
   887    892   ** This function is called to handle an FTS INSERT command. In other words,
   888    893   ** an INSERT statement of the form:
   889    894   **
   890         -**     INSERT INTO fts(fts) VALUES($pVal)
          895  +**     INSERT INTO fts(fts) VALUES($pCmd)
          896  +**     INSERT INTO fts(fts, rank) VALUES($pCmd, $pVal)
   891    897   **
   892    898   ** Argument pVal is the value assigned to column "fts" by the INSERT 
   893    899   ** statement. This function returns SQLITE_OK if successful, or an SQLite
   894    900   ** error code if an error occurs.
   895    901   **
   896    902   ** The commands implemented by this function are documented in the "Special
   897    903   ** INSERT Directives" section of the documentation. It should be updated if
   898    904   ** more commands are added to this function.
   899    905   */
   900         -static int fts5SpecialCommand(Fts5Table *pTab, sqlite3_value *pVal){
   901         -  const char *z = (const char*)sqlite3_value_text(pVal);
   902         -  int n = sqlite3_value_bytes(pVal);
   903         -  int rc = SQLITE_ERROR;
          906  +static int fts5SpecialCommand(
          907  +  Fts5Table *pTab,                /* Fts5 table object */
          908  +  sqlite3_value *pCmd,            /* Value inserted into special column */
          909  +  sqlite3_value *pVal             /* Value inserted into rowid column */
          910  +){
          911  +  const char *z = (const char*)sqlite3_value_text(pCmd);
          912  +  int rc = SQLITE_OK;
          913  +  int bError = 0;
   904    914   
   905    915     if( 0==sqlite3_stricmp("integrity-check", z) ){
   906    916       rc = sqlite3Fts5StorageIntegrity(pTab->pStorage);
   907         -  }else
   908         -
   909         -  if( n>5 && 0==sqlite3_strnicmp("pgsz=", z, 5) ){
   910         -    int pgsz = atoi(&z[5]);
   911         -    if( pgsz<32 ) pgsz = 32;
   912         -    sqlite3Fts5IndexPgsz(pTab->pIndex, pgsz);
   913         -    rc = SQLITE_OK;
   914         -  }else
   915         -  
   916         -  if( n>10 && 0==sqlite3_strnicmp("automerge=", z, 10) ){
   917         -    int nAutomerge = atoi(&z[10]);
   918         -    sqlite3Fts5IndexAutomerge(pTab->pIndex, nAutomerge);
   919         -    rc = SQLITE_OK;
          917  +  }else{
          918  +    rc = sqlite3Fts5ConfigSetValue(pTab->pConfig, z, pVal, &bError);
          919  +    if( rc==SQLITE_OK && bError ){
          920  +      rc = SQLITE_ERROR;
          921  +    }else{
          922  +      rc = sqlite3Fts5StorageConfigValue(pTab->pStorage, z, pVal);
          923  +    }
   920    924     }
   921         -
   922    925     return rc;
   923    926   }
   924    927   
   925    928   /* 
   926    929   ** This function is the implementation of the xUpdate callback used by 
   927    930   ** FTS3 virtual tables. It is invoked by SQLite each time a row is to be
   928    931   ** inserted, updated or deleted.
................................................................................
   949    952     **   2. The "new" rowid.
   950    953     **   3. Values for each of the nCol matchable columns.
   951    954     **   4. Values for the two hidden columns (<tablename> and "rank").
   952    955     */
   953    956     assert( nArg==1 || nArg==(2 + pConfig->nCol + 2) );
   954    957   
   955    958     if( nArg>1 && SQLITE_NULL!=sqlite3_value_type(apVal[2 + pConfig->nCol]) ){
   956         -    return fts5SpecialCommand(pTab, apVal[2 + pConfig->nCol]);
          959  +    return fts5SpecialCommand(pTab, 
          960  +        apVal[2 + pConfig->nCol], apVal[2 + pConfig->nCol + 1]
          961  +    );
   957    962     }
   958    963   
   959    964     eType0 = sqlite3_value_type(apVal[0]);
   960    965     eConflict = sqlite3_vtab_on_conflict(pConfig->db);
   961    966   
   962    967     assert( eType0==SQLITE_INTEGER || eType0==SQLITE_NULL );
   963    968     if( eType0==SQLITE_INTEGER ){
................................................................................
  1100   1105           sqlite3Fts5PoslistReaderInit(-1, a, n, &aIter[i]);
  1101   1106         }
  1102   1107   
  1103   1108         while( 1 ){
  1104   1109           int *aInst;
  1105   1110           int iBest = -1;
  1106   1111           for(i=0; i<nIter; i++){
  1107         -          if( aIter[i].bEof==0 && (iBest<0 || aIter[i].iPos<iBest) ){
         1112  +          if( (aIter[i].bEof==0) 
         1113  +           && (iBest<0 || aIter[i].iPos<aIter[iBest].iPos) 
         1114  +          ){
  1108   1115               iBest = i;
  1109   1116             }
  1110   1117           }
  1111   1118   
  1112   1119           if( iBest<0 ) break;
  1113   1120           nInst++;
  1114   1121           if( sqlite3Fts5BufferGrow(&rc, &buf, nInst * sizeof(int) * 3) ) break;

Changes to ext/fts5/fts5Int.h.

    54     54   */
    55     55   
    56     56   typedef struct Fts5Config Fts5Config;
    57     57   
    58     58   /*
    59     59   ** An instance of the following structure encodes all information that can
    60     60   ** be gleaned from the CREATE VIRTUAL TABLE statement.
           61  +**
           62  +** And all information loaded from the %_config table.
    61     63   */
    62     64   struct Fts5Config {
    63     65     sqlite3 *db;                    /* Database handle */
    64     66     char *zDb;                      /* Database holding FTS index (e.g. "main") */
    65     67     char *zName;                    /* Name of FTS index */
    66     68     int nCol;                       /* Number of columns */
    67     69     char **azCol;                   /* Column names */
    68     70     int nPrefix;                    /* Number of prefix indexes */
    69     71     int *aPrefix;                   /* Sizes in bytes of nPrefix prefix indexes */
    70     72     Fts5Tokenizer *pTok;
    71     73     fts5_tokenizer *pTokApi;
           74  +
           75  +  /* Values loaded from the %_config table */
           76  +  int iCookie;                    /* Incremented when %_config is modified */
           77  +  int pgsz;                       /* Approximate page size used in %_data */
    72     78   };
    73     79   
    74     80   int sqlite3Fts5ConfigParse(
    75     81       Fts5Global*, sqlite3*, int, const char **, Fts5Config**, char**
    76     82   );
    77     83   void sqlite3Fts5ConfigFree(Fts5Config*);
    78     84   
................................................................................
    83     89     const char *pText, int nText,   /* Text to tokenize */
    84     90     void *pCtx,                     /* Context passed to xToken() */
    85     91     int (*xToken)(void*, const char*, int, int, int, int)    /* Callback */
    86     92   );
    87     93   
    88     94   void sqlite3Fts5Dequote(char *z);
    89     95   
           96  +/* Load the contents of the %_config table */
           97  +int sqlite3Fts5ConfigLoad(Fts5Config*);
           98  +
           99  +/* Set the value of a single config attribute */
          100  +int sqlite3Fts5ConfigSetValue(Fts5Config*, const char*, sqlite3_value*, int*);
          101  +
    90    102   /*
    91    103   ** End of interface to code in fts5_config.c.
    92    104   **************************************************************************/
    93    105   
    94    106   /**************************************************************************
    95    107   ** Interface to code in fts5_buffer.c.
    96    108   */
................................................................................
   282    294   
   283    295   /* 
   284    296   ** Called during virtual module initialization to register UDF 
   285    297   ** fts5_decode() with SQLite 
   286    298   */
   287    299   int sqlite3Fts5IndexInit(sqlite3*);
   288    300   
   289         -/*
   290         -** Set the page size to use when writing. It doesn't matter if this
   291         -** changes mid-transaction, or if inconsistent values are used by 
   292         -** multiple clients.
   293         -*/
   294         -void sqlite3Fts5IndexPgsz(Fts5Index *p, int pgsz);
   295         -
   296    301   void sqlite3Fts5IndexAutomerge(Fts5Index *p, int nMerge);
   297    302   
   298    303   /*
   299    304   ** Return the total number of entries read from the %_data table by 
   300    305   ** this connection since it was created.
   301    306   */
   302    307   int sqlite3Fts5IndexReads(Fts5Index *p);
................................................................................
   360    365   
   361    366   typedef struct Fts5Storage Fts5Storage;
   362    367   
   363    368   int sqlite3Fts5StorageOpen(Fts5Config*, Fts5Index*, int, Fts5Storage**, char**);
   364    369   int sqlite3Fts5StorageClose(Fts5Storage *p, int bDestroy);
   365    370   
   366    371   int sqlite3Fts5DropTable(Fts5Config*, const char *zPost);
   367         -int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, char **pzErr);
          372  +int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, int, char **);
   368    373   
   369    374   int sqlite3Fts5StorageDelete(Fts5Storage *p, i64);
   370    375   int sqlite3Fts5StorageInsert(Fts5Storage *p, sqlite3_value **apVal, int, i64*);
   371    376   
   372    377   int sqlite3Fts5StorageIntegrity(Fts5Storage *p);
   373    378   
   374    379   int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt **);
................................................................................
   376    381   
   377    382   int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol);
   378    383   int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnAvg);
   379    384   int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow);
   380    385   
   381    386   int sqlite3Fts5StorageSync(Fts5Storage *p, int bCommit);
   382    387   int sqlite3Fts5StorageRollback(Fts5Storage *p);
          388  +
          389  +int sqlite3Fts5StorageConfigValue(Fts5Storage *p, const char*, sqlite3_value*);
   383    390   
   384    391   /*
   385    392   ** End of interface to code in fts5_storage.c.
   386    393   **************************************************************************/
   387    394   
   388    395   
   389    396   /**************************************************************************

Changes to ext/fts5/fts5_aux.c.

    17     17   /*************************************************************************
    18     18   ** Start of highlight() implementation.
    19     19   */
    20     20   typedef struct HighlightContext HighlightContext;
    21     21   struct HighlightContext {
    22     22     const Fts5ExtensionApi *pApi;   /* API offered by current FTS version */
    23     23     Fts5Context *pFts;              /* First arg to pass to pApi functions */
           24  +  int nInst;                      /* Total number of phrase instances */
    24     25     int iInst;                      /* Current phrase instance index */
    25     26     int iStart;                     /* First token of current phrase */
    26     27     int iEnd;                       /* Last token of current phrase */
    27     28   
    28     29     const char *zOpen;              /* Opening highlight */
    29     30     const char *zClose;             /* Closing highlight */
    30     31     int iCol;                       /* Column to read from */
................................................................................
    31     32   
    32     33     const char *zIn;                /* Input text */
    33     34     int nIn;                        /* Size of input text in bytes */
    34     35     int iOff;                       /* Current offset within zIn[] */
    35     36     char *zOut;                     /* Output value */
    36     37   };
    37     38   
    38         -static int fts5HighlightAppend(HighlightContext *p, const char *z, int n){
    39         -  if( n<0 ) n = strlen(z);
    40         -  p->zOut = sqlite3_mprintf("%z%.*s", p->zOut, n, z);
    41         -  if( p->zOut==0 ) return SQLITE_NOMEM;
    42         -  return SQLITE_OK;
           39  +/*
           40  +** Append text to the HighlightContext output string - p->zOut. Argument
           41  +** z points to a buffer containing n bytes of text to append. If n is 
           42  +** negative, everything up until the first '\0' is appended to the output.
           43  +*/
           44  +static void fts5HighlightAppend(
           45  +  int *pRc, 
           46  +  HighlightContext *p, 
           47  +  const char *z, int n
           48  +){
           49  +  if( *pRc==SQLITE_OK ){
           50  +    if( n<0 ) n = strlen(z);
           51  +    p->zOut = sqlite3_mprintf("%z%.*s", p->zOut, n, z);
           52  +    if( p->zOut==0 ) *pRc = SQLITE_NOMEM;
           53  +  }
    43     54   }
    44     55   
    45     56   static int fts5HighlightCb(
    46     57     void *pContext,                 /* Pointer to HighlightContext object */
    47     58     const char *pToken,             /* Buffer containing token */
    48     59     int nToken,                     /* Size of token in bytes */
    49         -  int iStart,                     /* Start offset of token */
    50         -  int iEnd,                       /* End offset of token */
           60  +  int iStartOff,                  /* Start offset of token */
           61  +  int iEndOff,                    /* End offset of token */
    51     62     int iPos                        /* Position offset of token */
    52     63   ){
    53     64     HighlightContext *p = (HighlightContext*)pContext;
    54     65     int rc = SQLITE_OK;
    55     66   
    56     67     if( iPos==p->iStart ){
    57         -    rc = fts5HighlightAppend(p, &p->zIn[p->iOff], iStart - p->iOff);
    58         -    p->iOff = iStart;
    59         -    if( rc==SQLITE_OK ){
    60         -      rc = fts5HighlightAppend(p, p->zOpen, -1);
    61         -    }
    62         -  }
    63         -  
    64         -  if( rc==SQLITE_OK ){
    65         -    rc = fts5HighlightAppend(p, &p->zIn[p->iOff], iEnd - p->iOff);
    66         -    p->iOff = iEnd;
           68  +    fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iStartOff - p->iOff);
           69  +    fts5HighlightAppend(&rc, p, p->zOpen, -1);
           70  +    p->iOff = iStartOff;
    67     71     }
    68     72   
    69         -  if( rc==SQLITE_OK && iPos==p->iEnd ){
           73  +  if( iPos==p->iEnd ){
    70     74       int bClose = 1;
    71         -    do{
           75  +    for(p->iInst++; rc==SQLITE_OK && p->iInst<p->nInst; p->iInst++){
    72     76         int iP, iPCol, iOff;
    73         -      rc = p->pApi->xInst(p->pFts, ++p->iInst, &iP, &iPCol, &iOff);
    74         -      if( rc==SQLITE_RANGE || iPCol!=p->iCol ){
    75         -        p->iStart = -1;
    76         -        p->iEnd = -1;
    77         -        rc = SQLITE_OK;
           77  +      rc = p->pApi->xInst(p->pFts, p->iInst, &iP, &iPCol, &iOff);
           78  +      if( iPCol!=p->iCol ){
           79  +        p->iStart = p->iEnd = -1;
    78     80         }else{
    79         -        iEnd = iOff - 1 + p->pApi->xPhraseSize(p->pFts, iP);
           81  +        int iEnd = iOff - 1 + p->pApi->xPhraseSize(p->pFts, iP);
    80     82           if( iEnd<=p->iEnd ) continue;
    81     83           if( iOff<=p->iEnd ) bClose = 0;
    82     84           p->iStart = iOff;
    83     85           p->iEnd = iEnd;
    84     86         }
    85         -    }while( 0 );
           87  +      break;
           88  +    }
    86     89   
    87         -    if( rc==SQLITE_OK && bClose ){
    88         -      rc = fts5HighlightAppend(p, p->zClose, -1);
           90  +    if( bClose ){
           91  +      fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff);
           92  +      fts5HighlightAppend(&rc, p, p->zClose, -1);
           93  +      p->iOff = iEndOff;
    89     94       }
    90     95     }
    91     96   
    92     97     return rc;
    93     98   }
    94     99   
    95    100   static void fts5HighlightFunction(
................................................................................
   103    108     int rc;
   104    109   
   105    110     if( nVal!=3 ){
   106    111       const char *zErr = "wrong number of arguments to function highlight()";
   107    112       sqlite3_result_error(pCtx, zErr, -1);
   108    113       return;
   109    114     }
          115  +
   110    116     memset(&ctx, 0, sizeof(HighlightContext));
   111    117     ctx.iCol = sqlite3_value_int(apVal[0]);
   112    118     ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]);
   113    119     ctx.zClose = (const char*)sqlite3_value_text(apVal[2]);
   114         -  rc = pApi->xColumnText(pFts, ctx.iCol, &ctx.zIn, &ctx.nIn);
   115    120     ctx.pApi = pApi;
   116    121     ctx.pFts = pFts;
          122  +  rc = pApi->xColumnText(pFts, ctx.iCol, &ctx.zIn, &ctx.nIn);
          123  +  if( rc==SQLITE_OK ) rc = pApi->xInstCount(pFts, &ctx.nInst);
   117    124   
   118    125     /* Find the first phrase instance in the right column. */
   119    126     ctx.iStart = -1;
   120    127     ctx.iEnd = -1;
   121         -  while( rc==SQLITE_OK ){
          128  +  for( ; ctx.iInst<ctx.nInst && rc==SQLITE_OK; ctx.iInst++){
   122    129       int iP, iPCol, iOff;
   123    130       rc = pApi->xInst(pFts, ctx.iInst, &iP, &iPCol, &iOff);
   124         -    if( rc==SQLITE_OK && iPCol==ctx.iCol ){
          131  +    if( iPCol==ctx.iCol ){
   125    132         ctx.iStart = iOff;
   126    133         ctx.iEnd = iOff - 1 + pApi->xPhraseSize(pFts, iP);
   127    134         break;
   128    135       }
   129         -    ctx.iInst++;
   130    136     }
   131    137   
   132         -  if( rc==SQLITE_OK || rc==SQLITE_RANGE ){
          138  +  if( rc==SQLITE_OK ){
   133    139       rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx, fts5HighlightCb);
   134    140     }
   135         -  if( rc==SQLITE_OK ){
   136         -    rc = fts5HighlightAppend(&ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff);
   137         -  }
          141  +  fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff);
   138    142   
   139    143     if( rc==SQLITE_OK ){
   140    144       sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT);
   141    145     }else{
   142    146       sqlite3_result_error_code(pCtx, rc);
   143    147     }
   144    148     sqlite3_free(ctx.zOut);

Changes to ext/fts5/fts5_config.c.

    11     11   ******************************************************************************
    12     12   **
    13     13   ** This is an SQLite module implementing full-text search.
    14     14   */
    15     15   
    16     16   #include "fts5Int.h"
    17     17   
           18  +#define FTS5_DEFAULT_PAGE_SIZE   1000
           19  +
    18     20   /*
    19     21   ** Convert an SQL-style quoted string into a normal string by removing
    20     22   ** the quote characters.  The conversion is done in-place.  If the
    21     23   ** input does not begin with a quote character, then this routine
    22     24   ** is a no-op.
    23     25   **
    24     26   ** Examples:
................................................................................
   291    293     const char *pText, int nText,   /* Text to tokenize */
   292    294     void *pCtx,                     /* Context passed to xToken() */
   293    295     int (*xToken)(void*, const char*, int, int, int, int)    /* Callback */
   294    296   ){
   295    297     return pConfig->pTokApi->xTokenize(pConfig->pTok, pCtx, pText, nText, xToken);
   296    298   }
   297    299   
          300  +int sqlite3Fts5ConfigSetValue(
          301  +  Fts5Config *pConfig, 
          302  +  const char *zKey, 
          303  +  sqlite3_value *pVal,
          304  +  int *pbBadkey
          305  +){
          306  +  int rc = SQLITE_OK;
          307  +  if(      0==sqlite3_stricmp(zKey, "cookie") ){
          308  +    pConfig->iCookie = sqlite3_value_int(pVal);
          309  +  }
          310  +  else if( 0==sqlite3_stricmp(zKey, "pgsz") ){
          311  +    if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
          312  +      pConfig->pgsz = sqlite3_value_int(pVal);
          313  +    }else{
          314  +      if( pbBadkey ) *pbBadkey = 1;
          315  +    }
          316  +  }
          317  +  else if( 0==sqlite3_stricmp(zKey, "automerge") ){
          318  +    // todo
          319  +  }
          320  +  else if( 0==sqlite3_stricmp(zKey, "rank") ){
          321  +    // todo
          322  +  }else{
          323  +    if( pbBadkey ) *pbBadkey = 1;
          324  +  }
          325  +  return rc;
          326  +}
          327  +
          328  +/*
          329  +** Load the contents of the %_config table into memory.
          330  +*/
          331  +int sqlite3Fts5ConfigLoad(Fts5Config *pConfig){
          332  +  const char *zSelect = "SELECT k, v FROM %Q.'%q_config'";
          333  +  char *zSql;
          334  +  sqlite3_stmt *p = 0;
          335  +  int rc;
          336  +
          337  +  /* Set default values */
          338  +  pConfig->pgsz = FTS5_DEFAULT_PAGE_SIZE;
          339  +  pConfig->iCookie = 0;
          340  +
          341  +  zSql = sqlite3_mprintf(zSelect, pConfig->zDb, pConfig->zName);
          342  +  if( zSql==0 ){
          343  +    rc = SQLITE_NOMEM;
          344  +  }else{
          345  +    rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &p, 0);
          346  +    sqlite3_free(zSql);
          347  +  }
          348  +
          349  +  assert( rc==SQLITE_OK || p==0 );
          350  +  if( rc==SQLITE_OK ){
          351  +    while( SQLITE_ROW==sqlite3_step(p) ){
          352  +      const char *zK = (const char*)sqlite3_column_text(p, 0);
          353  +      sqlite3_value *pVal = sqlite3_column_value(p, 1);
          354  +      sqlite3Fts5ConfigSetValue(pConfig, zK, pVal, 0);
          355  +    }
          356  +    rc = sqlite3_finalize(p);
          357  +  }
          358  +
          359  +  return rc;
          360  +}
   298    361   

Changes to ext/fts5/fts5_index.c.

    37     37   **     large doclists with very small doclists.
    38     38   **
    39     39   **   * extra fields in the "structure record" record the state of ongoing
    40     40   **     incremental merge operations.
    41     41   **
    42     42   */
    43     43   
    44         -#define FTS5_DEFAULT_PAGE_SIZE   1000
    45         -
    46     44   #define FTS5_WORK_UNIT      64    /* Number of leaf pages in unit of work */
    47     45   #define FTS5_MIN_MERGE       4    /* Minimum number of segments to merge */
    48     46   #define FTS5_CRISIS_MERGE   16    /* Maximum number of segments to merge */
    49     47   
    50     48   #define FTS5_MIN_DLIDX_SIZE  4    /* Add dlidx if this many empty pages */
    51     49   
    52     50   /*
................................................................................
   286    284   
   287    285   /*
   288    286   ** One object per %_data table.
   289    287   */
   290    288   struct Fts5Index {
   291    289     Fts5Config *pConfig;            /* Virtual table configuration */
   292    290     char *zDataTbl;                 /* Name of %_data table */
   293         -  int pgsz;                       /* Target page size for this index */
   294    291     int nMinMerge;                  /* Minimum input segments in a merge */
   295    292     int nCrisisMerge;               /* Maximum allowed segments per level */
   296    293     int nWorkUnit;                  /* Leaf pages in a "unit" of work */
   297    294   
   298    295     /*
   299    296     ** Variables related to the accumulation of tokens and doclists within the
   300    297     ** in-memory hash tables before they are flushed to disk.
................................................................................
  2531   2528     return 0;
  2532   2529   }
  2533   2530   
  2534   2531   /*
  2535   2532   ** Discard all data currently cached in the hash-tables.
  2536   2533   */
  2537   2534   static void fts5IndexDiscardData(Fts5Index *p){
  2538         -  Fts5Config *pConfig = p->pConfig;
  2539         -  int i;
  2540         -  for(i=0; i<=pConfig->nPrefix; i++){
  2541         -    sqlite3Fts5HashClear(p->apHash[i]);
         2535  +  assert( p->apHash || p->nPendingData==0 );
         2536  +  if( p->apHash ){
         2537  +    Fts5Config *pConfig = p->pConfig;
         2538  +    int i;
         2539  +    for(i=0; i<=pConfig->nPrefix; i++){
         2540  +      sqlite3Fts5HashClear(p->apHash[i]);
         2541  +    }
         2542  +    p->nPendingData = 0;
  2542   2543     }
  2543         -  p->nPendingData = 0;
  2544   2544   }
  2545   2545   
  2546   2546   /*
  2547   2547   ** Return the size of the prefix, in bytes, that buffer (nNew/pNew) shares
  2548   2548   ** with buffer (nOld/pOld).
  2549   2549   */
  2550   2550   static int fts5PrefixCompress(
................................................................................
  2626   2626         fts5WriteBtreeGrow(p, pWriter);
  2627   2627         if( p->rc ) return;
  2628   2628       }
  2629   2629       pPage = &pWriter->aWriter[iHeight];
  2630   2630   
  2631   2631       fts5WriteBtreeNEmpty(p, pWriter);
  2632   2632   
  2633         -    if( pPage->buf.n>=p->pgsz ){
         2633  +    if( pPage->buf.n>=p->pConfig->pgsz ){
  2634   2634         /* pPage will be written to disk. The term will be written into the
  2635   2635         ** parent of pPage.  */
  2636   2636         i64 iRowid = FTS5_SEGMENT_ROWID(
  2637   2637             pWriter->iIdx, pWriter->iSegid, iHeight, pPage->pgno
  2638   2638         );
  2639   2639         fts5DataWrite(p, iRowid, pPage->buf.p, pPage->buf.n);
  2640   2640         fts5BufferZero(&pPage->buf);
................................................................................
  2757   2757     /* Update the Fts5PageWriter.term field. */
  2758   2758     fts5BufferSet(&p->rc, &pPage->term, nTerm, pTerm);
  2759   2759   
  2760   2760     pWriter->bFirstRowidInPage = 0;
  2761   2761     pWriter->bFirstRowidInDoclist = 1;
  2762   2762   
  2763   2763     /* If the current leaf page is full, flush it to disk. */
  2764         -  if( pPage->buf.n>=p->pgsz ){
         2764  +  if( pPage->buf.n>=p->pConfig->pgsz ){
  2765   2765       fts5WriteFlushLeaf(p, pWriter);
  2766   2766       pWriter->bFirstRowidInPage = 1;
  2767   2767     }
  2768   2768   }
  2769   2769   
  2770   2770   /*
  2771   2771   ** Append a docid to the writers output. 
................................................................................
  2792   2792       assert( iRowid<pWriter->iPrevRowid );
  2793   2793       fts5BufferAppendVarint(&p->rc, &pPage->buf, pWriter->iPrevRowid - iRowid);
  2794   2794     }
  2795   2795     pWriter->iPrevRowid = iRowid;
  2796   2796     pWriter->bFirstRowidInDoclist = 0;
  2797   2797     pWriter->bFirstRowidInPage = 0;
  2798   2798   
  2799         -  if( pPage->buf.n>=p->pgsz ){
         2799  +  if( pPage->buf.n>=p->pConfig->pgsz ){
  2800   2800       fts5WriteFlushLeaf(p, pWriter);
  2801   2801       pWriter->bFirstRowidInPage = 1;
  2802   2802     }
  2803   2803   }
  2804   2804   
  2805   2805   static void fts5WriteAppendPoslistInt(
  2806   2806     Fts5Index *p, 
  2807   2807     Fts5SegWriter *pWriter,
  2808   2808     int iVal
  2809   2809   ){
  2810   2810     Fts5PageWriter *pPage = &pWriter->aWriter[0];
  2811   2811     fts5BufferAppendVarint(&p->rc, &pPage->buf, iVal);
  2812         -  if( pPage->buf.n>=p->pgsz ){
         2812  +  if( pPage->buf.n>=p->pConfig->pgsz ){
  2813   2813       fts5WriteFlushLeaf(p, pWriter);
  2814   2814       pWriter->bFirstRowidInPage = 1;
  2815   2815     }
  2816   2816   }
  2817   2817   
  2818   2818   static void fts5WriteAppendPoslistData(
  2819   2819     Fts5Index *p, 
................................................................................
  2821   2821     const u8 *aData, 
  2822   2822     int nData
  2823   2823   ){
  2824   2824     Fts5PageWriter *pPage = &pWriter->aWriter[0];
  2825   2825     const u8 *a = aData;
  2826   2826     int n = nData;
  2827   2827     
  2828         -  while( p->rc==SQLITE_OK && (pPage->buf.n + n)>=p->pgsz ){
  2829         -    int nReq = p->pgsz - pPage->buf.n;
         2828  +  while( p->rc==SQLITE_OK && (pPage->buf.n + n)>=p->pConfig->pgsz ){
         2829  +    int nReq = p->pConfig->pgsz - pPage->buf.n;
  2830   2830       int nCopy = 0;
  2831   2831       while( nCopy<nReq ){
  2832   2832         i64 dummy;
  2833   2833         nCopy += getVarint(&a[nCopy], (u64*)&dummy);
  2834   2834       }
  2835   2835       fts5BufferAppendBlob(&p->rc, &pPage->buf, nCopy, a);
  2836   2836       a += nCopy;
................................................................................
  3367   3367     Fts5Index *p;                   /* New object */
  3368   3368   
  3369   3369     *pp = p = (Fts5Index*)sqlite3_malloc(sizeof(Fts5Index));
  3370   3370     if( !p ) return SQLITE_NOMEM;
  3371   3371   
  3372   3372     memset(p, 0, sizeof(Fts5Index));
  3373   3373     p->pConfig = pConfig;
  3374         -  p->pgsz = 1000;
  3375   3374     p->nMinMerge = FTS5_MIN_MERGE;
  3376   3375     p->nCrisisMerge = FTS5_CRISIS_MERGE;
  3377   3376     p->nWorkUnit = FTS5_WORK_UNIT;
  3378   3377     p->nMaxPendingData = 1024*1024;
  3379   3378     p->zDataTbl = sqlite3_mprintf("%s_data", pConfig->zName);
  3380   3379     if( p->zDataTbl==0 ){
  3381   3380       rc = SQLITE_NOMEM;
  3382   3381     }else if( bCreate ){
  3383   3382       int i;
  3384   3383       Fts5Structure s;
  3385   3384       rc = sqlite3Fts5CreateTable(
  3386         -        pConfig, "data", "id INTEGER PRIMARY KEY, block BLOB", pzErr
         3385  +        pConfig, "data", "id INTEGER PRIMARY KEY, block BLOB", 0, pzErr
  3387   3386       );
  3388   3387       if( rc==SQLITE_OK ){
  3389   3388         memset(&s, 0, sizeof(Fts5Structure));
  3390   3389         for(i=0; i<pConfig->nPrefix+1; i++){
  3391   3390           fts5StructureWrite(p, i, &s);
  3392   3391         }
  3393   3392         rc = p->rc;
................................................................................
  3982   3981   int sqlite3Fts5IndexInit(sqlite3 *db){
  3983   3982     int rc = sqlite3_create_function(
  3984   3983         db, "fts5_decode", 2, SQLITE_UTF8, 0, fts5DecodeFunction, 0, 0
  3985   3984     );
  3986   3985     return rc;
  3987   3986   }
  3988   3987   
  3989         -/*
  3990         -** Set the target page size for the index object.
  3991         -*/
  3992         -void sqlite3Fts5IndexPgsz(Fts5Index *p, int pgsz){
  3993         -  p->pgsz = pgsz;
  3994         -}
  3995         -
  3996   3988   /*
  3997   3989   ** Set the minimum number of segments that an auto-merge operation should
  3998   3990   ** attempt to merge together. A value of 1 sets the object to use the 
  3999   3991   ** compile time default. Zero or less disables auto-merge altogether.
  4000   3992   */
  4001   3993   void sqlite3Fts5IndexAutomerge(Fts5Index *p, int nMinMerge){
  4002   3994     if( nMinMerge==1 ){

Changes to ext/fts5/fts5_storage.c.

    16     16   
    17     17   struct Fts5Storage {
    18     18     Fts5Config *pConfig;
    19     19     Fts5Index *pIndex;
    20     20     int bTotalsValid;               /* True if nTotalRow/aTotalSize[] are valid */
    21     21     i64 nTotalRow;                  /* Total number of rows in FTS table */
    22     22     i64 *aTotalSize;                /* Total sizes of each column */ 
    23         -  sqlite3_stmt *aStmt[9];
           23  +  sqlite3_stmt *aStmt[10];
    24     24   };
    25     25   
    26     26   
    27     27   #if FTS5_STMT_SCAN_ASC!=0 
    28     28   # error "FTS5_STMT_SCAN_ASC mismatch" 
    29     29   #endif
    30     30   #if FTS5_STMT_SCAN_DESC!=1 
................................................................................
    38     38   #define FTS5_STMT_REPLACE_CONTENT 4
    39     39   
    40     40   #define FTS5_STMT_DELETE_CONTENT  5
    41     41   #define FTS5_STMT_REPLACE_DOCSIZE  6
    42     42   #define FTS5_STMT_DELETE_DOCSIZE  7
    43     43   
    44     44   #define FTS5_STMT_LOOKUP_DOCSIZE  8
           45  +
           46  +#define FTS5_STMT_REPLACE_CONFIG 9
    45     47   
    46     48   /*
    47     49   ** Prepare the two insert statements - Fts5Storage.pInsertContent and
    48     50   ** Fts5Storage.pInsertDocsize - if they have not already been prepared.
    49     51   ** Return SQLITE_OK if successful, or an SQLite error code if an error
    50     52   ** occurs.
    51     53   */
................................................................................
    66     68         "INSERT INTO %Q.'%q_content' VALUES(%s)",         /* INSERT_CONTENT  */
    67     69         "REPLACE INTO %Q.'%q_content' VALUES(%s)",        /* REPLACE_CONTENT */
    68     70         "DELETE FROM %Q.'%q_content' WHERE id=?",         /* DELETE_CONTENT  */
    69     71         "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)",       /* REPLACE_DOCSIZE  */
    70     72         "DELETE FROM %Q.'%q_docsize' WHERE id=?",         /* DELETE_DOCSIZE  */
    71     73   
    72     74         "SELECT sz FROM %Q.'%q_docsize' WHERE id=?",      /* LOOKUP_DOCSIZE  */
           75  +
           76  +      "REPLACE INTO %Q.'%q_config' VALUES(?,?)",        /* REPLACE_CONFIG */
    73     77       };
    74     78       Fts5Config *pConfig = p->pConfig;
    75     79       char *zSql = 0;
    76     80   
    77     81       if( eStmt==FTS5_STMT_INSERT_CONTENT || eStmt==FTS5_STMT_REPLACE_CONTENT ){
    78     82         int nCol = pConfig->nCol + 1;
    79     83         char *zBind;
................................................................................
   127    131   ** Create the shadow table named zPost, with definition zDefn. Return
   128    132   ** SQLITE_OK if successful, or an SQLite error code otherwise.
   129    133   */
   130    134   int sqlite3Fts5CreateTable(
   131    135     Fts5Config *pConfig,            /* FTS5 configuration */
   132    136     const char *zPost,              /* Shadow table to create (e.g. "content") */
   133    137     const char *zDefn,              /* Columns etc. for shadow table */
          138  +  int bWithout,                   /* True for without rowid */
   134    139     char **pzErr                    /* OUT: Error message */
   135    140   ){
   136    141     int rc;
   137         -  char *zSql = sqlite3_mprintf("CREATE TABLE %Q.'%q_%q'(%s)",
   138         -      pConfig->zDb, pConfig->zName, zPost, zDefn
          142  +  char *zSql = sqlite3_mprintf("CREATE TABLE %Q.'%q_%q'(%s)%s",
          143  +      pConfig->zDb, pConfig->zName, zPost, zDefn, 
          144  +      (bWithout ? " WITHOUT ROWID" :"")
   139    145     );
   140    146     if( zSql==0 ){
   141    147       rc = SQLITE_NOMEM;
   142    148     }else{
   143    149       char *zErr = 0;
   144    150       assert( *pzErr==0 );
   145    151       rc = sqlite3_exec(pConfig->db, zSql, 0, 0, &zErr);
................................................................................
   189    195       if( zDefn==0 ){
   190    196         rc = SQLITE_NOMEM;
   191    197       }else{
   192    198         int iOff = sprintf(zDefn, "id INTEGER PRIMARY KEY");
   193    199         for(i=0; i<pConfig->nCol; i++){
   194    200           iOff += sprintf(&zDefn[iOff], ", c%d", i);
   195    201         }
   196         -      rc = sqlite3Fts5CreateTable(pConfig, "content", zDefn, pzErr);
          202  +      rc = sqlite3Fts5CreateTable(pConfig, "content", zDefn, 0, pzErr);
   197    203       }
   198    204       sqlite3_free(zDefn);
   199    205       if( rc==SQLITE_OK ){
   200    206         rc = sqlite3Fts5CreateTable(
   201         -          pConfig, "docsize", "id INTEGER PRIMARY KEY, sz BLOB", pzErr
          207  +          pConfig, "docsize", "id INTEGER PRIMARY KEY, sz BLOB", 0, pzErr
          208  +      );
          209  +    }
          210  +    if( rc==SQLITE_OK ){
          211  +      rc = sqlite3Fts5CreateTable(
          212  +          pConfig, "config", "k PRIMARY KEY, v", 1, pzErr
   202    213         );
   203    214       }
   204    215     }
   205    216   
   206    217     if( rc ){
   207    218       sqlite3Fts5StorageClose(p, 0);
   208    219       *pp = 0;
................................................................................
   221    232     for(i=0; i<ArraySize(p->aStmt); i++){
   222    233       sqlite3_finalize(p->aStmt[i]);
   223    234     }
   224    235   
   225    236     /* If required, remove the shadow tables from the database */
   226    237     if( bDestroy ){
   227    238       rc = sqlite3Fts5DropTable(p->pConfig, "content");
   228         -    if( rc==SQLITE_OK ) sqlite3Fts5DropTable(p->pConfig, "docsize");
          239  +    if( rc==SQLITE_OK ) rc = sqlite3Fts5DropTable(p->pConfig, "docsize");
          240  +    if( rc==SQLITE_OK ) rc = sqlite3Fts5DropTable(p->pConfig, "config");
   229    241     }
   230    242   
   231    243     sqlite3_free(p);
   232    244     return rc;
   233    245   }
   234    246   
   235    247   typedef struct Fts5InsertCtx Fts5InsertCtx;
................................................................................
   739    751     return sqlite3Fts5IndexSync(p->pIndex, bCommit);
   740    752   }
   741    753   
   742    754   int sqlite3Fts5StorageRollback(Fts5Storage *p){
   743    755     p->bTotalsValid = 0;
   744    756     return sqlite3Fts5IndexRollback(p->pIndex);
   745    757   }
          758  +
          759  +int sqlite3Fts5StorageConfigValue(
          760  +  Fts5Storage *p, 
          761  +  const char *z, 
          762  +  sqlite3_value *pVal
          763  +){
          764  +  sqlite3_stmt *pReplace = 0;
          765  +  int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_CONFIG, &pReplace);
          766  +  if( rc==SQLITE_OK ){
          767  +    sqlite3_bind_text(pReplace, 1, z, -1, SQLITE_TRANSIENT);
          768  +    sqlite3_bind_value(pReplace, 2, pVal);
          769  +    sqlite3_step(pReplace);
          770  +    rc = sqlite3_reset(pReplace);
          771  +  }
          772  +  return rc;
          773  +}
          774  +
   746    775   

Changes to test/fts5aa.test.

    26     26     CREATE VIRTUAL TABLE t1 USING fts5(a, b, c);
    27     27     SELECT name, sql FROM sqlite_master;
    28     28   } {
    29     29     t1 {CREATE VIRTUAL TABLE t1 USING fts5(a, b, c)}
    30     30     t1_data {CREATE TABLE 't1_data'(id INTEGER PRIMARY KEY, block BLOB)}
    31     31     t1_content {CREATE TABLE 't1_content'(id INTEGER PRIMARY KEY, c0, c1, c2)}
    32     32     t1_docsize {CREATE TABLE 't1_docsize'(id INTEGER PRIMARY KEY, sz BLOB)}
           33  +  t1_config {CREATE TABLE 't1_config'(k PRIMARY KEY, v) WITHOUT ROWID}
    33     34   }
    34     35   
    35     36   do_execsql_test 1.1 {
    36     37     DROP TABLE t1;
    37     38     SELECT name, sql FROM sqlite_master;
    38     39   } {
    39     40   }
................................................................................
    80     81   }
    81     82   
    82     83   #-------------------------------------------------------------------------
    83     84   #
    84     85   reset_db
    85     86   do_execsql_test 4.0 {
    86     87     CREATE VIRTUAL TABLE t1 USING fts5(x,y);
    87         -  INSERT INTO t1(t1) VALUES('pgsz=32');
           88  +  INSERT INTO t1(t1, rowid) VALUES('pgsz', 32);
    88     89   }
    89     90   foreach {i x y} {
    90     91      1  {g f d b f} {h h e i a}
    91     92      2  {f i g j e} {i j c f f}
    92     93      3  {e e i f a} {e h f d f}
    93     94      4  {h j f j i} {h a c f j}
    94     95      5  {d b j c g} {f e i b e}
................................................................................
   104    105   }
   105    106   
   106    107   #-------------------------------------------------------------------------
   107    108   #
   108    109   reset_db
   109    110   do_execsql_test 5.0 {
   110    111     CREATE VIRTUAL TABLE t1 USING fts5(x,y);
   111         -  INSERT INTO t1(t1) VALUES('pgsz=32');
          112  +  INSERT INTO t1(t1, rowid) VALUES('pgsz', 32);
   112    113   }
   113    114   foreach {i x y} {
   114    115      1  {dd abc abc abc abcde} {aaa dd ddd ddd aab}
   115    116      2  {dd aab d aaa b} {abcde c aaa aaa aaa}
   116    117      3  {abcde dd b b dd} {abc abc d abc ddddd}
   117    118      4  {aaa abcde dddd dddd abcde} {abc b b abcde abc}
   118    119      5  {aab dddd d dddd c} {ddd abcde dddd abcde c}
................................................................................
   129    130   
   130    131   #-------------------------------------------------------------------------
   131    132   #
   132    133   breakpoint
   133    134   reset_db
   134    135   do_execsql_test 6.0 {
   135    136     CREATE VIRTUAL TABLE t1 USING fts5(x,y);
   136         -  INSERT INTO t1(t1) VALUES('pgsz=32');
          137  +  INSERT INTO t1(t1, rowid) VALUES('pgsz', 32);
   137    138   }
   138    139   
   139    140   do_execsql_test 6.1 {
   140    141     INSERT  INTO t1(rowid, x, y) VALUES(22, 'a b c', 'c b a');
   141    142     REPLACE INTO t1(rowid, x, y) VALUES(22, 'd e f', 'f e d');
   142    143   }
   143    144   
................................................................................
   147    148   
   148    149   #-------------------------------------------------------------------------
   149    150   #
   150    151   reset_db
   151    152   expr srand(0)
   152    153   do_execsql_test 7.0 {
   153    154     CREATE VIRTUAL TABLE t1 USING fts5(x,y,z);
   154         -  INSERT INTO t1(t1) VALUES('pgsz=32');
          155  +  INSERT INTO t1(t1, rowid) VALUES('pgsz', 32);
   155    156   }
   156    157   
   157    158   proc doc {} {
   158    159     set v [list aaa aab abc abcde b c d dd ddd dddd ddddd]
   159    160     set ret [list]
   160    161     for {set j 0} {$j < 20} {incr j} {
   161    162       lappend ret [lindex $v [expr int(rand()*[llength $v])]]
................................................................................
   186    187   }
   187    188   
   188    189   #-------------------------------------------------------------------------
   189    190   #
   190    191   reset_db
   191    192   do_execsql_test 8.0 {
   192    193     CREATE VIRTUAL TABLE t1 USING fts5(x, prefix="1,2,3");
   193         -  INSERT INTO t1(t1) VALUES('pgsz=32');
          194  +  INSERT INTO t1(t1, rowid) VALUES('pgsz', 32);
   194    195   }
   195    196   
   196    197   do_execsql_test 8.1 {
   197    198     INSERT INTO t1 VALUES('the quick brown fox');
   198    199     INSERT INTO t1(t1) VALUES('integrity-check');
   199    200   }
   200    201   
................................................................................
   203    204   #
   204    205   reset_db
   205    206   
   206    207   expr srand(0)
   207    208   
   208    209   do_execsql_test 9.0 {
   209    210     CREATE VIRTUAL TABLE t1 USING fts5(x,y,z, prefix="1,2,3");
   210         -  INSERT INTO t1(t1) VALUES('pgsz=32');
          211  +  INSERT INTO t1(t1, rowid) VALUES('pgsz', 32);
   211    212   }
   212    213   
   213    214   proc doc {} {
   214    215     set v [list aaa aab abc abcde b c d dd ddd dddd ddddd]
   215    216     set ret [list]
   216    217     for {set j 0} {$j < 20} {incr j} {
   217    218       lappend ret [lindex $v [expr int(rand()*[llength $v])]]

Changes to test/fts5ab.test.

    55     55   } {}
    56     56   
    57     57   #-------------------------------------------------------------------------
    58     58   
    59     59   reset_db
    60     60   do_execsql_test 2.1 {
    61     61     CREATE VIRTUAL TABLE t1 USING fts5(x);
    62         -  INSERT INTO t1(t1) VALUES('pgsz=32');
           62  +  INSERT INTO t1(t1, rowid) VALUES('pgsz', 32);
    63     63     INSERT INTO t1 VALUES('one');
    64     64     INSERT INTO t1 VALUES('two');
    65     65     INSERT INTO t1 VALUES('three');
    66     66   }
    67     67   
    68     68   do_catchsql_test 2.2 {
    69     69     SELECT rowid, * FROM t1 WHERE t1 MATCH 'AND AND'
................................................................................
    95     95   }
    96     96   
    97     97   #-------------------------------------------------------------------------
    98     98   #
    99     99   reset_db
   100    100   do_execsql_test 3.0 {
   101    101     CREATE VIRTUAL TABLE t1 USING fts5(a,b);
   102         -  INSERT INTO t1(t1) VALUES('pgsz=32');
          102  +  INSERT INTO t1(t1, rowid) VALUES('pgsz', 32);
   103    103   }
   104    104   
   105    105   foreach {tn a b} {
   106    106      1 {abashed abandons abase abash abaft} {abases abased}
   107    107      2 {abasing abases abaft abated abandons} {abases abandoned}
   108    108      3 {abatement abash abash abated abase} {abasements abashing}
   109    109      4 {abaft abasements abase abasement abasing} {abasement abases}

Changes to test/fts5ac.test.

    21     21   ifcapable !fts5 {
    22     22     finish_test
    23     23     return
    24     24   }
    25     25   
    26     26   do_execsql_test 1.0 {
    27     27     CREATE VIRTUAL TABLE xx USING fts5(x,y);
    28         -  INSERT INTO xx(xx) VALUES('pgsz=32');
           28  +  INSERT INTO xx(xx, rowid) VALUES('pgsz', 32);
    29     29   }
    30     30   
    31     31   set data {
    32     32       0   {p o q e z k z p n f y u z y n y}   {l o o l v v k}
    33     33       1   {p k h h p y l l h i p v n}         {p p l u r i f a j g e r r x w}
    34     34       2   {l s z j k i m p s}                 {l w e j t j e e i t w r o p o}
    35     35       3   {x g y m y m h p}                   {k j j b r e y y a k y}

Changes to test/fts5ad.test.

    51     51       SELECT rowid FROM yy WHERE yy MATCH $match ORDER BY rowid ASC
    52     52     } $res
    53     53   }
    54     54   
    55     55   foreach {T create} {
    56     56     2 {
    57     57       CREATE VIRTUAL TABLE t1 USING fts5(a, b);
    58         -    INSERT INTO t1(t1) VALUES('pgsz=32');
           58  +    INSERT INTO t1(t1, rowid) VALUES('pgsz', 32);
    59     59     }
    60     60     
    61     61     3 {
    62     62       CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix=1,2,3,4,5);
    63         -    INSERT INTO t1(t1) VALUES('pgsz=32');
           63  +    INSERT INTO t1(t1, rowid) VALUES('pgsz', 32);
    64     64     }
    65     65   
    66     66   } {
    67     67   
    68     68     do_test $T.1 { 
    69     69       execsql { DROP TABLE IF EXISTS t1 }
    70     70       execsql $create

Changes to test/fts5ae.test.

    21     21   ifcapable !fts5 {
    22     22     finish_test
    23     23     return
    24     24   }
    25     25   
    26     26   do_execsql_test 1.0 {
    27     27     CREATE VIRTUAL TABLE t1 USING fts5(a, b);
    28         -  INSERT INTO t1(t1) VALUES('pgsz=32');
           28  +  INSERT INTO t1(t1, rowid) VALUES('pgsz', 32);
    29     29   }
    30     30   
    31     31   do_execsql_test 1.1 {
    32     32     INSERT INTO t1 VALUES('hello', 'world');
    33     33     SELECT rowid FROM t1 WHERE t1 MATCH 'hello' ORDER BY rowid ASC;
    34     34   } {1}
    35     35   

Changes to test/fts5ah.test.

    24     24   
    25     25   #-------------------------------------------------------------------------
    26     26   # This file contains tests for very large doclists.
    27     27   #
    28     28   
    29     29   do_test 1.0 {
    30     30     execsql { CREATE VIRTUAL TABLE t1 USING fts5(a) }
    31         -  execsql { INSERT INTO t1(t1) VALUES('pgsz=128') }
           31  +  execsql { INSERT INTO t1(t1, rowid) VALUES('pgsz', 128) }
    32     32     for {set i 1} {$i <= 10000} {incr i} {
    33     33       set v {x x x x x x x x x x x x x x x x x x x x}
    34     34       if {($i % 2139)==0} {lset v 3 Y ; lappend Y $i}
    35     35       if {($i % 1577)==0} {lset v 5 W ; lappend W $i}
    36     36       execsql { INSERT INTO t1 VALUES($v) }
    37     37     }
    38     38   } {}

Changes to test/fts5aj.test.

    42     42     }
    43     43     set res
    44     44   }
    45     45   
    46     46   expr srand(0)
    47     47   do_execsql_test 1.0 {
    48     48     CREATE VIRTUAL TABLE t1 USING fts5(x);
    49         -  INSERT INTO t1(t1) VALUES('pgsz=64');
           49  +  INSERT INTO t1(t1, rowid) VALUES('pgsz', 64);
    50     50   }
    51     51   
    52     52   for {set iTest 0} {$iTest < 50000} {incr iTest} {
    53     53     if {$iTest > 1000} { execsql { DELETE FROM t1 WHERE rowid=($iTest-1000) } }
    54     54     set new [doc]
    55     55     execsql { INSERT INTO t1 VALUES($new) }
    56     56     if {$iTest==10000} { set sz1 [db one {SELECT count(*) FROM t1_data}] }

Changes to test/fts5ak.test.

   101    101   }
   102    102   
   103    103   do_execsql_test 2.5 {
   104    104     SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c c+d+e'
   105    105   } {
   106    106     {a [b c d e] f g h i j}
   107    107   }
          108  +
          109  +do_execsql_test 2.6.1 {
          110  +  SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'f d'
          111  +} {
          112  +  {a b c [d] e [f] g h i j}
          113  +}
          114  +
          115  +do_execsql_test 2.6.2 {
          116  +  SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'd f'
          117  +} {
          118  +  {a b c [d] e [f] g h i j}
          119  +}
   108    120   
   109    121   
   110    122   finish_test
   111    123   

Added test/fts5al.test.

            1  +# 2014 November 24
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this script is testing the FTS5 module.
           13  +#
           14  +# Specifically, this function tests the %_config table.
           15  +#
           16  +
           17  +set testdir [file dirname $argv0]
           18  +source $testdir/tester.tcl
           19  +set testprefix fts5al
           20  +
           21  +# If SQLITE_ENABLE_FTS5 is defined, omit this file.
           22  +ifcapable !fts5 {
           23  +  finish_test
           24  +  return
           25  +}
           26  +
           27  +do_execsql_test 1.1 {
           28  +  CREATE VIRTUAL TABLE ft1 USING fts5(x);
           29  +  SELECT * FROM ft1_config;
           30  +} {}
           31  +
           32  +do_execsql_test 1.2 {
           33  +  INSERT INTO ft1(ft1, rank) VALUES('pgsz', 32);
           34  +  SELECT * FROM ft1_config;
           35  +} {pgsz 32}
           36  +
           37  +do_execsql_test 1.3 {
           38  +  INSERT INTO ft1(ft1, rank) VALUES('pgsz', 64);
           39  +  SELECT * FROM ft1_config;
           40  +} {pgsz 64}
           41  +
           42  +finish_test
           43  +