/ Check-in [19504c41]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix the xColumnSize() extension API.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts5
Files: files | file ages | folders
SHA1: 19504c4108472d2ad1281221642b8bd06eb69f4e
User & Date: dan 2014-07-21 11:44:47
Context
2014-07-21
14:22
Add the xTokenize extension API. check-in: 8c6b0aff user: dan tags: fts5
11:44
Fix the xColumnSize() extension API. check-in: 19504c41 user: dan tags: fts5
2014-07-19
20:27
Add simple tests for the xColumnText() extension api. check-in: 1e9053ab user: dan tags: fts5
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5.c.

   615    615   
   616    616   static int fts5ApiColumnCount(Fts5Context *pCtx){
   617    617     Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
   618    618     return ((Fts5Table*)(pCsr->base.pVtab))->pConfig->nCol;
   619    619   }
   620    620   
   621    621   static int fts5ApiColumnAvgSize(Fts5Context *pCtx, int iCol, int *pnToken){
   622         -  assert( 0 );
   623         -  return 0;
          622  +  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
          623  +  Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
          624  +  return sqlite3Fts5StorageAvgsize(pTab->pStorage, iCol, pnToken);
   624    625   }
   625    626   
   626    627   static int fts5ApiTokenize(
   627    628     Fts5Context *pCtx, 
   628    629     const char *pText, int nText, 
   629    630     void *pUserData,
   630    631     int (*xToken)(void*, const char*, int, int, int, int)

Changes to ext/fts5/fts5Int.h.

   263    263   int sqlite3Fts5IndexIntegrityCheck(Fts5Index*, u64 cksum);
   264    264   
   265    265   /* Called during startup to register a UDF with SQLite */
   266    266   int sqlite3Fts5IndexInit(sqlite3*);
   267    267   
   268    268   void sqlite3Fts5IndexPgsz(Fts5Index *p, int pgsz);
   269    269   
          270  +int sqlite3Fts5IndexGetAverages(Fts5Index *p, Fts5Buffer *pBuf);
          271  +int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8*, int);
          272  +
   270    273   /*
   271    274   ** End of interface to code in fts5_index.c.
   272    275   **************************************************************************/
   273    276   
   274    277   /**************************************************************************
   275    278   ** Interface to code in fts5_storage.c. fts5_storage.c contains contains 
   276    279   ** code to access the data stored in the %_content and %_docsize tables.
................................................................................
   293    296   
   294    297   int sqlite3Fts5StorageIntegrity(Fts5Storage *p);
   295    298   
   296    299   int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt **);
   297    300   void sqlite3Fts5StorageStmtRelease(Fts5Storage *p, int eStmt, sqlite3_stmt*);
   298    301   
   299    302   int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol);
   300         -int sqlite3Fts5StorageAvgsize(Fts5Storage *p, int *aCol);
          303  +int sqlite3Fts5StorageAvgsize(Fts5Storage *p, int iCol, int *pnAvg);
   301    304   
   302    305   
   303    306   /*
   304    307   ** End of interface to code in fts5_storage.c.
   305    308   **************************************************************************/
   306    309   
   307    310   

Changes to ext/fts5/fts5_aux.c.

    39     39     int i;
    40     40   
    41     41     if( nVal>=1 ){
    42     42       zReq = (const char*)sqlite3_value_text(apVal[0]);
    43     43     }
    44     44   
    45     45     memset(&s, 0, sizeof(Fts5Buffer));
           46  +  nCol = pApi->xColumnCount(pFts);
           47  +
           48  +  if( zReq==0 ){
           49  +    sqlite3Fts5BufferAppendPrintf(&rc, &s, "columnavgsize ");
           50  +  }
           51  +  if( 0==zReq || 0==sqlite3_stricmp(zReq, "columnavgsize") ){
           52  +    if( zReq==0 && nCol>1 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, "{");
           53  +    for(i=0; rc==SQLITE_OK && i<nCol; i++){
           54  +      int colsz = 0;
           55  +      rc = pApi->xColumnAvgSize(pFts, i, &colsz);
           56  +      sqlite3Fts5BufferAppendPrintf(&rc, &s, "%s%d", i==0?"":" ", colsz);
           57  +    }
           58  +    if( zReq==0 && nCol>1 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, "}");
           59  +  }
    46     60   
    47     61     if( zReq==0 ){
    48     62       sqlite3Fts5BufferAppendPrintf(&rc, &s, "columncount ");
    49     63     }
    50         -  nCol = pApi->xColumnCount(pFts);
    51     64     if( 0==zReq || 0==sqlite3_stricmp(zReq, "columncount") ){
    52     65       sqlite3Fts5BufferAppendPrintf(&rc, &s, "%d", nCol);
    53     66     }
    54     67   
    55     68     if( zReq==0 ){
    56     69       sqlite3Fts5BufferAppendPrintf(&rc, &s, "columnsize ");
    57     70     }

Changes to ext/fts5/fts5_index.c.

   730    730   static void fts5DataReference(Fts5Data *pData){
   731    731     pData->nRef++;
   732    732   }
   733    733   
   734    734   /*
   735    735   ** INSERT OR REPLACE a record into the %_data table.
   736    736   */
   737         -static void fts5DataWrite(Fts5Index *p, i64 iRowid, u8 *pData, int nData){
          737  +static void fts5DataWrite(Fts5Index *p, i64 iRowid, const u8 *pData, int nData){
   738    738     if( p->rc!=SQLITE_OK ) return;
   739    739   
   740    740     if( p->pWriter==0 ){
   741    741       int rc;
   742    742       Fts5Config *pConfig = p->pConfig;
   743    743       char *zSql = sqlite3_mprintf(
   744    744           "REPLACE INTO '%q'.%Q(id, block) VALUES(?,?)", pConfig->zDb, p->zDataTbl
................................................................................
  2730   2730       if( rc==SQLITE_OK ){
  2731   2731         memset(&s, 0, sizeof(Fts5Structure));
  2732   2732         for(i=0; i<pConfig->nPrefix+1; i++){
  2733   2733           fts5StructureWrite(p, i, &s);
  2734   2734         }
  2735   2735         rc = p->rc;
  2736   2736       }
         2737  +    sqlite3Fts5IndexSetAverages(p, (const u8*)"", 0);
  2737   2738     }
  2738   2739   
  2739   2740     if( rc ){
  2740   2741       sqlite3Fts5IndexClose(p, 0);
  2741   2742       *pp = 0;
  2742   2743     }
  2743   2744     return rc;
................................................................................
  3614   3615         fts5StructureRelease(pIter->pStruct);
  3615   3616         fts5BufferFree(&pIter->poslist);
  3616   3617       }
  3617   3618       fts5CloseReader(pIter->pIndex);
  3618   3619       sqlite3_free(pIter);
  3619   3620     }
  3620   3621   }
         3622  +
         3623  +/*
         3624  +** Read the "averages" record into the buffer supplied as the second 
         3625  +** argument. Return SQLITE_OK if successful, or an SQLite error code
         3626  +** if an error occurs.
         3627  +*/
         3628  +int sqlite3Fts5IndexGetAverages(Fts5Index *p, Fts5Buffer *pBuf){
         3629  +  fts5DataReadOrBuffer(p, pBuf, FTS5_AVERAGES_ROWID);
         3630  +  return p->rc;
         3631  +}
         3632  +
         3633  +/*
         3634  +** Replace the current "averages" record with the contents of the buffer 
         3635  +** supplied as the second argument.
         3636  +*/
         3637  +int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8 *pData, int nData){
         3638  +  fts5DataWrite(p, FTS5_AVERAGES_ROWID, pData, nData);
         3639  +  return p->rc;
         3640  +}
  3621   3641   

Changes to ext/fts5/fts5_storage.c.

    13     13   */
    14     14   
    15     15   #include "fts5Int.h"
    16     16   
    17     17   struct Fts5Storage {
    18     18     Fts5Config *pConfig;
    19     19     Fts5Index *pIndex;
           20  +  i64 nTotalRow;                  /* Total number of rows in FTS table */
           21  +  i64 *aTotalSize;                /* Total sizes of each column */ 
    20     22     sqlite3_stmt *aStmt[9];
    21     23   };
    22     24   
    23     25   
    24     26   #if FTS5_STMT_SCAN_ASC!=0 
    25     27   # error "FTS5_STMT_SCAN_ASC mismatch" 
    26     28   #endif
................................................................................
   164    166     Fts5Index *pIndex, 
   165    167     int bCreate, 
   166    168     Fts5Storage **pp,
   167    169     char **pzErr                    /* OUT: Error message */
   168    170   ){
   169    171     int rc;
   170    172     Fts5Storage *p;                 /* New object */
          173  +  int nByte;                      /* Bytes of space to allocate */
   171    174   
   172         -  *pp = p = (Fts5Storage*)sqlite3_malloc(sizeof(Fts5Storage));
          175  +  nByte = sizeof(Fts5Storage)               /* Fts5Storage object */
          176  +        + pConfig->nCol * sizeof(i64);      /* Fts5Storage.aTotalSize[] */
          177  +  *pp = p = (Fts5Storage*)sqlite3_malloc(nByte);
   173    178     if( !p ) return SQLITE_NOMEM;
   174    179   
   175         -  memset(p, 0, sizeof(Fts5Storage));
          180  +  memset(p, 0, nByte);
          181  +  p->aTotalSize = (i64*)&p[1];
   176    182     p->pConfig = pConfig;
   177    183     p->pIndex = pIndex;
   178    184   
   179    185     if( bCreate ){
   180    186       int i;
   181    187       char *zDefn = sqlite3_malloc(32 + pConfig->nCol * 10);
   182    188       if( zDefn==0 ){
................................................................................
   281    287         for(iCol=1; iCol<=pConfig->nCol; iCol++){
   282    288           rc = sqlite3Fts5Tokenize(pConfig, 
   283    289               (const char*)sqlite3_column_text(pSeek, iCol),
   284    290               sqlite3_column_bytes(pSeek, iCol),
   285    291               (void*)&ctx,
   286    292               fts5StorageInsertCallback
   287    293           );
          294  +        p->aTotalSize[iCol-1] -= (i64)ctx.szCol;
   288    295         }
          296  +      p->nTotalRow--;
   289    297       }
   290    298       rc2 = sqlite3_reset(pSeek);
   291    299       if( rc==SQLITE_OK ) rc = rc2;
   292    300     }
   293    301   
   294    302     return rc;
   295    303   }
................................................................................
   310    318       sqlite3_bind_int64(pReplace, 1, iRowid);
   311    319       sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC);
   312    320       sqlite3_step(pReplace);
   313    321       rc = sqlite3_reset(pReplace);
   314    322     }
   315    323     return rc;
   316    324   }
          325  +
          326  +/*
          327  +** Load the contents of the "averages" record from disk into the
          328  +** p->nTotalRow and p->aTotalSize[] variables.
          329  +**
          330  +** Return SQLITE_OK if successful, or an SQLite error code if an error
          331  +** occurs.
          332  +*/
          333  +static int fts5StorageLoadTotals(Fts5Storage *p){
          334  +  int nCol = p->pConfig->nCol;
          335  +  Fts5Buffer buf;
          336  +  int rc;
          337  +  memset(&buf, 0, sizeof(buf));
          338  +
          339  +  memset(p->aTotalSize, 0, sizeof(i64) * nCol);
          340  +  p->nTotalRow = 0;
          341  +  rc = sqlite3Fts5IndexGetAverages(p->pIndex, &buf);
          342  +  if( rc==SQLITE_OK && buf.n ){
          343  +    int i = 0;
          344  +    int iCol;
          345  +    i += getVarint(&buf.p[i], (u64*)&p->nTotalRow);
          346  +    for(iCol=0; i<buf.n && iCol<nCol; iCol++){
          347  +      i += getVarint(&buf.p[i], (u64*)&p->aTotalSize[iCol]);
          348  +    }
          349  +  }
          350  +  sqlite3_free(buf.p);
          351  +
          352  +  return rc;
          353  +}
          354  +
          355  +/*
          356  +** Store the current contents of the p->nTotalRow and p->aTotalSize[] 
          357  +** variables in the "averages" record on disk.
          358  +**
          359  +** Return SQLITE_OK if successful, or an SQLite error code if an error
          360  +** occurs.
          361  +*/
          362  +static int fts5StorageSaveTotals(Fts5Storage *p){
          363  +  int nCol = p->pConfig->nCol;
          364  +  int i;
          365  +  Fts5Buffer buf;
          366  +  int rc = SQLITE_OK;
          367  +  memset(&buf, 0, sizeof(buf));
          368  +
          369  +  sqlite3Fts5BufferAppendVarint(&rc, &buf, p->nTotalRow);
          370  +  for(i=0; i<nCol; i++){
          371  +    sqlite3Fts5BufferAppendVarint(&rc, &buf, p->aTotalSize[i]);
          372  +  }
          373  +  if( rc==SQLITE_OK ){
          374  +    rc = sqlite3Fts5IndexSetAverages(p->pIndex, buf.p, buf.n);
          375  +  }
          376  +  sqlite3_free(buf.p);
          377  +
          378  +  return rc;
          379  +}
          380  +
   317    381   
   318    382   /*
   319    383   ** Insert a new row into the FTS table.
   320    384   */
   321    385   int sqlite3Fts5StorageInsert(
   322    386     Fts5Storage *p,                 /* Storage module to write to */
   323    387     sqlite3_value **apVal,          /* Array of values passed to xUpdate() */
................................................................................
   329    393     sqlite3_stmt *pInsert;          /* Statement used to write %_content table */
   330    394     int eStmt;                      /* Type of statement used on %_content */
   331    395     int i;                          /* Counter variable */
   332    396     Fts5InsertCtx ctx;              /* Tokenization callback context object */
   333    397     Fts5Buffer buf;                 /* Buffer used to build up %_docsize blob */
   334    398   
   335    399     memset(&buf, 0, sizeof(Fts5Buffer));
          400  +  rc = fts5StorageLoadTotals(p);
   336    401   
   337    402     /* Insert the new row into the %_content table. */
   338         -  if( eConflict==SQLITE_REPLACE ){
   339         -    eStmt = FTS5_STMT_REPLACE_CONTENT;
   340         -    if( sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){
   341         -      rc = fts5StorageDeleteFromIndex(p, sqlite3_value_int64(apVal[1]));
          403  +  if( rc==SQLITE_OK ){
          404  +    if( eConflict==SQLITE_REPLACE ){
          405  +      eStmt = FTS5_STMT_REPLACE_CONTENT;
          406  +      if( sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){
          407  +        rc = fts5StorageDeleteFromIndex(p, sqlite3_value_int64(apVal[1]));
          408  +      }
          409  +    }else{
          410  +      eStmt = FTS5_STMT_INSERT_CONTENT;
   342    411       }
   343         -  }else{
   344         -    eStmt = FTS5_STMT_INSERT_CONTENT;
   345    412     }
   346    413     if( rc==SQLITE_OK ){
   347    414       rc = fts5StorageGetStmt(p, eStmt, &pInsert);
   348    415     }
   349    416     for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){
   350    417       rc = sqlite3_bind_value(pInsert, i, apVal[i]);
   351    418     }
................................................................................
   363    430       rc = sqlite3Fts5Tokenize(pConfig, 
   364    431           (const char*)sqlite3_value_text(apVal[ctx.iCol+2]),
   365    432           sqlite3_value_bytes(apVal[ctx.iCol+2]),
   366    433           (void*)&ctx,
   367    434           fts5StorageInsertCallback
   368    435       );
   369    436       sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol);
          437  +    p->aTotalSize[ctx.iCol] += (i64)ctx.szCol;
   370    438     }
          439  +  p->nTotalRow++;
   371    440   
   372    441     /* Write the %_docsize record */
   373    442     if( rc==SQLITE_OK ){
   374    443       rc = fts5StorageInsertDocsize(p, *piRowid, &buf);
   375    444     }
   376    445     sqlite3_free(buf.p);
          446  +
          447  +  /* Write the averages record */
          448  +  if( rc==SQLITE_OK ){
          449  +    rc = fts5StorageSaveTotals(p);
          450  +  }
   377    451   
   378    452     return rc;
   379    453   }
   380    454   
   381    455   /*
   382    456   ** Context object used by sqlite3Fts5StorageIntegrity().
   383    457   */
................................................................................
   534    608       if( bCorrupt && rc==SQLITE_OK ){
   535    609         rc = SQLITE_CORRUPT_VTAB;
   536    610       }
   537    611     }
   538    612     return rc;
   539    613   }
   540    614   
   541         -int sqlite3Fts5StorageAvgsize(Fts5Storage *p, int *aCol){
   542         -  return 0;
          615  +int sqlite3Fts5StorageAvgsize(Fts5Storage *p, int iCol, int *pnAvg){
          616  +  int rc = fts5StorageLoadTotals(p);
          617  +  if( rc==SQLITE_OK ){
          618  +    int nAvg = 1;
          619  +    if( p->nTotalRow ){
          620  +      nAvg = (int)((p->aTotalSize[iCol] + (p->nTotalRow/2)) / p->nTotalRow);
          621  +      if( nAvg<1 ) nAvg = 1;
          622  +      *pnAvg = nAvg;
          623  +    }
          624  +  }
          625  +  return rc;
   543    626   }
   544    627   

Changes to test/fts5aa.test.

   125    125     do_execsql_test 5.$i.1 { INSERT INTO t1 VALUES($x, $y) }
   126    126     do_execsql_test 5.$i.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
   127    127     if {[set_test_counter errors]} break
   128    128   }
   129    129   
   130    130   #-------------------------------------------------------------------------
   131    131   #
          132  +breakpoint
   132    133   reset_db
   133    134   do_execsql_test 6.0 {
   134    135     CREATE VIRTUAL TABLE t1 USING fts5(x,y);
   135    136     INSERT INTO t1(t1) VALUES('pgsz=32');
   136    137   }
   137    138   
   138    139   do_execsql_test 6.1 {

Changes to test/fts5ae.test.

   124    124   do_execsql_test 4.1 {
   125    125     SELECT rowid, fts5_test(t4, 'poslist') FROM t4 WHERE t4 MATCH 'a OR b AND c';
   126    126   } {
   127    127     1 {0.5 {} {}} 
   128    128   }
   129    129   
   130    130   #-------------------------------------------------------------------------
   131         -# Test that the xColumnSize() API works.
          131  +# Test that the xColumnSize() and xColumnAvgsize() APIs work.
   132    132   #
   133    133   
   134    134   reset_db
   135    135   do_execsql_test 5.1 {
   136    136     CREATE VIRTUAL TABLE t5 USING fts5(x, y);
   137    137     INSERT INTO t5 VALUES('a b c d', 'e f g h i j');
   138    138     INSERT INTO t5 VALUES('', 'a');
................................................................................
   151    151     SELECT rowid, fts5_test(t5, 'columntext') FROM t5 WHERE t5 MATCH 'a'
   152    152     ORDER BY rowid DESC;
   153    153   } {
   154    154     3 {a {}}
   155    155     2 {{} a}
   156    156     1 {{a b c d} {e f g h i j}}
   157    157   }
          158  +
          159  +do_execsql_test 5.3 {
          160  +  SELECT rowid, fts5_test(t5, 'columnavgsize') FROM t5 WHERE t5 MATCH 'a'
          161  +  ORDER BY rowid DESC;
          162  +} {
          163  +  3 {2 2}
          164  +  2 {2 2}
          165  +  1 {2 2}
          166  +}
          167  +
          168  +do_execsql_test 5.4 {
          169  +  INSERT INTO t5 VALUES('x y z', 'v w x y z');
          170  +  SELECT rowid, fts5_test(t5, 'columnavgsize') FROM t5 WHERE t5 MATCH 'a'
          171  +  ORDER BY rowid DESC;
          172  +} {
          173  +  3 {2 3}
          174  +  2 {2 3}
          175  +  1 {2 3}
          176  +}
          177  +
   158    178   
   159    179   
   160    180   finish_test
   161    181