/ Check-in [33ef2210]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Have fts5 cache the decoded structure of fts5 indexes in memory. Use "PRAGMA data_version" to detect stale caches.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts5-data-version
Files: files | file ages | folders
SHA1: 33ef2210ef19e55c8d460bfe9d3dc146034c8acc
User & Date: dan 2016-03-16 19:48:10
Context
2016-03-16
20:53
Merge the changes on the reusable-pragma branch into this one. check-in: 6c4a17b9 user: dan tags: fts5-data-version
19:48
Have fts5 cache the decoded structure of fts5 indexes in memory. Use "PRAGMA data_version" to detect stale caches. check-in: 33ef2210 user: dan tags: fts5-data-version
01:03
Add the SQLITE_OMIT_CODEC_FROM_TCL compile-time option. check-in: 45f7f0c8 user: drh tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5Int.h.

   476    476   ** this connection since it was created.
   477    477   */
   478    478   int sqlite3Fts5IndexReads(Fts5Index *p);
   479    479   
   480    480   int sqlite3Fts5IndexReinit(Fts5Index *p);
   481    481   int sqlite3Fts5IndexOptimize(Fts5Index *p);
   482    482   int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge);
          483  +int sqlite3Fts5IndexReset(Fts5Index *p);
   483    484   
   484    485   int sqlite3Fts5IndexLoadConfig(Fts5Index *p);
   485    486   
   486    487   /*
   487    488   ** End of interface to code in fts5_index.c.
   488    489   **************************************************************************/
   489    490   
................................................................................
   618    619       Fts5Storage *p, const char*, sqlite3_value*, int
   619    620   );
   620    621   
   621    622   int sqlite3Fts5StorageDeleteAll(Fts5Storage *p);
   622    623   int sqlite3Fts5StorageRebuild(Fts5Storage *p);
   623    624   int sqlite3Fts5StorageOptimize(Fts5Storage *p);
   624    625   int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge);
          626  +int sqlite3Fts5StorageReset(Fts5Storage *p);
   625    627   
   626    628   /*
   627    629   ** End of interface to code in fts5_storage.c.
   628    630   **************************************************************************/
   629    631   
   630    632   
   631    633   /**************************************************************************

Changes to ext/fts5/fts5_index.c.

   300    300     sqlite3_blob *pReader;          /* RO incr-blob open on %_data table */
   301    301     sqlite3_stmt *pWriter;          /* "INSERT ... %_data VALUES(?,?)" */
   302    302     sqlite3_stmt *pDeleter;         /* "DELETE FROM %_data ... id>=? AND id<=?" */
   303    303     sqlite3_stmt *pIdxWriter;       /* "INSERT ... %_idx VALUES(?,?,?,?)" */
   304    304     sqlite3_stmt *pIdxDeleter;      /* "DELETE FROM %_idx WHERE segid=? */
   305    305     sqlite3_stmt *pIdxSelect;
   306    306     int nRead;                      /* Total number of blocks read */
          307  +
          308  +  sqlite3_stmt *pDataVersion;
          309  +  i64 iStructVersion;             /* data_version when pStruct read */
          310  +  Fts5Structure *pStruct;         /* Current db structure (or NULL) */
   307    311   };
   308    312   
   309    313   struct Fts5DoclistIter {
   310    314     u8 *aEof;                       /* Pointer to 1 byte past end of doclist */
   311    315   
   312    316     /* Output variables. aPoslist==0 at EOF */
   313    317     i64 iRowid;
................................................................................
   954    958         }
   955    959         pLvl->aSeg = aNew;
   956    960       }else{
   957    961         *pRc = SQLITE_NOMEM;
   958    962       }
   959    963     }
   960    964   }
          965  +
          966  +static Fts5Structure *fts5StructureReadUncached(Fts5Index *p){
          967  +  Fts5Structure *pRet = 0;
          968  +  Fts5Config *pConfig = p->pConfig;
          969  +  int iCookie;                    /* Configuration cookie */
          970  +  Fts5Data *pData;
          971  +
          972  +  pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID);
          973  +  if( p->rc==SQLITE_OK ){
          974  +    /* TODO: Do we need this if the leaf-index is appended? Probably... */
          975  +    memset(&pData->p[pData->nn], 0, FTS5_DATA_PADDING);
          976  +    p->rc = fts5StructureDecode(pData->p, pData->nn, &iCookie, &pRet);
          977  +    if( p->rc==SQLITE_OK && pConfig->iCookie!=iCookie ){
          978  +      p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie);
          979  +    }
          980  +    fts5DataRelease(pData);
          981  +    if( p->rc!=SQLITE_OK ){
          982  +      fts5StructureRelease(pRet);
          983  +      pRet = 0;
          984  +    }
          985  +  }
          986  +
          987  +  return pRet;
          988  +}
          989  +
          990  +static i64 fts5IndexDataVersion(Fts5Index *p){
          991  +  i64 iVersion = 0;
          992  +
          993  +  if( p->pDataVersion==0 ){
          994  +    p->rc = fts5IndexPrepareStmt(p, &p->pDataVersion, 
          995  +        sqlite3_mprintf("PRAGMA %Q.data_version", p->pConfig->zDb)
          996  +    );
          997  +    if( p->rc ) return 0;
          998  +  }
          999  +
         1000  +  if( SQLITE_ROW==sqlite3_step(p->pDataVersion) ){
         1001  +    iVersion = sqlite3_column_int64(p->pDataVersion, 0);
         1002  +  }
         1003  +  p->rc = sqlite3_reset(p->pDataVersion);
         1004  +
         1005  +  return iVersion;
         1006  +}
   961   1007   
   962   1008   /*
   963   1009   ** Read, deserialize and return the structure record.
   964   1010   **
   965   1011   ** The Fts5Structure.aLevel[] and each Fts5StructureLevel.aSeg[] array
   966   1012   ** are over-allocated as described for function fts5StructureDecode() 
   967   1013   ** above.
   968   1014   **
   969   1015   ** If an error occurs, NULL is returned and an error code left in the
   970   1016   ** Fts5Index handle. If an error has already occurred when this function
   971   1017   ** is called, it is a no-op.
   972   1018   */
   973   1019   static Fts5Structure *fts5StructureRead(Fts5Index *p){
   974         -  Fts5Config *pConfig = p->pConfig;
   975         -  Fts5Structure *pRet = 0;        /* Object to return */
   976         -  int iCookie;                    /* Configuration cookie */
   977         -  Fts5Data *pData;
   978         -
   979         -  pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID);
   980         -  if( p->rc ) return 0;
   981         -  /* TODO: Do we need this if the leaf-index is appended? Probably... */
   982         -  memset(&pData->p[pData->nn], 0, FTS5_DATA_PADDING);
   983         -  p->rc = fts5StructureDecode(pData->p, pData->nn, &iCookie, &pRet);
   984         -  if( p->rc==SQLITE_OK && pConfig->iCookie!=iCookie ){
   985         -    p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie);
   986         -  }
   987         -
   988         -  fts5DataRelease(pData);
   989         -  if( p->rc!=SQLITE_OK ){
   990         -    fts5StructureRelease(pRet);
   991         -    pRet = 0;
         1020  +  Fts5Structure *pRet;            /* Object to return */
         1021  +
         1022  +  if( p->pStruct ){
         1023  +    pRet = p->pStruct;
         1024  +#ifdef SQLITE_DEBUG
         1025  +    {
         1026  +      Fts5Structure *pTest = fts5StructureReadUncached(p);
         1027  +      if( pTest ){
         1028  +        int i, j;
         1029  +        assert_nc( pRet->nSegment==pTest->nSegment );
         1030  +        assert_nc( pRet->nLevel==pTest->nLevel );
         1031  +        for(i=0; i<pTest->nLevel; i++){
         1032  +          assert_nc( pRet->aLevel[i].nMerge==pTest->aLevel[i].nMerge );
         1033  +          assert_nc( pRet->aLevel[i].nSeg==pTest->aLevel[i].nSeg );
         1034  +          for(j=0; j<pTest->aLevel[i].nSeg; j++){
         1035  +            Fts5StructureSegment *p1 = &pTest->aLevel[i].aSeg[j];
         1036  +            Fts5StructureSegment *p2 = &pRet->aLevel[i].aSeg[j];
         1037  +            assert_nc( p1->iSegid==p2->iSegid );
         1038  +            assert_nc( p1->pgnoFirst==p2->pgnoFirst );
         1039  +            assert_nc( p1->pgnoLast==p2->pgnoLast );
         1040  +          }
         1041  +        }
         1042  +        fts5StructureRelease(pTest);
         1043  +      }
         1044  +    }
         1045  +#endif
         1046  +  }else{
         1047  +    pRet = fts5StructureReadUncached(p);
         1048  +  }
         1049  +
         1050  +  if( pRet ){
         1051  +    fts5StructureRef(pRet);
         1052  +    p->pStruct = pRet;
         1053  +    p->iStructVersion = fts5IndexDataVersion(p);
   992   1054     }
   993   1055     return pRet;
   994   1056   }
         1057  +
         1058  +static void fts5StructureInvalidate(Fts5Index *p){
         1059  +  if( p->pStruct ){
         1060  +    fts5StructureRelease(p->pStruct);
         1061  +    p->pStruct = 0;
         1062  +  }
         1063  +}
   995   1064   
   996   1065   /*
   997   1066   ** Return the total number of segments in index structure pStruct. This
   998   1067   ** function is only ever used as part of assert() conditions.
   999   1068   */
  1000   1069   #ifdef SQLITE_DEBUG
  1001   1070   static int fts5StructureCountSegments(Fts5Structure *pStruct){
................................................................................
  4324   4393     int iSegid;
  4325   4394     int pgnoLast = 0;                 /* Last leaf page number in segment */
  4326   4395   
  4327   4396     /* Obtain a reference to the index structure and allocate a new segment-id
  4328   4397     ** for the new level-0 segment.  */
  4329   4398     pStruct = fts5StructureRead(p);
  4330   4399     iSegid = fts5AllocateSegid(p, pStruct);
         4400  +  fts5StructureInvalidate(p);
  4331   4401   
  4332   4402     if( iSegid ){
  4333   4403       const int pgsz = p->pConfig->pgsz;
  4334   4404       int eDetail = p->pConfig->eDetail;
  4335   4405       Fts5StructureSegment *pSeg;   /* New segment within pStruct */
  4336   4406       Fts5Buffer *pBuf;             /* Buffer in which to assemble leaf page */
  4337   4407       Fts5Buffer *pPgidx;           /* Buffer in which to assemble pgidx */
................................................................................
  4543   4613   int sqlite3Fts5IndexOptimize(Fts5Index *p){
  4544   4614     Fts5Structure *pStruct;
  4545   4615     Fts5Structure *pNew = 0;
  4546   4616   
  4547   4617     assert( p->rc==SQLITE_OK );
  4548   4618     fts5IndexFlush(p);
  4549   4619     pStruct = fts5StructureRead(p);
         4620  +  fts5StructureInvalidate(p);
  4550   4621   
  4551   4622     if( pStruct ){
  4552   4623       pNew = fts5IndexOptimizeStruct(p, pStruct);
  4553   4624     }
  4554   4625     fts5StructureRelease(pStruct);
  4555   4626   
  4556   4627     assert( pNew==0 || pNew->nSegment>0 );
................................................................................
  4573   4644   ** This is called to implement the special "VALUES('merge', $nMerge)"
  4574   4645   ** INSERT command.
  4575   4646   */
  4576   4647   int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){
  4577   4648     Fts5Structure *pStruct = fts5StructureRead(p);
  4578   4649     if( pStruct ){
  4579   4650       int nMin = p->pConfig->nUsermerge;
         4651  +    fts5StructureInvalidate(p);
  4580   4652       if( nMerge<0 ){
  4581   4653         Fts5Structure *pNew = fts5IndexOptimizeStruct(p, pStruct);
  4582   4654         fts5StructureRelease(pStruct);
  4583   4655         pStruct = pNew;
  4584   4656         nMin = 2;
  4585   4657         nMerge = nMerge*-1;
  4586   4658       }
................................................................................
  5000   5072   ** to the database. Additionally, assume that the contents of the %_data
  5001   5073   ** table may have changed on disk. So any in-memory caches of %_data 
  5002   5074   ** records must be invalidated.
  5003   5075   */
  5004   5076   int sqlite3Fts5IndexRollback(Fts5Index *p){
  5005   5077     fts5CloseReader(p);
  5006   5078     fts5IndexDiscardData(p);
         5079  +  fts5StructureInvalidate(p);
  5007   5080     /* assert( p->rc==SQLITE_OK ); */
  5008   5081     return SQLITE_OK;
  5009   5082   }
  5010   5083   
  5011   5084   /*
  5012   5085   ** The %_data table is completely empty when this function is called. This
  5013   5086   ** function populates it with the initial structure objects for each index,
  5014   5087   ** and the initial version of the "averages" record (a zero-byte blob).
  5015   5088   */
  5016   5089   int sqlite3Fts5IndexReinit(Fts5Index *p){
  5017   5090     Fts5Structure s;
         5091  +  fts5StructureInvalidate(p);
  5018   5092     memset(&s, 0, sizeof(Fts5Structure));
  5019   5093     fts5DataWrite(p, FTS5_AVERAGES_ROWID, (const u8*)"", 0);
  5020   5094     fts5StructureWrite(p, &s);
  5021   5095     return fts5IndexReturn(p);
  5022   5096   }
  5023   5097   
  5024   5098   /*
................................................................................
  5069   5143   /*
  5070   5144   ** Close a handle opened by an earlier call to sqlite3Fts5IndexOpen().
  5071   5145   */
  5072   5146   int sqlite3Fts5IndexClose(Fts5Index *p){
  5073   5147     int rc = SQLITE_OK;
  5074   5148     if( p ){
  5075   5149       assert( p->pReader==0 );
         5150  +    fts5StructureInvalidate(p);
  5076   5151       sqlite3_finalize(p->pWriter);
  5077   5152       sqlite3_finalize(p->pDeleter);
  5078   5153       sqlite3_finalize(p->pIdxWriter);
  5079   5154       sqlite3_finalize(p->pIdxDeleter);
  5080   5155       sqlite3_finalize(p->pIdxSelect);
         5156  +    sqlite3_finalize(p->pDataVersion);
  5081   5157       sqlite3Fts5HashFree(p->pHash);
  5082   5158       sqlite3_free(p->zDataTbl);
  5083   5159       sqlite3_free(p);
  5084   5160     }
  5085   5161     return rc;
  5086   5162   }
  5087   5163   
................................................................................
  6329   6405     if( rc==SQLITE_OK ){
  6330   6406       rc = sqlite3_create_function(
  6331   6407           db, "fts5_rowid", -1, SQLITE_UTF8, 0, fts5RowidFunction, 0, 0
  6332   6408       );
  6333   6409     }
  6334   6410     return rc;
  6335   6411   }
         6412  +
         6413  +
         6414  +int sqlite3Fts5IndexReset(Fts5Index *p){
         6415  +  assert( p->pStruct==0 || p->iStructVersion!=0 );
         6416  +  if( fts5IndexDataVersion(p)!=p->iStructVersion ){
         6417  +    fts5StructureInvalidate(p);
         6418  +  }
         6419  +  return fts5IndexReturn(p);
         6420  +}

Changes to ext/fts5/fts5_main.c.

   592    592         pInfo->aConstraintUsage[pC->iConsIndex].omit = (unsigned char)pC->omit;
   593    593       }
   594    594     }
   595    595   
   596    596     pInfo->idxNum = idxFlags;
   597    597     return SQLITE_OK;
   598    598   }
          599  +
          600  +static int fts5NewTransaction(Fts5Table *pTab){
          601  +  Fts5Cursor *pCsr;
          602  +  for(pCsr=pTab->pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){
          603  +    if( pCsr->base.pVtab==(sqlite3_vtab*)pTab ) return SQLITE_OK;
          604  +  }
          605  +  return sqlite3Fts5StorageReset(pTab->pStorage);
          606  +}
   599    607   
   600    608   /*
   601    609   ** Implementation of xOpen method.
   602    610   */
   603    611   static int fts5OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){
   604    612     Fts5Table *pTab = (Fts5Table*)pVTab;
   605    613     Fts5Config *pConfig = pTab->pConfig;
   606         -  Fts5Cursor *pCsr;               /* New cursor object */
          614  +  Fts5Cursor *pCsr = 0;           /* New cursor object */
   607    615     int nByte;                      /* Bytes of space to allocate */
   608         -  int rc = SQLITE_OK;             /* Return code */
          616  +  int rc;                         /* Return code */
   609    617   
   610         -  nByte = sizeof(Fts5Cursor) + pConfig->nCol * sizeof(int);
   611         -  pCsr = (Fts5Cursor*)sqlite3_malloc(nByte);
   612         -  if( pCsr ){
   613         -    Fts5Global *pGlobal = pTab->pGlobal;
   614         -    memset(pCsr, 0, nByte);
   615         -    pCsr->aColumnSize = (int*)&pCsr[1];
   616         -    pCsr->pNext = pGlobal->pCsr;
   617         -    pGlobal->pCsr = pCsr;
   618         -    pCsr->iCsrId = ++pGlobal->iNextId;
   619         -  }else{
   620         -    rc = SQLITE_NOMEM;
          618  +  rc = fts5NewTransaction(pTab);
          619  +  if( rc==SQLITE_OK ){
          620  +    nByte = sizeof(Fts5Cursor) + pConfig->nCol * sizeof(int);
          621  +    pCsr = (Fts5Cursor*)sqlite3_malloc(nByte);
          622  +    if( pCsr ){
          623  +      Fts5Global *pGlobal = pTab->pGlobal;
          624  +      memset(pCsr, 0, nByte);
          625  +      pCsr->aColumnSize = (int*)&pCsr[1];
          626  +      pCsr->pNext = pGlobal->pCsr;
          627  +      pGlobal->pCsr = pCsr;
          628  +      pCsr->iCsrId = ++pGlobal->iNextId;
          629  +    }else{
          630  +      rc = SQLITE_NOMEM;
          631  +    }
   621    632     }
   622    633     *ppCsr = (sqlite3_vtab_cursor*)pCsr;
   623    634     return rc;
   624    635   }
   625    636   
   626    637   static int fts5StmtType(Fts5Cursor *pCsr){
   627    638     if( pCsr->ePlan==FTS5_PLAN_SCAN ){
................................................................................
  1574   1585     return rc;
  1575   1586   }
  1576   1587   
  1577   1588   /*
  1578   1589   ** Implementation of xBegin() method. 
  1579   1590   */
  1580   1591   static int fts5BeginMethod(sqlite3_vtab *pVtab){
  1581         -  UNUSED_PARAM(pVtab);  /* Call below is a no-op for NDEBUG builds */
  1582   1592     fts5CheckTransactionState((Fts5Table*)pVtab, FTS5_BEGIN, 0);
         1593  +  fts5NewTransaction((Fts5Table*)pVtab);
  1583   1594     return SQLITE_OK;
  1584   1595   }
  1585   1596   
  1586   1597   /*
  1587   1598   ** Implementation of xCommit() method. This is a no-op. The contents of
  1588   1599   ** the pending-terms hash-table have already been flushed into the database
  1589   1600   ** by fts5SyncMethod().

Changes to ext/fts5/fts5_storage.c.

   634    634   int sqlite3Fts5StorageOptimize(Fts5Storage *p){
   635    635     return sqlite3Fts5IndexOptimize(p->pIndex);
   636    636   }
   637    637   
   638    638   int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge){
   639    639     return sqlite3Fts5IndexMerge(p->pIndex, nMerge);
   640    640   }
          641  +
          642  +int sqlite3Fts5StorageReset(Fts5Storage *p){
          643  +  return sqlite3Fts5IndexReset(p->pIndex);
          644  +}
   641    645   
   642    646   /*
   643    647   ** Allocate a new rowid. This is used for "external content" tables when
   644    648   ** a NULL value is inserted into the rowid column. The new rowid is allocated
   645    649   ** by inserting a dummy row into the %_docsize table. The dummy will be
   646    650   ** overwritten later.
   647    651   **

Changes to ext/fts5/test/fts5corrupt3.test.

   175    175   for {set i 1} {1} {incr i} {
   176    176     set struct [db one {SELECT block FROM t1_data WHERE id=10}]
   177    177     binary scan $struct c* var
   178    178     set end [lindex $var end]
   179    179     if {$end<=$i} break
   180    180     lset var end [expr $end - $i]
   181    181     set struct [binary format c* $var]
          182  +
          183  +  db close
          184  +  sqlite3 db test.db
          185  +
   182    186     db eval {
   183    187       BEGIN;
   184    188       UPDATE t1_data SET block = $struct WHERE id=10;
   185    189     }
   186    190     do_test 4.1.$i {
   187    191       incr nErr [catch { db eval { SELECT rowid FROM t1 WHERE t1 MATCH 'x*' } }]
   188    192       set {} {}

Changes to ext/fts5/test/fts5dlidx.test.

   174    174   
   175    175   do_execsql_test 3.2 {
   176    176     SELECT rowid FROM abc WHERE abc 
   177    177     MATCH 'IteratorpItercurrentlypointstothefirstrowidofadoclist' 
   178    178     ORDER BY rowid DESC;
   179    179   } {16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1}
   180    180   
   181         -do_execsql_test 3.2 {
          181  +do_execsql_test 3.3 {
   182    182     INSERT INTO abc(abc) VALUES('integrity-check');
   183    183     INSERT INTO abc(abc) VALUES('optimize');
   184    184     INSERT INTO abc(abc) VALUES('integrity-check');
   185    185   }
   186    186   
   187    187   set v [lindex $vocab 0]
   188    188   set i 0
   189    189   foreach v $vocab {
   190         -  do_execsql_test 3.3.[incr i] {
          190  +  do_execsql_test 3.4.[incr i] {
   191    191       SELECT rowid FROM abc WHERE abc MATCH $v
   192    192     } {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16}
   193    193   }
   194    194   
   195    195   } ;# foreach_detail_mode
   196    196   
   197    197   
   198    198   
   199    199   finish_test
   200    200