/ Check-in [05dfdad4]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Cache the value of the "totals" record in memory during transactions.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts5
Files: files | file ages | folders
SHA1:05dfdad445b22f375b71abe0b1fa1bf7ca331be7
User & Date: dan 2014-08-12 16:07:35
Context
2014-08-18
19:30
Add an "automerge=0" mode that disables auto-merging and falls back to fts4-style crisis merges. check-in: 2397404e user: dan tags: fts5
2014-08-12
16:07
Cache the value of the "totals" record in memory during transactions. check-in: 05dfdad4 user: dan tags: fts5
08:36
Automatically resize the hash table used by fts5. check-in: f1cb48f4 user: dan tags: fts5
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5.c.

930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
...
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
....
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
/*
** Implementation of xSync() method. 
*/
static int fts5SyncMethod(sqlite3_vtab *pVtab){
  int rc;
  Fts5Table *pTab = (Fts5Table*)pVtab;
  fts5CheckTransactionState(pTab, FTS5_SYNC, 0);
  rc = sqlite3Fts5IndexSync(pTab->pIndex, 1);
  return rc;
}

/*
** Implementation of xBegin() method. 
*/
static int fts5BeginMethod(sqlite3_vtab *pVtab){
................................................................................
** Implementation of xRollback(). Discard the contents of the pending-terms
** hash-table. Any changes made to the database are reverted by SQLite.
*/
static int fts5RollbackMethod(sqlite3_vtab *pVtab){
  int rc;
  Fts5Table *pTab = (Fts5Table*)pVtab;
  fts5CheckTransactionState(pTab, FTS5_ROLLBACK, 0);
  rc = sqlite3Fts5IndexRollback(pTab->pIndex);
  return rc;
}

static void *fts5ApiUserData(Fts5Context *pCtx){
  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  return pCsr->pAux->pUserData;
}
................................................................................
** The xSavepoint() method.
**
** Flush the contents of the pending-terms table to disk.
*/
static int fts5SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
  Fts5Table *pTab = (Fts5Table*)pVtab;
  fts5CheckTransactionState(pTab, FTS5_SAVEPOINT, iSavepoint);
  return sqlite3Fts5IndexSync(pTab->pIndex, 0);
}

/*
** The xRelease() method.
**
** This is a no-op.
*/
static int fts5ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){
  Fts5Table *pTab = (Fts5Table*)pVtab;
  fts5CheckTransactionState(pTab, FTS5_RELEASE, iSavepoint);
  return sqlite3Fts5IndexSync(pTab->pIndex, 0);
}

/*
** The xRollbackTo() method.
**
** Discard the contents of the pending terms table.
*/
static int fts5RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){
  Fts5Table *pTab = (Fts5Table*)pVtab;
  fts5CheckTransactionState(pTab, FTS5_ROLLBACKTO, iSavepoint);
  return sqlite3Fts5IndexRollback(pTab->pIndex);
}

/*
** Register a new auxiliary function with global context pGlobal.
*/
int sqlite3Fts5CreateAux(
  Fts5Global *pGlobal,            /* Global context (one per db handle) */







|







 







|







 







|










|










|







930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
...
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
....
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
/*
** Implementation of xSync() method. 
*/
static int fts5SyncMethod(sqlite3_vtab *pVtab){
  int rc;
  Fts5Table *pTab = (Fts5Table*)pVtab;
  fts5CheckTransactionState(pTab, FTS5_SYNC, 0);
  rc = sqlite3Fts5StorageSync(pTab->pStorage, 1);
  return rc;
}

/*
** Implementation of xBegin() method. 
*/
static int fts5BeginMethod(sqlite3_vtab *pVtab){
................................................................................
** Implementation of xRollback(). Discard the contents of the pending-terms
** hash-table. Any changes made to the database are reverted by SQLite.
*/
static int fts5RollbackMethod(sqlite3_vtab *pVtab){
  int rc;
  Fts5Table *pTab = (Fts5Table*)pVtab;
  fts5CheckTransactionState(pTab, FTS5_ROLLBACK, 0);
  rc = sqlite3Fts5StorageRollback(pTab->pStorage);
  return rc;
}

static void *fts5ApiUserData(Fts5Context *pCtx){
  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  return pCsr->pAux->pUserData;
}
................................................................................
** The xSavepoint() method.
**
** Flush the contents of the pending-terms table to disk.
*/
static int fts5SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
  Fts5Table *pTab = (Fts5Table*)pVtab;
  fts5CheckTransactionState(pTab, FTS5_SAVEPOINT, iSavepoint);
  return sqlite3Fts5StorageSync(pTab->pStorage, 0);
}

/*
** The xRelease() method.
**
** This is a no-op.
*/
static int fts5ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){
  Fts5Table *pTab = (Fts5Table*)pVtab;
  fts5CheckTransactionState(pTab, FTS5_RELEASE, iSavepoint);
  return sqlite3Fts5StorageSync(pTab->pStorage, 0);
}

/*
** The xRollbackTo() method.
**
** Discard the contents of the pending terms table.
*/
static int fts5RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){
  Fts5Table *pTab = (Fts5Table*)pVtab;
  fts5CheckTransactionState(pTab, FTS5_ROLLBACKTO, iSavepoint);
  return sqlite3Fts5StorageRollback(pTab->pStorage);
}

/*
** Register a new auxiliary function with global context pGlobal.
*/
int sqlite3Fts5CreateAux(
  Fts5Global *pGlobal,            /* Global context (one per db handle) */

Changes to ext/fts5/fts5Int.h.

349
350
351
352
353
354
355


356
357
358
359
360
361
362
int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt **);
void sqlite3Fts5StorageStmtRelease(Fts5Storage *p, int eStmt, sqlite3_stmt*);

int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol);
int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnAvg);
int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow);




/*
** End of interface to code in fts5_storage.c.
**************************************************************************/


/**************************************************************************







>
>







349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt **);
void sqlite3Fts5StorageStmtRelease(Fts5Storage *p, int eStmt, sqlite3_stmt*);

int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol);
int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnAvg);
int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow);

int sqlite3Fts5StorageSync(Fts5Storage *p, int bCommit);
int sqlite3Fts5StorageRollback(Fts5Storage *p);

/*
** End of interface to code in fts5_storage.c.
**************************************************************************/


/**************************************************************************

Changes to ext/fts5/fts5_storage.c.

13
14
15
16
17
18
19

20
21
22
23
24
25
26
...
313
314
315
316
317
318
319
320
321



322
323
324
325
326


327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343

344
345
346
347
348
349
350
351
...
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
...
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
...
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
...
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723

















*/

#include "fts5Int.h"

struct Fts5Storage {
  Fts5Config *pConfig;
  Fts5Index *pIndex;

  i64 nTotalRow;                  /* Total number of rows in FTS table */
  i64 *aTotalSize;                /* Total sizes of each column */ 
  sqlite3_stmt *aStmt[9];
};


#if FTS5_STMT_SCAN_ASC!=0 
................................................................................
    sqlite3_step(pReplace);
    rc = sqlite3_reset(pReplace);
  }
  return rc;
}

/*
** Load the contents of the "averages" record from disk into the
** p->nTotalRow and p->aTotalSize[] variables.



**
** Return SQLITE_OK if successful, or an SQLite error code if an error
** occurs.
*/
static int fts5StorageLoadTotals(Fts5Storage *p){


  int nCol = p->pConfig->nCol;
  Fts5Buffer buf;
  int rc;
  memset(&buf, 0, sizeof(buf));

  memset(p->aTotalSize, 0, sizeof(i64) * nCol);
  p->nTotalRow = 0;
  rc = sqlite3Fts5IndexGetAverages(p->pIndex, &buf);
  if( rc==SQLITE_OK && buf.n ){
    int i = 0;
    int iCol;
    i += getVarint(&buf.p[i], (u64*)&p->nTotalRow);
    for(iCol=0; i<buf.n && iCol<nCol; iCol++){
      i += getVarint(&buf.p[i], (u64*)&p->aTotalSize[iCol]);
    }
  }
  sqlite3_free(buf.p);


  return rc;
}

/*
** Store the current contents of the p->nTotalRow and p->aTotalSize[] 
** variables in the "averages" record on disk.
**
................................................................................
/*
** Remove a row from the FTS table.
*/
int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel){
  int rc;
  sqlite3_stmt *pDel;

  rc = fts5StorageLoadTotals(p);

  /* Delete the index records */
  if( rc==SQLITE_OK ){
    rc = fts5StorageDeleteFromIndex(p, iDel);
  }

  /* Delete the %_docsize record */
................................................................................
  sqlite3_value **apVal,          /* Array of values passed to xUpdate() */
  int eConflict,                  /* on conflict clause */
  i64 *piRowid                    /* OUT: rowid of new record */
){
  Fts5Config *pConfig = p->pConfig;
  int rc = SQLITE_OK;             /* Return code */
  sqlite3_stmt *pInsert;          /* Statement used to write %_content table */
  int eStmt;                      /* Type of statement used on %_content */
  int i;                          /* Counter variable */
  Fts5InsertCtx ctx;              /* Tokenization callback context object */
  Fts5Buffer buf;                 /* Buffer used to build up %_docsize blob */

  memset(&buf, 0, sizeof(Fts5Buffer));
  rc = fts5StorageLoadTotals(p);

  /* Insert the new row into the %_content table. */
  if( rc==SQLITE_OK ){
    if( eConflict==SQLITE_REPLACE ){
      eStmt = FTS5_STMT_REPLACE_CONTENT;
      if( sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){
        rc = fts5StorageDeleteFromIndex(p, sqlite3_value_int64(apVal[1]));
................................................................................
    rc2 = sqlite3_reset(pScan);
    if( rc==SQLITE_OK ) rc = rc2;
  }

  /* Test that the "totals" (sometimes called "averages") record looks Ok */
  if( rc==SQLITE_OK ){
    int i;
    rc = fts5StorageLoadTotals(p);
    for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){
      if( p->aTotalSize[i]!=aTotalSize[i] ) rc = SQLITE_CORRUPT_VTAB;
    }
  }

  /* Check that the %_docsize and %_content tables contain the expected
  ** number of rows.  */
................................................................................
      rc = SQLITE_CORRUPT_VTAB;
    }
  }
  return rc;
}

int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnToken){
  int rc = fts5StorageLoadTotals(p);
  if( rc==SQLITE_OK ){
    *pnToken = p->aTotalSize[iCol];
  }
  return rc;
}

int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow){
  int rc = fts5StorageLoadTotals(p);
  if( rc==SQLITE_OK ){
    *pnRow = p->nTotalRow;
  }
  return rc;
}

























>







 







|
|
>
>
>




|
>
>
|
|
<
|

|
|
|
|
|
|
|
|
|
|
|
|
>
|







 







|







 







|





|







 







|







 







|







|






>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
...
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334

335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
...
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
...
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
...
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
...
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
*/

#include "fts5Int.h"

struct Fts5Storage {
  Fts5Config *pConfig;
  Fts5Index *pIndex;
  int bTotalsValid;               /* True if nTotalRow/aTotalSize[] are valid */
  i64 nTotalRow;                  /* Total number of rows in FTS table */
  i64 *aTotalSize;                /* Total sizes of each column */ 
  sqlite3_stmt *aStmt[9];
};


#if FTS5_STMT_SCAN_ASC!=0 
................................................................................
    sqlite3_step(pReplace);
    rc = sqlite3_reset(pReplace);
  }
  return rc;
}

/*
** Load the contents of the "averages" record from disk into the 
** p->nTotalRow and p->aTotalSize[] variables. If successful, and if
** argument bCache is true, set the p->bTotalsValid flag to indicate
** that the contents of aTotalSize[] and nTotalRow are valid until
** further notice.
**
** Return SQLITE_OK if successful, or an SQLite error code if an error
** occurs.
*/
static int fts5StorageLoadTotals(Fts5Storage *p, int bCache){
  int rc = SQLITE_OK;
  if( p->bTotalsValid==0 ){
    int nCol = p->pConfig->nCol;
    Fts5Buffer buf;

    memset(&buf, 0, sizeof(buf));

    memset(p->aTotalSize, 0, sizeof(i64) * nCol);
    p->nTotalRow = 0;
    rc = sqlite3Fts5IndexGetAverages(p->pIndex, &buf);
    if( rc==SQLITE_OK && buf.n ){
      int i = 0;
      int iCol;
      i += getVarint(&buf.p[i], (u64*)&p->nTotalRow);
      for(iCol=0; i<buf.n && iCol<nCol; iCol++){
        i += getVarint(&buf.p[i], (u64*)&p->aTotalSize[iCol]);
      }
    }
    sqlite3_free(buf.p);
    p->bTotalsValid = bCache;
  }
  return rc;
}

/*
** Store the current contents of the p->nTotalRow and p->aTotalSize[] 
** variables in the "averages" record on disk.
**
................................................................................
/*
** Remove a row from the FTS table.
*/
int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel){
  int rc;
  sqlite3_stmt *pDel;

  rc = fts5StorageLoadTotals(p, 1);

  /* Delete the index records */
  if( rc==SQLITE_OK ){
    rc = fts5StorageDeleteFromIndex(p, iDel);
  }

  /* Delete the %_docsize record */
................................................................................
  sqlite3_value **apVal,          /* Array of values passed to xUpdate() */
  int eConflict,                  /* on conflict clause */
  i64 *piRowid                    /* OUT: rowid of new record */
){
  Fts5Config *pConfig = p->pConfig;
  int rc = SQLITE_OK;             /* Return code */
  sqlite3_stmt *pInsert;          /* Statement used to write %_content table */
  int eStmt = 0;                  /* Type of statement used on %_content */
  int i;                          /* Counter variable */
  Fts5InsertCtx ctx;              /* Tokenization callback context object */
  Fts5Buffer buf;                 /* Buffer used to build up %_docsize blob */

  memset(&buf, 0, sizeof(Fts5Buffer));
  rc = fts5StorageLoadTotals(p, 1);

  /* Insert the new row into the %_content table. */
  if( rc==SQLITE_OK ){
    if( eConflict==SQLITE_REPLACE ){
      eStmt = FTS5_STMT_REPLACE_CONTENT;
      if( sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){
        rc = fts5StorageDeleteFromIndex(p, sqlite3_value_int64(apVal[1]));
................................................................................
    rc2 = sqlite3_reset(pScan);
    if( rc==SQLITE_OK ) rc = rc2;
  }

  /* Test that the "totals" (sometimes called "averages") record looks Ok */
  if( rc==SQLITE_OK ){
    int i;
    rc = fts5StorageLoadTotals(p, 0);
    for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){
      if( p->aTotalSize[i]!=aTotalSize[i] ) rc = SQLITE_CORRUPT_VTAB;
    }
  }

  /* Check that the %_docsize and %_content tables contain the expected
  ** number of rows.  */
................................................................................
      rc = SQLITE_CORRUPT_VTAB;
    }
  }
  return rc;
}

int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnToken){
  int rc = fts5StorageLoadTotals(p, 0);
  if( rc==SQLITE_OK ){
    *pnToken = p->aTotalSize[iCol];
  }
  return rc;
}

int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow){
  int rc = fts5StorageLoadTotals(p, 0);
  if( rc==SQLITE_OK ){
    *pnRow = p->nTotalRow;
  }
  return rc;
}

/*
** Flush any data currently held in-memory to disk.
*/
int sqlite3Fts5StorageSync(Fts5Storage *p, int bCommit){
  if( bCommit && p->bTotalsValid ){
    int rc = fts5StorageSaveTotals(p);
    p->bTotalsValid = 0;
    if( rc!=SQLITE_OK ) return rc;
  }
  return sqlite3Fts5IndexSync(p->pIndex, bCommit);
}

int sqlite3Fts5StorageRollback(Fts5Storage *p){
  p->bTotalsValid = 0;
  return sqlite3Fts5IndexRollback(p->pIndex);
}