/ Check-in [3b72df40]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Rationalize some code in fts5_storage.c. Add tests to cover recently added branches in fts5.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1:3b72df405ac9b3a71144f45317d32e25f0084c4b
User & Date: dan 2016-01-15 19:54:47
Context
2016-01-15
21:55
Remove an unnecessary memset(). check-in: 689421a9 user: drh tags: trunk
19:54
Rationalize some code in fts5_storage.c. Add tests to cover recently added branches in fts5. check-in: 3b72df40 user: dan tags: trunk
16:11
Remove unnecessary de-initialization of the Parse object. check-in: 75ab30c5 user: drh tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5Int.h.

582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
...
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
int sqlite3Fts5StorageOpen(Fts5Config*, Fts5Index*, int, Fts5Storage**, char**);
int sqlite3Fts5StorageClose(Fts5Storage *p);
int sqlite3Fts5StorageRename(Fts5Storage*, const char *zName);

int sqlite3Fts5DropAll(Fts5Config*);
int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, int, char **);

int sqlite3Fts5StorageDelete(Fts5Storage *p, i64);
int sqlite3Fts5StorageContentInsert(Fts5Storage *p, sqlite3_value**, i64*);
int sqlite3Fts5StorageIndexInsert(Fts5Storage *p, sqlite3_value**, i64);

int sqlite3Fts5StorageIntegrity(Fts5Storage *p);

int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt**, char**);
void sqlite3Fts5StorageStmtRelease(Fts5Storage *p, int eStmt, sqlite3_stmt*);
................................................................................
int sqlite3Fts5StorageSync(Fts5Storage *p, int bCommit);
int sqlite3Fts5StorageRollback(Fts5Storage *p);

int sqlite3Fts5StorageConfigValue(
    Fts5Storage *p, const char*, sqlite3_value*, int
);

int sqlite3Fts5StorageSpecialDelete(Fts5Storage *p, i64 iDel, sqlite3_value**);

int sqlite3Fts5StorageDeleteAll(Fts5Storage *p);
int sqlite3Fts5StorageRebuild(Fts5Storage *p);
int sqlite3Fts5StorageOptimize(Fts5Storage *p);
int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge);

/*
** End of interface to code in fts5_storage.c.







|







 







<
<







582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
...
602
603
604
605
606
607
608


609
610
611
612
613
614
615
int sqlite3Fts5StorageOpen(Fts5Config*, Fts5Index*, int, Fts5Storage**, char**);
int sqlite3Fts5StorageClose(Fts5Storage *p);
int sqlite3Fts5StorageRename(Fts5Storage*, const char *zName);

int sqlite3Fts5DropAll(Fts5Config*);
int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, int, char **);

int sqlite3Fts5StorageDelete(Fts5Storage *p, i64, sqlite3_value**);
int sqlite3Fts5StorageContentInsert(Fts5Storage *p, sqlite3_value**, i64*);
int sqlite3Fts5StorageIndexInsert(Fts5Storage *p, sqlite3_value**, i64);

int sqlite3Fts5StorageIntegrity(Fts5Storage *p);

int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt**, char**);
void sqlite3Fts5StorageStmtRelease(Fts5Storage *p, int eStmt, sqlite3_stmt*);
................................................................................
int sqlite3Fts5StorageSync(Fts5Storage *p, int bCommit);
int sqlite3Fts5StorageRollback(Fts5Storage *p);

int sqlite3Fts5StorageConfigValue(
    Fts5Storage *p, const char*, sqlite3_value*, int
);



int sqlite3Fts5StorageDeleteAll(Fts5Storage *p);
int sqlite3Fts5StorageRebuild(Fts5Storage *p);
int sqlite3Fts5StorageOptimize(Fts5Storage *p);
int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge);

/*
** End of interface to code in fts5_storage.c.

Changes to ext/fts5/fts5_main.c.

1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
....
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
....
1743
1744
1745
1746
1747
1748
1749

1750

1751
1752
1753
1754
1755
1756
1757
  sqlite3_value **apVal, 
  sqlite3_int64 *piRowid
){
  int rc = SQLITE_OK;
  int eType1 = sqlite3_value_type(apVal[1]);
  if( eType1==SQLITE_INTEGER ){
    sqlite3_int64 iDel = sqlite3_value_int64(apVal[1]);
    rc = sqlite3Fts5StorageSpecialDelete(pTab->pStorage, iDel, &apVal[2]);
  }
  return rc;
}

static void fts5StorageInsert(
  int *pRc, 
  Fts5Table *pTab, 
................................................................................
      );
      rc = SQLITE_ERROR;
    }

    /* Case 1: DELETE */
    else if( nArg==1 ){
      i64 iDel = sqlite3_value_int64(apVal[0]);  /* Rowid to delete */
      rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel);
    }

    /* Case 2: INSERT */
    else if( eType0!=SQLITE_INTEGER ){     
      /* If this is a REPLACE, first remove the current entry (if any) */
      if( eConflict==SQLITE_REPLACE 
       && sqlite3_value_type(apVal[1])==SQLITE_INTEGER 
      ){
        i64 iNew = sqlite3_value_int64(apVal[1]);  /* Rowid to delete */
        rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew);
      }
      fts5StorageInsert(&rc, pTab, apVal, pRowid);
    }

    /* Case 2: UPDATE */
    else{
      i64 iOld = sqlite3_value_int64(apVal[0]);  /* Old rowid */
      i64 iNew = sqlite3_value_int64(apVal[1]);  /* New rowid */
      if( iOld!=iNew ){
        if( eConflict==SQLITE_REPLACE ){
          rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld);
          if( rc==SQLITE_OK ){
            rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew);
          }
          fts5StorageInsert(&rc, pTab, apVal, pRowid);
        }else{
          rc = sqlite3Fts5StorageContentInsert(pTab->pStorage, apVal, pRowid);
          if( rc==SQLITE_OK ){
            rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld);
          }
          if( rc==SQLITE_OK ){
            rc = sqlite3Fts5StorageIndexInsert(pTab->pStorage, apVal, *pRowid);
          }
        }
      }else{
        rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld);
        fts5StorageInsert(&rc, pTab, apVal, pRowid);
      }
    }
  }

  pTab->pConfig->pzErrmsg = 0;
  return rc;
................................................................................
    int i;

    /* Initialize all iterators */
    for(i=0; i<nIter && rc==SQLITE_OK; i++){
      const u8 *a;
      int n; 
      rc = fts5CsrPoslist(pCsr, i, &a, &n);

      sqlite3Fts5PoslistReaderInit(a, n, &aIter[i]);

    }

    if( rc==SQLITE_OK ){
      while( 1 ){
        int *aInst;
        int iBest = -1;
        for(i=0; i<nIter; i++){







|







 







|









|










|

|





|






|







 







>
|
>







1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
....
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
....
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
  sqlite3_value **apVal, 
  sqlite3_int64 *piRowid
){
  int rc = SQLITE_OK;
  int eType1 = sqlite3_value_type(apVal[1]);
  if( eType1==SQLITE_INTEGER ){
    sqlite3_int64 iDel = sqlite3_value_int64(apVal[1]);
    rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, &apVal[2]);
  }
  return rc;
}

static void fts5StorageInsert(
  int *pRc, 
  Fts5Table *pTab, 
................................................................................
      );
      rc = SQLITE_ERROR;
    }

    /* Case 1: DELETE */
    else if( nArg==1 ){
      i64 iDel = sqlite3_value_int64(apVal[0]);  /* Rowid to delete */
      rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, 0);
    }

    /* Case 2: INSERT */
    else if( eType0!=SQLITE_INTEGER ){     
      /* If this is a REPLACE, first remove the current entry (if any) */
      if( eConflict==SQLITE_REPLACE 
       && sqlite3_value_type(apVal[1])==SQLITE_INTEGER 
      ){
        i64 iNew = sqlite3_value_int64(apVal[1]);  /* Rowid to delete */
        rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0);
      }
      fts5StorageInsert(&rc, pTab, apVal, pRowid);
    }

    /* Case 2: UPDATE */
    else{
      i64 iOld = sqlite3_value_int64(apVal[0]);  /* Old rowid */
      i64 iNew = sqlite3_value_int64(apVal[1]);  /* New rowid */
      if( iOld!=iNew ){
        if( eConflict==SQLITE_REPLACE ){
          rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0);
          if( rc==SQLITE_OK ){
            rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0);
          }
          fts5StorageInsert(&rc, pTab, apVal, pRowid);
        }else{
          rc = sqlite3Fts5StorageContentInsert(pTab->pStorage, apVal, pRowid);
          if( rc==SQLITE_OK ){
            rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0);
          }
          if( rc==SQLITE_OK ){
            rc = sqlite3Fts5StorageIndexInsert(pTab->pStorage, apVal, *pRowid);
          }
        }
      }else{
        rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0);
        fts5StorageInsert(&rc, pTab, apVal, pRowid);
      }
    }
  }

  pTab->pConfig->pzErrmsg = 0;
  return rc;
................................................................................
    int i;

    /* Initialize all iterators */
    for(i=0; i<nIter && rc==SQLITE_OK; i++){
      const u8 *a;
      int n; 
      rc = fts5CsrPoslist(pCsr, i, &a, &n);
      if( rc==SQLITE_OK ){
        sqlite3Fts5PoslistReaderInit(a, n, &aIter[i]);
      }
    }

    if( rc==SQLITE_OK ){
      while( 1 ){
        int *aInst;
        int iBest = -1;
        for(i=0; i<nIter; i++){

Changes to ext/fts5/fts5_storage.c.

374
375
376
377
378
379
380
381




382
383
384



385

386
387
388
389
390
391
392




393
394
395
396
397
398
399
400



401
402
403






404
405
406
407

408
409
410
411
412
413
414
415
416
417
418
419
420
...
486
487
488
489
490
491
492
493
494
495
496
497

498
499
500
501
502
503
504
505
506
507
508
509
...
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
}

/*
** If a row with rowid iDel is present in the %_content table, add the
** delete-markers to the FTS index necessary to delete it. Do not actually
** remove the %_content row at this time though.
*/
static int fts5StorageDeleteFromIndex(Fts5Storage *p, i64 iDel){




  Fts5Config *pConfig = p->pConfig;
  sqlite3_stmt *pSeek;            /* SELECT to read row iDel from %_data */
  int rc;                         /* Return code */





  rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP, &pSeek, 0);
  if( rc==SQLITE_OK ){
    int rc2;
    sqlite3_bind_int64(pSeek, 1, iDel);
    if( sqlite3_step(pSeek)==SQLITE_ROW ){
      int iCol;
      Fts5InsertCtx ctx;




      ctx.pStorage = p;
      ctx.iCol = -1;
      rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel);
      for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){
        if( pConfig->abUnindexed[iCol-1] ) continue;
        ctx.szCol = 0;
        rc = sqlite3Fts5Tokenize(pConfig, 
            FTS5_TOKENIZE_DOCUMENT,



            (const char*)sqlite3_column_text(pSeek, iCol),
            sqlite3_column_bytes(pSeek, iCol),
            (void*)&ctx,






            fts5StorageInsertCallback
        );
        p->aTotalSize[iCol-1] -= (i64)ctx.szCol;
      }

      p->nTotalRow--;
    }
    rc2 = sqlite3_reset(pSeek);
    if( rc==SQLITE_OK ) rc = rc2;
  }

  return rc;
}


/*
** Insert a record into the %_docsize table. Specifically, do:
**
................................................................................

  return rc;
}

/*
** Remove a row from the FTS table.
*/
int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel){
  Fts5Config *pConfig = p->pConfig;
  int rc;
  sqlite3_stmt *pDel = 0;


  rc = fts5StorageLoadTotals(p, 1);

  /* Delete the index records */
  if( rc==SQLITE_OK ){
    rc = fts5StorageDeleteFromIndex(p, iDel);
  }

  /* Delete the %_docsize record */
  if( rc==SQLITE_OK && pConfig->bColumnsize ){
    rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE, &pDel, 0);
    if( rc==SQLITE_OK ){
      sqlite3_bind_int64(pDel, 1, iDel);
................................................................................
  }

  /* Delete the %_content record */
  if( pConfig->eContent==FTS5_CONTENT_NORMAL ){
    if( rc==SQLITE_OK ){
      rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_CONTENT, &pDel, 0);
    }
    if( rc==SQLITE_OK ){
      sqlite3_bind_int64(pDel, 1, iDel);
      sqlite3_step(pDel);
      rc = sqlite3_reset(pDel);
    }
  }

  /* Write the averages record */
  if( rc==SQLITE_OK ){
    rc = fts5StorageSaveTotals(p);
  }

  return rc;
}

int sqlite3Fts5StorageSpecialDelete(
  Fts5Storage *p, 
  i64 iDel, 
  sqlite3_value **apVal
){
  Fts5Config *pConfig = p->pConfig;
  int rc;
  sqlite3_stmt *pDel = 0;

  assert( pConfig->eContent!=FTS5_CONTENT_NORMAL );
  rc = fts5StorageLoadTotals(p, 1);

  /* Delete the index records */
  if( rc==SQLITE_OK ){
    int iCol;
    Fts5InsertCtx ctx;
    ctx.pStorage = p;
    ctx.iCol = -1;

    rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel);
    for(iCol=0; rc==SQLITE_OK && iCol<pConfig->nCol; iCol++){
      if( pConfig->abUnindexed[iCol] ) continue;
      ctx.szCol = 0;
      rc = sqlite3Fts5Tokenize(pConfig, 
        FTS5_TOKENIZE_DOCUMENT,
        (const char*)sqlite3_value_text(apVal[iCol]),
        sqlite3_value_bytes(apVal[iCol]),
        (void*)&ctx,
        fts5StorageInsertCallback
      );
      p->aTotalSize[iCol] -= (i64)ctx.szCol;
    }
    p->nTotalRow--;
  }

  /* Delete the %_docsize record */
  if( pConfig->bColumnsize ){
    if( rc==SQLITE_OK ){
      rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE, &pDel, 0);
    }
    if( rc==SQLITE_OK ){
      sqlite3_bind_int64(pDel, 1, iDel);
      sqlite3_step(pDel);
      rc = sqlite3_reset(pDel);
    }
  }








|
>
>
>
>

|

>
>
>

>
|
|
<

|
<
<
>
>
>
>
|
|
|
|
|
<
<
<
>
>
>
|
|
<
>
>
>
>
>
>
|
|
|
|
>
|
|
|
|
<
<







 







|




>




|







 







<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<







374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395

396
397


398
399
400
401
402
403
404
405
406



407
408
409
410
411

412
413
414
415
416
417
418
419
420
421
422
423
424
425
426


427
428
429
430
431
432
433
...
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
...
527
528
529
530
531
532
533























































534
535
536
537
538
539
540
}

/*
** If a row with rowid iDel is present in the %_content table, add the
** delete-markers to the FTS index necessary to delete it. Do not actually
** remove the %_content row at this time though.
*/
static int fts5StorageDeleteFromIndex(
  Fts5Storage *p, 
  i64 iDel, 
  sqlite3_value **apVal
){
  Fts5Config *pConfig = p->pConfig;
  sqlite3_stmt *pSeek = 0;        /* SELECT to read row iDel from %_data */
  int rc;                         /* Return code */
  int rc2;                        /* sqlite3_reset() return code */
  int iCol;
  Fts5InsertCtx ctx;

  if( apVal==0 ){
    rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP, &pSeek, 0);
    if( rc!=SQLITE_OK ) return rc;

    sqlite3_bind_int64(pSeek, 1, iDel);
    if( sqlite3_step(pSeek)!=SQLITE_ROW ){


      return sqlite3_reset(pSeek);
    }
  }

  ctx.pStorage = p;
  ctx.iCol = -1;
  rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel);
  for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){
    if( pConfig->abUnindexed[iCol-1]==0 ){



      const char *zText;
      int nText;
      if( pSeek ){
        zText = (const char*)sqlite3_column_text(pSeek, iCol);
        nText = sqlite3_column_bytes(pSeek, iCol);

      }else{
        zText = (const char*)sqlite3_value_text(apVal[iCol-1]);
        nText = sqlite3_value_bytes(apVal[iCol-1]);
      }
      ctx.szCol = 0;
      rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, 
          zText, nText, (void*)&ctx, fts5StorageInsertCallback
      );
      p->aTotalSize[iCol-1] -= (i64)ctx.szCol;
    }
  }
  p->nTotalRow--;

  rc2 = sqlite3_reset(pSeek);
  if( rc==SQLITE_OK ) rc = rc2;


  return rc;
}


/*
** Insert a record into the %_docsize table. Specifically, do:
**
................................................................................

  return rc;
}

/*
** Remove a row from the FTS table.
*/
int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **apVal){
  Fts5Config *pConfig = p->pConfig;
  int rc;
  sqlite3_stmt *pDel = 0;

  assert( pConfig->eContent!=FTS5_CONTENT_NORMAL || apVal==0 );
  rc = fts5StorageLoadTotals(p, 1);

  /* Delete the index records */
  if( rc==SQLITE_OK ){
    rc = fts5StorageDeleteFromIndex(p, iDel, apVal);
  }

  /* Delete the %_docsize record */
  if( rc==SQLITE_OK && pConfig->bColumnsize ){
    rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE, &pDel, 0);
    if( rc==SQLITE_OK ){
      sqlite3_bind_int64(pDel, 1, iDel);
................................................................................
  }

  /* Delete the %_content record */
  if( pConfig->eContent==FTS5_CONTENT_NORMAL ){
    if( rc==SQLITE_OK ){
      rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_CONTENT, &pDel, 0);
    }























































    if( rc==SQLITE_OK ){
      sqlite3_bind_int64(pDel, 1, iDel);
      sqlite3_step(pDel);
      rc = sqlite3_reset(pDel);
    }
  }

Changes to ext/fts5/fts5_vocab.c.

423
424
425
426
427
428
429




430
431
432
433
434
435
436
...
440
441
442
443
444
445
446




447
448
449
450
451
452
453
454
...
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
                pCsr->aDoc[0]++;
              }else{
                int iCol = -1;
                while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
                  int ii = FTS5_POS2COLUMN(iPos);
                  pCsr->aCnt[ii]++;
                  if( iCol!=ii ){




                    pCsr->aDoc[ii]++;
                    iCol = ii;
                  }
                }
              }
            }
            break;
................................................................................
              pCsr->aDoc[0]++;
            }else{
              Fts5Buffer buf = {0, 0, 0};
              rc = sqlite3Fts5IterPoslistBuffer(pCsr->pIter, &buf);
              if( rc==SQLITE_OK ){
                while( 0==sqlite3Fts5PoslistNext64(buf.p, buf.n, &iOff,&iPos) ){
                  assert_nc( iPos>=0 && iPos<nCol );




                  if( iPos<nCol ) pCsr->aDoc[iPos]++;
                }
              }
              sqlite3Fts5BufferFree(&buf);
            }
            break;

          default: 
................................................................................
          }
          if( sqlite3Fts5IterEof(pCsr->pIter) ) break;
        }
      }
    }
  }

  if( pCsr->bEof==0 && pTab->eType==FTS5_VOCAB_COL ){
    while( pCsr->aDoc[pCsr->iCol]==0 ) pCsr->iCol++;
    assert( pCsr->iCol<pCsr->pConfig->nCol );
  }
  return rc;
}

/*







>
>
>
>







 







>
>
>
>
|







 







|







423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
...
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
...
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
                pCsr->aDoc[0]++;
              }else{
                int iCol = -1;
                while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
                  int ii = FTS5_POS2COLUMN(iPos);
                  pCsr->aCnt[ii]++;
                  if( iCol!=ii ){
                    if( ii>=nCol ){
                      rc = FTS5_CORRUPT;
                      break;
                    }
                    pCsr->aDoc[ii]++;
                    iCol = ii;
                  }
                }
              }
            }
            break;
................................................................................
              pCsr->aDoc[0]++;
            }else{
              Fts5Buffer buf = {0, 0, 0};
              rc = sqlite3Fts5IterPoslistBuffer(pCsr->pIter, &buf);
              if( rc==SQLITE_OK ){
                while( 0==sqlite3Fts5PoslistNext64(buf.p, buf.n, &iOff,&iPos) ){
                  assert_nc( iPos>=0 && iPos<nCol );
                  if( iPos>=nCol ){
                    rc = FTS5_CORRUPT;
                    break;
                  }
                  pCsr->aDoc[iPos]++;
                }
              }
              sqlite3Fts5BufferFree(&buf);
            }
            break;

          default: 
................................................................................
          }
          if( sqlite3Fts5IterEof(pCsr->pIter) ) break;
        }
      }
    }
  }

  if( rc==SQLITE_OK && pCsr->bEof==0 && pTab->eType==FTS5_VOCAB_COL ){
    while( pCsr->aDoc[pCsr->iCol]==0 ) pCsr->iCol++;
    assert( pCsr->iCol<pCsr->pConfig->nCol );
  }
  return rc;
}

/*

Changes to ext/fts5/test/fts5fault5.test.

61
62
63
64
65
66
67
68
69
70
71
72
73
74




75

76
77

78
79
80

81
82
83
84
85
86
87
..
93
94
95
96
97
98
99






















100
101
102
103
104
do_faultsim_test 2.2 -faults oom-t* -body {
  db eval { INSERT INTO tt(tt) VALUES('integrity-check') }
} -test {
  faultsim_test_result {0 {}}
}

#-------------------------------------------------------------------------
# OOM while scanning an fts5vocab table.
#
reset_db
do_test 3.0 {
  execsql {
    CREATE VIRTUAL TABLE tt USING fts5(x);
    CREATE VIRTUAL TABLE tv USING fts5vocab(tt, 'row');




    INSERT INTO tt(tt, rank) VALUES('pgsz', 32);

    BEGIN;
  }

  for {set i 0} {$i < 20} {incr i} {
    set str [string repeat "$i " 50]
    execsql { INSERT INTO tt VALUES($str) }

  }
  execsql COMMIT
} {}

do_faultsim_test 3.1 -faults oom-t* -body {
  db eval {
    SELECT term FROM tv;
................................................................................
do_faultsim_test 3.2 -faults oom-t* -body {
  db eval {
    SELECT term FROM tv WHERE term BETWEEN '1' AND '2';
  }
} -test {
  faultsim_test_result {0 {1 10 11 12 13 14 15 16 17 18 19 2}}
}

























finish_test








|






>
>
>
>

>


>



>







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>





61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
...
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
do_faultsim_test 2.2 -faults oom-t* -body {
  db eval { INSERT INTO tt(tt) VALUES('integrity-check') }
} -test {
  faultsim_test_result {0 {}}
}

#-------------------------------------------------------------------------
# OOM while scanning fts5vocab tables.
#
reset_db
do_test 3.0 {
  execsql {
    CREATE VIRTUAL TABLE tt USING fts5(x);
    CREATE VIRTUAL TABLE tv USING fts5vocab(tt, 'row');

    CREATE VIRTUAL TABLE tt2 USING fts5(x, detail=col);
    CREATE VIRTUAL TABLE tv2 USING fts5vocab(tt2, 'col');

    INSERT INTO tt(tt, rank) VALUES('pgsz', 32);
    INSERT INTO tt2(tt2, rank) VALUES('pgsz', 32);
    BEGIN;
  }

  for {set i 0} {$i < 20} {incr i} {
    set str [string repeat "$i " 50]
    execsql { INSERT INTO tt VALUES($str) }
    execsql { INSERT INTO tt2 VALUES($str) }
  }
  execsql COMMIT
} {}

do_faultsim_test 3.1 -faults oom-t* -body {
  db eval {
    SELECT term FROM tv;
................................................................................
do_faultsim_test 3.2 -faults oom-t* -body {
  db eval {
    SELECT term FROM tv WHERE term BETWEEN '1' AND '2';
  }
} -test {
  faultsim_test_result {0 {1 10 11 12 13 14 15 16 17 18 19 2}}
}

breakpoint
do_execsql_test 3.3.0 {
  SELECT * FROM tv2;
} {
  0 x 1 {} 1 x 1 {} 10 x 1 {} 11 x 1 {} 12 x 1 {} 13 x 1 {}        
  14 x 1 {} 15 x 1 {} 16 x 1 {} 17 x 1 {} 18 x 1 {} 19  x 1 {}     
  2 x 1 {} 3 x 1 {} 4 x 1 {} 5 x 1 {} 6 x 1 {} 7 x 1 {} 8 x 1 {}   
  9 x 1 {}
}
do_faultsim_test 3.3 -faults oom-t* -body {
  db eval {
    SELECT * FROM tv2;
  }
} -test {
  faultsim_test_result [list 0 [list                                   \
      0 x 1 {} 1 x 1 {} 10 x 1 {} 11 x 1 {} 12 x 1 {} 13 x 1 {}        \
      14 x 1 {} 15 x 1 {} 16 x 1 {} 17 x 1 {} 18 x 1 {} 19  x 1 {}     \
      2 x 1 {} 3 x 1 {} 4 x 1 {} 5 x 1 {} 6 x 1 {} 7 x 1 {} 8 x 1 {}   \
      9 x 1 {}
  ]]
}



finish_test

Changes to ext/fts5/test/fts5fault8.test.

20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44








45
46
47
48
49
ifcapable !fts5 {
  finish_test
  return
}

foreach_detail_mode $testprefix {

if {[detail_is_none]==0} continue

fts5_aux_test_functions db
do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
  INSERT INTO t1 VALUES('a b c d', '1 2 3 4');
  INSERT INTO t1 VALUES('a b a b', NULL);
  INSERT INTO t1 VALUES(NULL, '1 2 1 2');
}

do_faultsim_test 1 -faults oom-t* -body {
  execsql { 
    SELECT rowid, fts5_test_poslist(t1) FROM t1 WHERE t1 MATCH 'b OR 2' 
  }
} -test {
  faultsim_test_result {0 {1 {0.0.1 1.1.1} 2 {0.0.1 0.0.3} 3 {1.1.1 1.1.3}}} \
                       {1 SQLITE_NOMEM}
}









}

finish_test








<
<








|







>
>
>
>
>
>
>
>





20
21
22
23
24
25
26


27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
ifcapable !fts5 {
  finish_test
  return
}

foreach_detail_mode $testprefix {



fts5_aux_test_functions db
do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
  INSERT INTO t1 VALUES('a b c d', '1 2 3 4');
  INSERT INTO t1 VALUES('a b a b', NULL);
  INSERT INTO t1 VALUES(NULL, '1 2 1 2');
}

do_faultsim_test 1 -faults oom-* -body {
  execsql { 
    SELECT rowid, fts5_test_poslist(t1) FROM t1 WHERE t1 MATCH 'b OR 2' 
  }
} -test {
  faultsim_test_result {0 {1 {0.0.1 1.1.1} 2 {0.0.1 0.0.3} 3 {1.1.1 1.1.3}}} \
                       {1 SQLITE_NOMEM}
}

do_faultsim_test 2 -faults oom-* -body {
  execsql { 
    INSERT INTO t1(t1) VALUES('integrity-check');
  }
} -test {
  faultsim_test_result {0 {}} {1 SQLITE_NOMEM}
}

}

finish_test

Changes to ext/fts5/test/fts5simple2.test.

263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
...
295
296
297
298
299
300
301
302
























303
304
  INSERT INTO t1 VALUES('a b c d');
} {}

do_execsql_test 14.1 {
  SELECT fts5_test_poslist(t1) FROM t1('b') ORDER BY rank;
} {0.0.1}

}

#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 15.1 {
  CREATE VIRTUAL TABLE t1 USING fts5(x, detail=none);
  BEGIN;
    INSERT INTO t1(rowid, x) VALUES(1, 'sqlite');
................................................................................
do_execsql_test 15.3.2 {
  SELECT rowid FROM t1('sqlite') ORDER BY rowid DESC;
} {}

do_test 15.4 {
  execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
} {}
  
























finish_test








<
<







 







|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>


263
264
265
266
267
268
269


270
271
272
273
274
275
276
...
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
  INSERT INTO t1 VALUES('a b c d');
} {}

do_execsql_test 14.1 {
  SELECT fts5_test_poslist(t1) FROM t1('b') ORDER BY rank;
} {0.0.1}



#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 15.1 {
  CREATE VIRTUAL TABLE t1 USING fts5(x, detail=none);
  BEGIN;
    INSERT INTO t1(rowid, x) VALUES(1, 'sqlite');
................................................................................
do_execsql_test 15.3.2 {
  SELECT rowid FROM t1('sqlite') ORDER BY rowid DESC;
} {}

do_test 15.4 {
  execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
} {}

}

#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 16.0 {
  CREATE VIRTUAL TABLE t2 USING fts5(x, detail=none);
  BEGIN;
    INSERT INTO t2(rowid, x) VALUES(1, 'a b c');
    INSERT INTO t2(rowid, x) VALUES(456, 'a b c');
    INSERT INTO t2(rowid, x) VALUES(1000, 'a b c');
  COMMIT;
  UPDATE t2 SET x=x;
}

do_execsql_test 16.1 {
  INSERT INTO t2(t2) VALUES('integrity-check');
} {}

do_execsql_test 16.2 {
  SELECT rowid FROM t2('b') ORDER BY rowid DESC
} {1000 456 1}

  
finish_test

Added ext/fts5/test/fts5update.test.



























































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
# 2016 Jan 16
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library.  The
# focus of this script is testing the FTS5 module.
#

source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5update

# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
  finish_test
  return
}

set docs {
  "eight zero iv eight 7"            "ix one 8 one three ii one"        
  "1 9 9 three viii"                 "5 zero ii 6 nine ix 3"            
  "3 zero 5 2 seven nine"            "two eight viii eight 1"           
  "4 six two 5 9 vii"                "viii ii four 8 i i iv"            
  "vii 0 iv seven 7 viii"            "five 1 nine vi seven"             
  "1 zero zero iii 1"                "one one six 6 nine seven"         
  "one v 4 zero 4 iii ii"            "2 3 eight six ix"                 
  "six iv 7 three 5"                 "ix zero 0 8 ii 7 3"               
  "four six nine 2 vii 3"            "five viii 5 8 0 7"                
}

foreach_detail_mode $::testprefix {

do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
} {}

do_test 1.1 {
  foreach {a b} $docs {
    execsql {INSERT INTO t1 VALUES($a, $b)}
  }
} {}

proc update {iRowid iA iB} {
  set a [lindex $::docs $iA]
  set b [lindex $::docs $iB]
  execsql { UPDATE t1 SET a=$a, b=$b WHERE rowid=$iRowid }
}

set nDoc [llength $::docs]
foreach n {1 5 10 50 100} {
  do_test 1.2.$n {
    execsql BEGIN
    for {set i 1} {$i <= 1000} {incr i} {
      set iRowid [expr {int(rand() * ($nDoc/2)) + 1}]
      set iA [expr {int(rand() * $nDoc)}]
      set iB [expr {int(rand() * $nDoc)}]
      update $iRowid $iA $iB

      if {($i % $n)==0} {
        execsql { COMMIT; BEGIN }
      }

      if {($i % $n)==100} {
        execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
      }
    }
    execsql COMMIT
    execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
  } {}
}

do_execsql_test 1.3 {
  UPDATE t1 SET a=a AND b=b;
  INSERT INTO t1(t1) VALUES('integrity-check');
}

do_test 1.4 {
  execsql { INSERT INTO t1(t1, rank) VALUES('pgsz', 32) }
  for {set i 0} {$i < 50} {incr i} {
    execsql { UPDATE t1 SET a=a AND b=b }
    execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
  }
} {}

}
finish_test


Changes to ext/fts5/test/fts5vocab.test.

52
53
54
55
56
57
58


59
60
61
62
63
64
65
...
385
386
387
388
389
390
391
392
393




















































394
395
  if {[detail_is_none]==0} { error "this is for detail=none mode" }
  set ret [list]
  foreach {a b c} $L {
    lappend ret $a {} $b {}
  }
  set ret
}



do_execsql_test 1.1.1 {
  CREATE VIRTUAL TABLE t1 USING fts5(one, prefix=1, detail=%DETAIL%);
  CREATE VIRTUAL TABLE v1 USING fts5vocab(t1, 'row');
  PRAGMA table_info = v1;
} {
  0 term {} 0 {} 0
................................................................................
if {![detail_is_none]} {
  do_execsql_test 7.3.2 {
    SELECT count(*) FROM txc, txc_c
      WHERE txc.term = txc_c.term AND txc.col=txc_c.col;
  } {57}
}

}





















































finish_test








>
>







 









>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>


52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
...
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
  if {[detail_is_none]==0} { error "this is for detail=none mode" }
  set ret [list]
  foreach {a b c} $L {
    lappend ret $a {} $b {}
  }
  set ret
}

if 1 {

do_execsql_test 1.1.1 {
  CREATE VIRTUAL TABLE t1 USING fts5(one, prefix=1, detail=%DETAIL%);
  CREATE VIRTUAL TABLE v1 USING fts5vocab(t1, 'row');
  PRAGMA table_info = v1;
} {
  0 term {} 0 {} 0
................................................................................
if {![detail_is_none]} {
  do_execsql_test 7.3.2 {
    SELECT count(*) FROM txc, txc_c
      WHERE txc.term = txc_c.term AND txc.col=txc_c.col;
  } {57}
}

}

#-------------------------------------------------------------------------
# Test the fts5vocab tables response to a specific types of corruption:
# where the fts5 index contains hits for columns that do not exist.
#
do_execsql_test 8.0 {
  CREATE VIRTUAL TABLE x1 USING fts5(a, b, c, detail=%DETAIL%);
  INSERT INTO x1 VALUES('a b c', 'd e f', 'g h i');
  INSERT INTO x1 VALUES('g h i', 'a b c', 'd e f');
  INSERT INTO x1 VALUES('d e f', 'g h i', 'a b c');
  CREATE VIRTUAL TABLE x1_r USING fts5vocab(x1, row);
  CREATE VIRTUAL TABLE x1_c USING fts5vocab(x1, col);
}

set resr [star_from_row {a 3 3 b 3 3 c 3 3 d 3 3 e 3 3 f 3 3 g 3 3 h 3 3 i 3 3}]
set resc [star_from_col {
  a a 1 1 a b 1 1 a c 1 1 b a 1 1 
  b b 1 1 b c 1 1 c a 1 1 c b 1 1 
  c c 1 1 d a 1 1 d b 1 1 d c 1 1
  e a 1 1 e b 1 1 e c 1 1 f a 1 1 
  f b 1 1 f c 1 1 g a 1 1 g b 1 1 
  g c 1 1 h a 1 1 h b 1 1 h c 1 1 
  i a 1 1 i b 1 1 i c 1 1
}]
if {[detail_is_none]} { set resc [row_to_col $resr] }

do_execsql_test 8.1.1 { SELECT * FROM x1_r; } $resr
do_execsql_test 8.1.2 { SELECT * FROM x1_c } $resc

do_execsql_test 8.2 {
  PRAGMA writable_schema = 1;
  UPDATE sqlite_master 
  SET sql = 'CREATE VIRTUAL TABLE x1 USING fts5(a, detail=%DETAIL%)'
  WHERE name = 'x1';
}
db close
sqlite3 db test.db
sqlite3_fts5_may_be_corrupt 1

do_execsql_test 8.2.1 { SELECT * FROM x1_r } $resr

if {[detail_is_none]} {
  do_execsql_test 8.2.2 { SELECT * FROM x1_c } $resc
} else {
  do_catchsql_test 8.2.2 { 
    SELECT * FROM x1_c 
  } {1 {database disk image is malformed}}
}

sqlite3_fts5_may_be_corrupt 0

}

finish_test

Changes to main.mk.

330
331
332
333
334
335
336
337

338
339
340
341
342
343
344
  $(TOP)/ext/misc/series.c \
  $(TOP)/ext/misc/spellfix.c \
  $(TOP)/ext/misc/totype.c \
  $(TOP)/ext/misc/wholenumber.c \
  $(TOP)/ext/misc/vfslog.c \
  $(TOP)/ext/fts5/fts5_tcl.c \
  $(TOP)/ext/fts5/fts5_test_mi.c \
  $(TOP)/ext/fts5/fts5_test_tok.c



#TESTSRC += $(TOP)/ext/fts2/fts2_tokenizer.c
#TESTSRC += $(TOP)/ext/fts3/fts3_tokenizer.c

TESTSRC2 = \
  $(TOP)/src/attach.c \







|
>







330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
  $(TOP)/ext/misc/series.c \
  $(TOP)/ext/misc/spellfix.c \
  $(TOP)/ext/misc/totype.c \
  $(TOP)/ext/misc/wholenumber.c \
  $(TOP)/ext/misc/vfslog.c \
  $(TOP)/ext/fts5/fts5_tcl.c \
  $(TOP)/ext/fts5/fts5_test_mi.c \
  $(TOP)/ext/fts5/fts5_test_tok.c \
  $(FTS5_SRC)


#TESTSRC += $(TOP)/ext/fts2/fts2_tokenizer.c
#TESTSRC += $(TOP)/ext/fts3/fts3_tokenizer.c

TESTSRC2 = \
  $(TOP)/src/attach.c \