SQLite

Check-in [3a1df69e58]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix the fts5 integrity-check code so that it works with detail=none tables.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | fts5-offsets
Files: files | file ages | folders
SHA1: 3a1df69e58e5830da1dff158eedbe3817743f58f
User & Date: dan 2015-12-31 18:39:14.423
Context
2016-01-02
19:01
Changes to run many fts5 tests with detail=none and detail=col tables as well as the default detail=full. Also fixes for the bugs uncovered by running said tests. (check-in: 6322a1d984 user: dan tags: fts5-offsets)
2015-12-31
18:39
Fix the fts5 integrity-check code so that it works with detail=none tables. (check-in: 3a1df69e58 user: dan tags: fts5-offsets)
17:36
Fix some problems with fts5 detail=none tables. Some still remain. (check-in: 6a6f7bc40d user: dan tags: fts5-offsets)
Changes
Unified Diff Ignore Whitespace Patch
Changes to ext/fts5/fts5_index.c.
︙︙
5195
5196
5197
5198
5199
5200
5201

5202
5203
5204
5205
5206
5207
5208




5209
5210
5211
5212
5213
5214
5215
5216
5217
5218
5219



5220
5221
5222
5223
5224
5225
5226

  Fts5Index *p,                   /* Fts5 index object */
  int iIdx,
  const char *z,                  /* Index key to query for */
  int n,                          /* Size of index key in bytes */
  int flags,                      /* Flags for Fts5IndexQuery */
  u64 *pCksum                     /* IN/OUT: Checksum value */
){

  u64 cksum = *pCksum;
  Fts5IndexIter *pIdxIter = 0;
  Fts5Buffer buf = {0, 0, 0};
  int rc = sqlite3Fts5IndexQuery(p, z, n, flags, 0, &pIdxIter);

  while( rc==SQLITE_OK && 0==sqlite3Fts5IterEof(pIdxIter) ){
    i64 rowid = sqlite3Fts5IterRowid(pIdxIter);




    rc = sqlite3Fts5IterPoslistBuffer(pIdxIter, &buf);
    if( rc==SQLITE_OK ){
      Fts5PoslistReader sReader;
      for(sqlite3Fts5PoslistReaderInit(buf.p, buf.n, &sReader);
          sReader.bEof==0;
          sqlite3Fts5PoslistReaderNext(&sReader)
      ){
        int iCol = FTS5_POS2COLUMN(sReader.iPos);
        int iOff = FTS5_POS2OFFSET(sReader.iPos);
        cksum ^= sqlite3Fts5IndexEntryCksum(rowid, iCol, iOff, iIdx, z, n);
      }



      rc = sqlite3Fts5IterNext(pIdxIter);
    }
  }
  sqlite3Fts5IterClose(pIdxIter);
  fts5BufferFree(&buf);

  *pCksum = cksum;








>







>
>
>
>
|
|
|
|
|
|
|
|
|
|
|
>
>
>








5195
5196
5197
5198
5199
5200
5201
5202
5203
5204
5205
5206
5207
5208
5209
5210
5211
5212
5213
5214
5215
5216
5217
5218
5219
5220
5221
5222
5223
5224
5225
5226
5227
5228
5229
5230
5231
5232
5233
5234

  Fts5Index *p,                   /* Fts5 index object */
  int iIdx,
  const char *z,                  /* Index key to query for */
  int n,                          /* Size of index key in bytes */
  int flags,                      /* Flags for Fts5IndexQuery */
  u64 *pCksum                     /* IN/OUT: Checksum value */
){
  int eDetail = p->pConfig->eDetail;
  u64 cksum = *pCksum;
  Fts5IndexIter *pIdxIter = 0;
  Fts5Buffer buf = {0, 0, 0};
  int rc = sqlite3Fts5IndexQuery(p, z, n, flags, 0, &pIdxIter);

  while( rc==SQLITE_OK && 0==sqlite3Fts5IterEof(pIdxIter) ){
    i64 rowid = sqlite3Fts5IterRowid(pIdxIter);

    if( eDetail==FTS5_DETAIL_NONE ){
      cksum ^= sqlite3Fts5IndexEntryCksum(rowid, 0, 0, iIdx, z, n);
    }else{
      rc = sqlite3Fts5IterPoslistBuffer(pIdxIter, &buf);
      if( rc==SQLITE_OK ){
        Fts5PoslistReader sReader;
        for(sqlite3Fts5PoslistReaderInit(buf.p, buf.n, &sReader);
            sReader.bEof==0;
            sqlite3Fts5PoslistReaderNext(&sReader)
           ){
          int iCol = FTS5_POS2COLUMN(sReader.iPos);
          int iOff = FTS5_POS2OFFSET(sReader.iPos);
          cksum ^= sqlite3Fts5IndexEntryCksum(rowid, iCol, iOff, iIdx, z, n);
        }
      }
    }
    if( rc==SQLITE_OK ){
      rc = sqlite3Fts5IterNext(pIdxIter);
    }
  }
  sqlite3Fts5IterClose(pIdxIter);
  fts5BufferFree(&buf);

  *pCksum = cksum;
︙︙
5519
5520
5521
5522
5523
5524
5525

5526
5527
5528
5529
5530
5531
5532

**
** Return SQLITE_CORRUPT if any of the internal checks fail, or if the
** checksum does not match. Return SQLITE_OK if all checks pass without
** error, or some other SQLite error code if another error (e.g. OOM)
** occurs.
*/
int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){

  u64 cksum2 = 0;                 /* Checksum based on contents of indexes */
  Fts5Buffer poslist = {0,0,0};   /* Buffer used to hold a poslist */
  Fts5IndexIter *pIter;           /* Used to iterate through entire index */
  Fts5Structure *pStruct;         /* Index structure */

#ifdef SQLITE_DEBUG
  /* Used by extra internal tests only run if NDEBUG is not defined */








>








5527
5528
5529
5530
5531
5532
5533
5534
5535
5536
5537
5538
5539
5540
5541

**
** Return SQLITE_CORRUPT if any of the internal checks fail, or if the
** checksum does not match. Return SQLITE_OK if all checks pass without
** error, or some other SQLite error code if another error (e.g. OOM)
** occurs.
*/
int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){
  int eDetail = p->pConfig->eDetail;
  u64 cksum2 = 0;                 /* Checksum based on contents of indexes */
  Fts5Buffer poslist = {0,0,0};   /* Buffer used to hold a poslist */
  Fts5IndexIter *pIter;           /* Used to iterate through entire index */
  Fts5Structure *pStruct;         /* Index structure */

#ifdef SQLITE_DEBUG
  /* Used by extra internal tests only run if NDEBUG is not defined */
︙︙
5570
5571
5572
5573
5574
5575
5576



5577
5578
5579
5580
5581
5582

5583
5584
5585
5586
5587
5588
5589

    int iOff = 0;               /* Offset within poslist */
    i64 iRowid = fts5MultiIterRowid(pIter);
    char *z = (char*)fts5MultiIterTerm(pIter, &n);

    /* If this is a new term, query for it. Update cksum3 with the results. */
    fts5TestTerm(p, &term, z, n, cksum2, &cksum3);




    poslist.n = 0;
    fts5SegiterPoslist(p, &pIter->aSeg[pIter->aFirst[1].iFirst] , 0, &poslist);
    while( 0==sqlite3Fts5PoslistNext64(poslist.p, poslist.n, &iOff, &iPos) ){
      int iCol = FTS5_POS2COLUMN(iPos);
      int iTokOff = FTS5_POS2OFFSET(iPos);
      cksum2 ^= sqlite3Fts5IndexEntryCksum(iRowid, iCol, iTokOff, -1, z, n);

    }
  }
  fts5TestTerm(p, &term, 0, 0, cksum2, &cksum3);

  fts5MultiIterFree(p, pIter);
  if( p->rc==SQLITE_OK && cksum!=cksum2 ) p->rc = FTS5_CORRUPT;









>
>
>
|
|
|
|
|
|
>








5579
5580
5581
5582
5583
5584
5585
5586
5587
5588
5589
5590
5591
5592
5593
5594
5595
5596
5597
5598
5599
5600
5601
5602

    int iOff = 0;               /* Offset within poslist */
    i64 iRowid = fts5MultiIterRowid(pIter);
    char *z = (char*)fts5MultiIterTerm(pIter, &n);

    /* If this is a new term, query for it. Update cksum3 with the results. */
    fts5TestTerm(p, &term, z, n, cksum2, &cksum3);

    if( eDetail==FTS5_DETAIL_NONE ){
      cksum2 ^= sqlite3Fts5IndexEntryCksum(iRowid, 0, 0, -1, z, n);
    }else{
      poslist.n = 0;
      fts5SegiterPoslist(p, &pIter->aSeg[pIter->aFirst[1].iFirst], 0, &poslist);
      while( 0==sqlite3Fts5PoslistNext64(poslist.p, poslist.n, &iOff, &iPos) ){
        int iCol = FTS5_POS2COLUMN(iPos);
        int iTokOff = FTS5_POS2OFFSET(iPos);
        cksum2 ^= sqlite3Fts5IndexEntryCksum(iRowid, iCol, iTokOff, -1, z, n);
      }
    }
  }
  fts5TestTerm(p, &term, 0, 0, cksum2, &cksum3);

  fts5MultiIterFree(p, pIter);
  if( p->rc==SQLITE_OK && cksum!=cksum2 ) p->rc = FTS5_CORRUPT;
︙︙
Changes to ext/fts5/fts5_storage.c.
︙︙
849
850
851
852
853
854
855


856





857








858
859
860
861
862
863
864

  int iPos;
  int iCol;

  if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){
    pCtx->szCol++;
  }



  iPos = pTermset ? pCtx->iCol : pCtx->szCol-1;





  iCol = pTermset ? 0 : pCtx->iCol;









  rc = sqlite3Fts5TermsetAdd(pTermset, 0, pToken, nToken, &bPresent);
  if( rc==SQLITE_OK && bPresent==0 ){
    pCtx->cksum ^= sqlite3Fts5IndexEntryCksum(
        pCtx->iRowid, iCol, iPos, 0, pToken, nToken
    );
  }








>
>
|
>
>
>
>
>
|
>
>
>
>
>
>
>
>








849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879

  int iPos;
  int iCol;

  if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){
    pCtx->szCol++;
  }

  switch( pCtx->pConfig->eDetail ){
    case FTS5_DETAIL_FULL:
      iPos = pCtx->szCol-1;
      iCol = pCtx->iCol;
      break;

    case FTS5_DETAIL_COLUMNS:
      iPos = pCtx->iCol;
      iCol = 0;
      break;

    default:
      assert( pCtx->pConfig->eDetail==FTS5_DETAIL_NONE );
      iPos = 0;
      iCol = 0;
      break;
  }

  rc = sqlite3Fts5TermsetAdd(pTermset, 0, pToken, nToken, &bPresent);
  if( rc==SQLITE_OK && bPresent==0 ){
    pCtx->cksum ^= sqlite3Fts5IndexEntryCksum(
        pCtx->iRowid, iCol, iPos, 0, pToken, nToken
    );
  }
︙︙
907
908
909
910
911
912
913



914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934

935
936
937




938
939
940
941
942
943
944

    int rc2;
    while( SQLITE_ROW==sqlite3_step(pScan) ){
      int i;
      ctx.iRowid = sqlite3_column_int64(pScan, 0);
      ctx.szCol = 0;
      if( pConfig->bColumnsize ){
        rc = sqlite3Fts5StorageDocsize(p, ctx.iRowid, aColSize);



      }
      for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){
        if( pConfig->abUnindexed[i] ) continue;
        ctx.iCol = i;
        ctx.szCol = 0;
        if( pConfig->eDetail!=FTS5_DETAIL_FULL ){
          rc = sqlite3Fts5TermsetNew(&ctx.pTermset);
        }
        if( rc==SQLITE_OK ){
          rc = sqlite3Fts5Tokenize(pConfig, 
              FTS5_TOKENIZE_DOCUMENT,
              (const char*)sqlite3_column_text(pScan, i+1),
              sqlite3_column_bytes(pScan, i+1),
              (void*)&ctx,
              fts5StorageIntegrityCallback
          );
        }
        if( rc==SQLITE_OK && pConfig->bColumnsize && ctx.szCol!=aColSize[i] ){
          rc = FTS5_CORRUPT;
        }
        aTotalSize[i] += ctx.szCol;

        sqlite3Fts5TermsetFree(ctx.pTermset);
        ctx.pTermset = 0;
      }




      if( rc!=SQLITE_OK ) break;
    }
    rc2 = sqlite3_reset(pScan);
    if( rc==SQLITE_OK ) rc = rc2;
  }

  /* Test that the "totals" (sometimes called "averages") record looks Ok */








>
>
>





|















>
|
|
|
>
>
>
>








922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967

    int rc2;
    while( SQLITE_ROW==sqlite3_step(pScan) ){
      int i;
      ctx.iRowid = sqlite3_column_int64(pScan, 0);
      ctx.szCol = 0;
      if( pConfig->bColumnsize ){
        rc = sqlite3Fts5StorageDocsize(p, ctx.iRowid, aColSize);
      }
      if( rc==SQLITE_OK && pConfig->eDetail==FTS5_DETAIL_NONE ){
        rc = sqlite3Fts5TermsetNew(&ctx.pTermset);
      }
      for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){
        if( pConfig->abUnindexed[i] ) continue;
        ctx.iCol = i;
        ctx.szCol = 0;
        if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
          rc = sqlite3Fts5TermsetNew(&ctx.pTermset);
        }
        if( rc==SQLITE_OK ){
          rc = sqlite3Fts5Tokenize(pConfig, 
              FTS5_TOKENIZE_DOCUMENT,
              (const char*)sqlite3_column_text(pScan, i+1),
              sqlite3_column_bytes(pScan, i+1),
              (void*)&ctx,
              fts5StorageIntegrityCallback
          );
        }
        if( rc==SQLITE_OK && pConfig->bColumnsize && ctx.szCol!=aColSize[i] ){
          rc = FTS5_CORRUPT;
        }
        aTotalSize[i] += ctx.szCol;
        if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
          sqlite3Fts5TermsetFree(ctx.pTermset);
          ctx.pTermset = 0;
        }
      }
      sqlite3Fts5TermsetFree(ctx.pTermset);
      ctx.pTermset = 0;

      if( rc!=SQLITE_OK ) break;
    }
    rc2 = sqlite3_reset(pScan);
    if( rc==SQLITE_OK ) rc = rc2;
  }

  /* Test that the "totals" (sometimes called "averages") record looks Ok */
︙︙
Changes to ext/fts5/test/fts5simple2.test.
︙︙
105
106
107
108
109
110
111
112
113
114
115
116
117



























118
119
120
121
122
123
124
125
126
127
128
129
130
131
132

133
134
135
136
137
138
139
140
141
142
143

do_execsql_test 7.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
  BEGIN;
  INSERT INTO t1 VALUES('a1 b1');
  INSERT INTO t1 VALUES('a1 b2');
  COMMIT;
}
do_execsql_test 7.0.4 { SELECT rowid FROM t1('b*') ORDER BY rowid DESC } {2 1}
do_execsql_test 7.0.5 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC } {2 1}

#-------------------------------------------------------------------------
#
reset_db



























do_execsql_test 7.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
  INSERT INTO t1 VALUES('a1 b1 c1');
  INSERT INTO t1 VALUES('a2 b2 c2');
  INSERT INTO t1 VALUES('a1 b1 c1');
}
do_execsql_test 7.0.1 { SELECT rowid FROM t1('b*') } {1 2 3}
do_execsql_test 7.0.2 { SELECT rowid FROM t1('a1') } {1 3}
do_execsql_test 7.0.3 { SELECT rowid FROM t1('c2') } {2}

do_execsql_test 7.0.4 { SELECT rowid FROM t1('b*') ORDER BY rowid DESC } {3 2 1}
do_execsql_test 7.0.5 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC } {3 1}
do_execsql_test 7.0.7 { SELECT rowid FROM t1('c2') ORDER BY rowid DESC } {2}

do_execsql_test 7.1.0 { INSERT INTO t1(t1) VALUES('optimize') }


do_execsql_test 7.1.1 { SELECT rowid FROM t1('b*') } {1 2 3}
do_execsql_test 7.1.2 { SELECT rowid FROM t1('a1') } {1 3}
do_execsql_test 7.1.3 { SELECT rowid FROM t1('c2') } {2}

do_execsql_test 7.2.1 { SELECT rowid FROM t1('b*') ORDER BY rowid DESC} {3 2 1}
do_execsql_test 7.2.2 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC} {3 1}
do_execsql_test 7.2.3 { SELECT rowid FROM t1('c2') ORDER BY rowid DESC} {2}

finish_test









|
|




>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|





|
|
|

|
|
|
|
|
>
|
<
|
|
|
<
<
<




105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161

162
163
164



165
166
167

do_execsql_test 7.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
  BEGIN;
  INSERT INTO t1 VALUES('a1 b1');
  INSERT INTO t1 VALUES('a1 b2');
  COMMIT;
}
do_execsql_test 7.1 { SELECT rowid FROM t1('b*') ORDER BY rowid DESC } {2 1}
do_execsql_test 7.2 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC } {2 1}

#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 8.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
  INSERT INTO t1 VALUES('a1 b1 c1');
  INSERT INTO t1 VALUES('a2 b2 c2');
  INSERT INTO t1 VALUES('a1 b1 c1');
}
do_execsql_test 8.0.1 { SELECT rowid FROM t1('b*') } {1 2 3}
do_execsql_test 8.0.2 { SELECT rowid FROM t1('a1') } {1 3}
do_execsql_test 8.0.3 { SELECT rowid FROM t1('c2') } {2}

do_execsql_test 8.0.4 { SELECT rowid FROM t1('b*') ORDER BY rowid DESC } {3 2 1}
do_execsql_test 8.0.5 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC } {3 1}
do_execsql_test 8.0.8 { SELECT rowid FROM t1('c2') ORDER BY rowid DESC } {2}

do_execsql_test 8.1.0 { INSERT INTO t1(t1) VALUES('optimize') }

do_execsql_test 8.1.1 { SELECT rowid FROM t1('b*') } {1 2 3}
do_execsql_test 8.1.2 { SELECT rowid FROM t1('a1') } {1 3}
do_execsql_test 8.1.3 { SELECT rowid FROM t1('c2') } {2}

do_execsql_test 8.2.1 { SELECT rowid FROM t1('b*') ORDER BY rowid DESC} {3 2 1}
do_execsql_test 8.2.2 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC} {3 1}
do_execsql_test 8.2.3 { SELECT rowid FROM t1('c2') ORDER BY rowid DESC} {2}

#--------------------------------------------------------------------------
#
reset_db
do_execsql_test 9.0.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
  INSERT INTO t1 VALUES('a1 b1 c1');
  INSERT INTO t1 VALUES('a2 b2 c2');
  INSERT INTO t1 VALUES('a1 b1 c1');
}
do_execsql_test 9.0.1 {
  INSERT INTO t1(t1) VALUES('integrity-check');
} {}

reset_db
do_execsql_test 9.1.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=none);
  INSERT INTO t1 VALUES('a1 b1 c1', 'x y z');
  INSERT INTO t1 VALUES('a2 b2 c2', '1 2 3');
  INSERT INTO t1 VALUES('a1 b1 c1', 'x 2 z');
}

do_execsql_test 9.2.1 {
  INSERT INTO t1(t1) VALUES('integrity-check');
} {}




finish_test