/ Check-in [c020a291]
Login
Overview
Comment:Further tests and fixes for fts5.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts5
Files: files | file ages | folders
SHA1:c020a291ed293a66d21c5885e50a7fee04aa6366
User & Date: dan 2015-01-21 20:30:14
Context
2015-01-22
19:13
Add further tests and fixes for fts5. check-in: 5b295897 user: dan tags: fts5
2015-01-21
20:30
Further tests and fixes for fts5. check-in: c020a291 user: dan tags: fts5
18:23
Fix an fts5 issue with loading doclist-indexes for a term that is the last thing on its leaf page. check-in: e0d61442 user: dan tags: fts5
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5_index.c.

1052
1053
1054
1055
1056
1057
1058

1059
1060
1061
1062

1063
1064
1065
1066
1067
1068
1069
....
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
....
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
}

/*
** Return the total number of segments in index structure pStruct.
*/
static int fts5StructureCountSegments(Fts5Structure *pStruct){
  int nSegment = 0;               /* Total number of segments */

  int iLvl;                       /* Used to iterate through levels */

  for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
    nSegment += pStruct->aLevel[iLvl].nSeg;

  }

  return nSegment;
}

/*
** Serialize and store the "structure" record for index iIdx.
................................................................................
** checksum does not match. Return SQLITE_OK if all checks pass without
** error, or some other SQLite error code if another error (e.g. OOM)
** occurs.
*/
int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){
  Fts5Config *pConfig = p->pConfig;
  int iIdx;                       /* Used to iterate through indexes */
  int rc;                         /* Return code */
  u64 cksum2 = 0;                 /* Checksum based on contents of indexes */

  /* Check that the checksum of the index matches the argument checksum */
  for(iIdx=0; iIdx<=pConfig->nPrefix; iIdx++){
    Fts5MultiSegIter *pIter;
    Fts5Structure *pStruct = fts5StructureRead(p, iIdx);
    for(fts5MultiIterNew(p, pStruct, iIdx, 0, 0, 0, 0, -1, 0, &pIter);
................................................................................
        fflush(stdout);
#endif
      }
    }
    fts5MultiIterFree(p, pIter);
    fts5StructureRelease(pStruct);
  }
  rc = p->rc;
  if( rc==SQLITE_OK && cksum!=cksum2 ) rc = FTS5_CORRUPT;

  /* Check that the internal nodes of each segment match the leaves */
  for(iIdx=0; rc==SQLITE_OK && iIdx<=pConfig->nPrefix; iIdx++){
    Fts5Structure *pStruct = fts5StructureRead(p, iIdx);
    if( pStruct ){
      int iLvl, iSeg;
      for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
        for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
          Fts5StructureSegment *pSeg = &pStruct->aLevel[iLvl].aSeg[iSeg];
          fts5IndexIntegrityCheckSegment(p, iIdx, pSeg);
        }
      }
    }
    fts5StructureRelease(pStruct);
    rc = p->rc;
  }

  return rc;
}


/*
** Indicate that all subsequent calls to sqlite3Fts5IndexWrite() pertain
** to the document with rowid iRowid.
*/







>
|
<
|
|
>







 







<







 







<
|


|











<


|







1052
1053
1054
1055
1056
1057
1058
1059
1060

1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
....
3879
3880
3881
3882
3883
3884
3885

3886
3887
3888
3889
3890
3891
3892
....
3911
3912
3913
3914
3915
3916
3917

3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932

3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
}

/*
** Return the total number of segments in index structure pStruct.
*/
static int fts5StructureCountSegments(Fts5Structure *pStruct){
  int nSegment = 0;               /* Total number of segments */
  if( pStruct ){
    int iLvl;                     /* Used to iterate through levels */

    for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
      nSegment += pStruct->aLevel[iLvl].nSeg;
    }
  }

  return nSegment;
}

/*
** Serialize and store the "structure" record for index iIdx.
................................................................................
** checksum does not match. Return SQLITE_OK if all checks pass without
** error, or some other SQLite error code if another error (e.g. OOM)
** occurs.
*/
int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){
  Fts5Config *pConfig = p->pConfig;
  int iIdx;                       /* Used to iterate through indexes */

  u64 cksum2 = 0;                 /* Checksum based on contents of indexes */

  /* Check that the checksum of the index matches the argument checksum */
  for(iIdx=0; iIdx<=pConfig->nPrefix; iIdx++){
    Fts5MultiSegIter *pIter;
    Fts5Structure *pStruct = fts5StructureRead(p, iIdx);
    for(fts5MultiIterNew(p, pStruct, iIdx, 0, 0, 0, 0, -1, 0, &pIter);
................................................................................
        fflush(stdout);
#endif
      }
    }
    fts5MultiIterFree(p, pIter);
    fts5StructureRelease(pStruct);
  }

  if( p->rc==SQLITE_OK && cksum!=cksum2 ) p->rc = FTS5_CORRUPT;

  /* Check that the internal nodes of each segment match the leaves */
  for(iIdx=0; p->rc==SQLITE_OK && iIdx<=pConfig->nPrefix; iIdx++){
    Fts5Structure *pStruct = fts5StructureRead(p, iIdx);
    if( pStruct ){
      int iLvl, iSeg;
      for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
        for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
          Fts5StructureSegment *pSeg = &pStruct->aLevel[iLvl].aSeg[iSeg];
          fts5IndexIntegrityCheckSegment(p, iIdx, pSeg);
        }
      }
    }
    fts5StructureRelease(pStruct);

  }

  return fts5IndexReturn(p);
}


/*
** Indicate that all subsequent calls to sqlite3Fts5IndexWrite() pertain
** to the document with rowid iRowid.
*/

Changes to ext/fts5/fts5_storage.c.

702
703
704
705
706
707
708

709
710
711
712

713
714
715
716
717
718
719
...
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
      pConfig->zDb, pConfig->zName, zSuffix
  );
  if( zSql==0 ){
    rc = SQLITE_NOMEM;
  }else{
    sqlite3_stmt *pCnt = 0;
    rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pCnt, 0);

    if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pCnt) ){
      *pnRow = sqlite3_column_int64(pCnt, 0);
    }
    rc = sqlite3_finalize(pCnt);

  }

  sqlite3_free(zSql);
  return rc;
}

/*
................................................................................
  Fts5Storage *p, 
  const char *z, 
  sqlite3_value *pVal
){
  sqlite3_stmt *pReplace = 0;
  int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_CONFIG, &pReplace, 0);
  if( rc==SQLITE_OK ){
    sqlite3_bind_text(pReplace, 1, z, -1, SQLITE_TRANSIENT);
    sqlite3_bind_value(pReplace, 2, pVal);
    sqlite3_step(pReplace);
    rc = sqlite3_reset(pReplace);
  }
  if( rc==SQLITE_OK ){
    int iNew = p->pConfig->iCookie + 1;
    rc = sqlite3Fts5IndexSetCookie(p->pIndex, iNew);







>
|
|
|
|
>







 







|







702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
...
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
      pConfig->zDb, pConfig->zName, zSuffix
  );
  if( zSql==0 ){
    rc = SQLITE_NOMEM;
  }else{
    sqlite3_stmt *pCnt = 0;
    rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pCnt, 0);
    if( rc==SQLITE_OK ){
      if( SQLITE_ROW==sqlite3_step(pCnt) ){
        *pnRow = sqlite3_column_int64(pCnt, 0);
      }
      rc = sqlite3_finalize(pCnt);
    }
  }

  sqlite3_free(zSql);
  return rc;
}

/*
................................................................................
  Fts5Storage *p, 
  const char *z, 
  sqlite3_value *pVal
){
  sqlite3_stmt *pReplace = 0;
  int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_CONFIG, &pReplace, 0);
  if( rc==SQLITE_OK ){
    sqlite3_bind_text(pReplace, 1, z, -1, SQLITE_STATIC);
    sqlite3_bind_value(pReplace, 2, pVal);
    sqlite3_step(pReplace);
    rc = sqlite3_reset(pReplace);
  }
  if( rc==SQLITE_OK ){
    int iNew = p->pConfig->iCookie + 1;
    rc = sqlite3Fts5IndexSetCookie(p->pIndex, iNew);

Changes to ext/fts5/test/fts5aa.test.

365
366
367
368
369
370
371










372
373
374
      RELEASE aaa;
    }
    incr nRow
  }
  set nRow
} {200}











finish_test









>
>
>
>
>
>
>
>
>
>



365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
      RELEASE aaa;
    }
    incr nRow
  }
  set nRow
} {200}

do_execsql_test 15.0 {
  INSERT INTO t1(t1) VALUES('integrity-check');
}
do_execsql_test 15.1 {
  UPDATE t1_content SET c1 = 'xyz xyz xyz xyz xyz abc' WHERE rowid = 1;
}
do_catchsql_test 15.2 {
  INSERT INTO t1(t1) VALUES('integrity-check');
} {1 {database disk image is malformed}}

finish_test


Changes to ext/fts5/test/fts5ah.test.

24
25
26
27
28
29
30


31
32
33
34
35
36


37
38
39
40
41
42




43
44
45
46
47
48
49
#-------------------------------------------------------------------------
# This file contains tests for very large doclists.
#

do_test 1.0 {
  execsql { CREATE VIRTUAL TABLE t1 USING fts5(a) }
  execsql { INSERT INTO t1(t1, rank) VALUES('pgsz', 128) }


  for {set i 1} {$i <= 10000} {incr i} {
    set v {x x x x x x x x x x x x x x x x x x x x}
    if {($i % 2139)==0} {lset v 3 Y ; lappend Y $i}
    if {($i % 1577)==0} {lset v 5 W ; lappend W $i}
    execsql { INSERT INTO t1 VALUES($v) }
  }


} {}

do_execsql_test 1.1 {
  SELECT rowid FROM t1 WHERE t1 MATCH 'x AND w'
} [lsort -integer -decr $W]





do_execsql_test 1.2 {
  SELECT rowid FROM t1 WHERE t1 MATCH 'y AND x'
} [lsort -integer -decr $Y]

do_execsql_test 1.3 {
  INSERT INTO t1(t1) VALUES('integrity-check');
}







>
>






>
>


|



>
>
>
>







24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
#-------------------------------------------------------------------------
# This file contains tests for very large doclists.
#

do_test 1.0 {
  execsql { CREATE VIRTUAL TABLE t1 USING fts5(a) }
  execsql { INSERT INTO t1(t1, rank) VALUES('pgsz', 128) }
  set v {w w w w w w w w w w w w w w w w w w w w}
  execsql { INSERT INTO t1(rowid, a) VALUES(0, $v) }
  for {set i 1} {$i <= 10000} {incr i} {
    set v {x x x x x x x x x x x x x x x x x x x x}
    if {($i % 2139)==0} {lset v 3 Y ; lappend Y $i}
    if {($i % 1577)==0} {lset v 5 W ; lappend W $i}
    execsql { INSERT INTO t1 VALUES($v) }
  }
  set v {w w w w w w w w w w w w w w w w w w w w}
  execsql { INSERT INTO t1 VALUES($v) }
} {}

do_execsql_test 1.1.1 {
  SELECT rowid FROM t1 WHERE t1 MATCH 'x AND w'
} [lsort -integer -decr $W]

do_execsql_test 1.1.2 {
  SELECT rowid FROM t1 WHERE t1 MATCH 'x* AND w*'
} [lsort -integer -decr $W]

do_execsql_test 1.2 {
  SELECT rowid FROM t1 WHERE t1 MATCH 'y AND x'
} [lsort -integer -decr $Y]

do_execsql_test 1.3 {
  INSERT INTO t1(t1) VALUES('integrity-check');
}

Changes to ext/fts5/test/fts5fault1.test.

27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
...
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
...
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
...
249
250
251
252
253
254
255

























256
257
258
259
260
261
262
263
264
265
266
267
268
#   1: CREATE VIRTUAL TABLE
#   2: INSERT statement
#   3: DELETE statement
#   4: MATCH expressions
#
#

if 0 {

faultsim_save_and_close
do_faultsim_test 1 -prep {
  faultsim_restore_and_reopen
} -body {
  execsql { CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix='1, 2, 3') }
} -test {
................................................................................
    execsql { SELECT rowid FROM t2 WHERE t2 MATCH '$expr' }
  " -test "
    faultsim_test_result {[list 0 $res]}
  "
}


}

#-------------------------------------------------------------------------
# The following tests use a larger database populated with random data.
#
# The database page size is set to 512 bytes and the FTS5 page size left
# at the default 1000 bytes. This means that reading a node may require
# pulling an overflow page from disk, which is an extra opportunity for
# an error to occur.
................................................................................
      SELECT max(rowid) FROM x1_data 
    )
  }
} -test {
  faultsim_test_result [list 0 1]
}

finish_test

#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 6.0 {
  CREATE VIRTUAL TABLE x1 USING fts5(x);
  INSERT INTO x1(x1, rank) VALUES('automerge', 0);
................................................................................
} -test {
  faultsim_test_result [list 0 {}]
  if {$testrc==0} {
    set nCnt [db one {SELECT count(*) FROM x1_data}]
    if {$nCnt!=3} { error "expected 3 entries but there are $nCnt" }
  }
}


























#-------------------------------------------------------------------------
do_faultsim_test 7.0 -faults oom* -prep {
  catch { db close }
} -body {
  sqlite3 db test.db
} -test {
  faultsim_test_result [list 0 {}] [list 1 {}]
}


finish_test








|







 







<
<







 







|







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>













27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
...
104
105
106
107
108
109
110


111
112
113
114
115
116
117
...
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
...
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
#   1: CREATE VIRTUAL TABLE
#   2: INSERT statement
#   3: DELETE statement
#   4: MATCH expressions
#
#

if 1 {

faultsim_save_and_close
do_faultsim_test 1 -prep {
  faultsim_restore_and_reopen
} -body {
  execsql { CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix='1, 2, 3') }
} -test {
................................................................................
    execsql { SELECT rowid FROM t2 WHERE t2 MATCH '$expr' }
  " -test "
    faultsim_test_result {[list 0 $res]}
  "
}




#-------------------------------------------------------------------------
# The following tests use a larger database populated with random data.
#
# The database page size is set to 512 bytes and the FTS5 page size left
# at the default 1000 bytes. This means that reading a node may require
# pulling an overflow page from disk, which is an extra opportunity for
# an error to occur.
................................................................................
      SELECT max(rowid) FROM x1_data 
    )
  }
} -test {
  faultsim_test_result [list 0 1]
}

}

#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 6.0 {
  CREATE VIRTUAL TABLE x1 USING fts5(x);
  INSERT INTO x1(x1, rank) VALUES('automerge', 0);
................................................................................
} -test {
  faultsim_test_result [list 0 {}]
  if {$testrc==0} {
    set nCnt [db one {SELECT count(*) FROM x1_data}]
    if {$nCnt!=3} { error "expected 3 entries but there are $nCnt" }
  }
}

do_faultsim_test 6.2 -faults oom* -prep {
  faultsim_restore_and_reopen
} -body {
  execsql { INSERT INTO x1(x1, rank) VALUES('pgsz', 32) }
} -test {
  faultsim_test_result [list 0 {}]
}

do_faultsim_test 6.3 -faults oom-* -prep {
  faultsim_restore_and_reopen
} -body {
  execsql { INSERT INTO x1(x1) VALUES('integrity-check') }
} -test {
  faultsim_test_result [list 0 {}]
}

do_faultsim_test 6.4 -faults oom-* -prep {
  faultsim_restore_and_reopen
} -body {
  execsql { INSERT INTO x1(x1) VALUES('optimize') }
} -test {
  faultsim_test_result [list 0 {}]
}


#-------------------------------------------------------------------------
do_faultsim_test 7.0 -faults oom* -prep {
  catch { db close }
} -body {
  sqlite3 db test.db
} -test {
  faultsim_test_result [list 0 {}] [list 1 {}]
}


finish_test

Changes to ext/fts5/test/fts5rowid.test.

146
147
148
149
150
151
152
153




























154
155
  } [expr $tn+1]
}

set res [db one {SELECT count(*) FROM x3_data}]
do_execsql_test 4.2 {
  SELECT count(fts5_decode(rowid, block)) FROM x3_data;
} $res





























finish_test









>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>


146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
  } [expr $tn+1]
}

set res [db one {SELECT count(*) FROM x3_data}]
do_execsql_test 4.2 {
  SELECT count(fts5_decode(rowid, block)) FROM x3_data;
} $res

#-------------------------------------------------------------------------
# Position lists with large values.
#
set strlist [list \
  "[string repeat {w } 400]a"  \
  "[string repeat {x } 400]a"  \
  "[string repeat {y } 400]a"  \
  "[string repeat {z } 400]a"  \
]
do_test 5.0 {
  execsql {
    BEGIN;
    CREATE VIRTUAL TABLE x4 USING fts5(a);
    INSERT INTO x4(x4, rank) VALUES('pgsz', 32);
  }
  foreach str $strlist { execsql { INSERT INTO x4 VALUES($str) } }
  execsql COMMIT
} {}

do_execsql_test 5.1 {
  SELECT rowid FROM x4 WHERE x4 MATCH 'a'
} {4 3 2 1}

set res [db one {SELECT count(*) FROM x4_data}]
do_execsql_test 5.2 {
  SELECT count(fts5_decode(rowid, block)) FROM x4_data;
} $res

finish_test