/ Check-in [c1f76686]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Improve test coverage of fts5_index.c.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: c1f76686cee3918b1be785a4071d68cb3afda0ef
User & Date: dan 2015-09-14 19:51:05
Context
2015-09-15
11:58
Fix a problem with fts5 "ORDER BY rowid DESC" queries and large terms. check-in: b26d8f79 user: dan tags: trunk
2015-09-14
22:53
Merge updates from trunk. check-in: 25c15771 user: mistachkin tags: mutexInitIsInitReCheck
19:51
Improve test coverage of fts5_index.c. check-in: c1f76686 user: dan tags: trunk
19:26
Testability improvements for the ONEPASS_MULTI enhancement. check-in: d2df93f2 user: drh tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5_index.c.

1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
....
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
....
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
....
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
....
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
....
4151
4152
4153
4154
4155
4156
4157
4158
4159
4160
4161
4162
4163
4164
4165
....
5286
5287
5288
5289
5290
5291
5292
5293
5294
5295
5296
5297
5298
5299
5300
  Fts5SegIter *pIter              /* Iterator to advance to next page */
){
  Fts5Data *pLeaf;
  Fts5StructureSegment *pSeg = pIter->pSeg;
  fts5DataRelease(pIter->pLeaf);
  pIter->iLeafPgno++;
  if( pIter->pNextLeaf ){
    assert( pIter->iLeafPgno<=pSeg->pgnoLast );
    pIter->pLeaf = pIter->pNextLeaf;
    pIter->pNextLeaf = 0;
  }else if( pIter->iLeafPgno<=pSeg->pgnoLast ){
    pIter->pLeaf = fts5DataRead(p, 
        FTS5_SEGMENT_ROWID(pSeg->iSegid, pIter->iLeafPgno)
    );
  }else{
................................................................................
    Fts5Data *pNew;
    pIter->iLeafPgno--;
    pNew = fts5DataRead(p, FTS5_SEGMENT_ROWID(
          pIter->pSeg->iSegid, pIter->iLeafPgno
    ));
    if( pNew ){
      if( pIter->iLeafPgno==pIter->iTermLeafPgno ){
        if( pIter->iTermLeafOffset<pNew->szLeaf ){
          pIter->pLeaf = pNew;
          pIter->iLeafOffset = pIter->iTermLeafOffset;
        }
      }else{
        int iRowidOff;
        iRowidOff = fts5LeafFirstRowidOff(pNew);
        if( iRowidOff ){
          pIter->pLeaf = pNew;
          pIter->iLeafOffset = iRowidOff;
        }
................................................................................
  Fts5SegWriter *pWriter,
  int nTerm, const u8 *pTerm 
){
  int nPrefix;                    /* Bytes of prefix compression for term */
  Fts5PageWriter *pPage = &pWriter->writer;
  Fts5Buffer *pPgidx = &pWriter->writer.pgidx;

  if( p->rc ) return;
  assert( pPage->buf.n>=4 );
  assert( pPage->buf.n>4 || pWriter->bFirstTermInPage );

  /* If the current leaf page is full, flush it to disk. */
  if( (pPage->buf.n + pPgidx->n + nTerm + 2)>=p->pConfig->pgsz ){
    if( pPage->buf.n>4 ){
      fts5WriteFlushLeaf(p, pWriter);
................................................................................
  Fts5Index *p, 
  Fts5SegWriter *pWriter,         /* Writer object */
  int *pnLeaf                     /* OUT: Number of leaf pages in b-tree */
){
  int i;
  Fts5PageWriter *pLeaf = &pWriter->writer;
  if( p->rc==SQLITE_OK ){
    if( pLeaf->pgno==1 && pLeaf->buf.n==0 ){
      *pnLeaf = 0;
    }else{
      if( pLeaf->buf.n>4 ){
        fts5WriteFlushLeaf(p, pWriter);
      }
      *pnLeaf = pLeaf->pgno-1;

      fts5WriteFlushBtree(p, pWriter);
    }
  }
  fts5BufferFree(&pLeaf->term);
  fts5BufferFree(&pLeaf->buf);
  fts5BufferFree(&pLeaf->pgidx);
  fts5BufferFree(&pWriter->btterm);

  for(i=0; i<pWriter->nDlidx; i++){
................................................................................
      const u8 *pDoclist;         /* Pointer to doclist for this term */
      int nDoclist;               /* Size of doclist in bytes */

      /* Write the term for this entry to disk. */
      sqlite3Fts5HashScanEntry(pHash, &zTerm, &pDoclist, &nDoclist);
      fts5WriteAppendTerm(p, &writer, strlen(zTerm), (const u8*)zTerm);

      if( writer.bFirstRowidInPage==0 
       && pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) 
      ){
        /* The entire doclist will fit on the current leaf. */
        fts5BufferSafeAppendBlob(pBuf, pDoclist, nDoclist);
      }else{
        i64 iRowid = 0;
        i64 iDelta = 0;
        int iOff = 0;

        /*  writer.bFirstRowidInPage = 0; */

        /* The entire doclist will not fit on this leaf. The following 
        ** loop iterates through the poslists that make up the current 
        ** doclist.  */
        while( p->rc==SQLITE_OK && iOff<nDoclist ){
          int nPos;
          int nCopy;
          int bDummy;
................................................................................
    for(fts5MultiIterNew(p, pStruct, 1, flags, pToken, nToken, -1, 0, &p1);
        fts5MultiIterEof(p, p1)==0;
        fts5MultiIterNext(p, p1, 0, 0)
    ){
      i64 iRowid = fts5MultiIterRowid(p1);
      int nTerm;
      const u8 *pTerm = fts5MultiIterTerm(p1, &nTerm);
      assert( memcmp(pToken, pTerm, MIN(nToken, nTerm))<=0 );
      if( nTerm<nToken || memcmp(pToken, pTerm, nToken) ) break;

      if( doclist.n>0 && iRowid<=iLastRowid ){
        for(i=0; p->rc==SQLITE_OK && doclist.n; i++){
          assert( i<nBuf );
          if( aBuf[i].n==0 ){
            fts5BufferSwap(&doclist, &aBuf[i]);
................................................................................
    int nPos;
    int bDummy;
    iOff += fts5GetPoslistSize(&a[iOff], &nPos, &bDummy);
    iOff += fts5DecodePoslist(pRc, pBuf, &a[iOff], MIN(n-iOff, nPos));
    if( iOff<n ){
      i64 iDelta;
      iOff += sqlite3Fts5GetVarint(&a[iOff], (u64*)&iDelta);
      if( iDelta==0 ) return iOff;
      iDocid += iDelta;
      sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " id=%lld", iDocid);
    }
  }

  return iOff;
}







<







 







<
|
|
<







 







|







 







|
<
<
|
|
|
|
<
|
<







 







|
|
<







<
<







 







|







 







<







1423
1424
1425
1426
1427
1428
1429

1430
1431
1432
1433
1434
1435
1436
....
1654
1655
1656
1657
1658
1659
1660

1661
1662

1663
1664
1665
1666
1667
1668
1669
....
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
....
3308
3309
3310
3311
3312
3313
3314
3315


3316
3317
3318
3319

3320

3321
3322
3323
3324
3325
3326
3327
....
3752
3753
3754
3755
3756
3757
3758
3759
3760

3761
3762
3763
3764
3765
3766
3767


3768
3769
3770
3771
3772
3773
3774
....
4141
4142
4143
4144
4145
4146
4147
4148
4149
4150
4151
4152
4153
4154
4155
....
5276
5277
5278
5279
5280
5281
5282

5283
5284
5285
5286
5287
5288
5289
  Fts5SegIter *pIter              /* Iterator to advance to next page */
){
  Fts5Data *pLeaf;
  Fts5StructureSegment *pSeg = pIter->pSeg;
  fts5DataRelease(pIter->pLeaf);
  pIter->iLeafPgno++;
  if( pIter->pNextLeaf ){

    pIter->pLeaf = pIter->pNextLeaf;
    pIter->pNextLeaf = 0;
  }else if( pIter->iLeafPgno<=pSeg->pgnoLast ){
    pIter->pLeaf = fts5DataRead(p, 
        FTS5_SEGMENT_ROWID(pSeg->iSegid, pIter->iLeafPgno)
    );
  }else{
................................................................................
    Fts5Data *pNew;
    pIter->iLeafPgno--;
    pNew = fts5DataRead(p, FTS5_SEGMENT_ROWID(
          pIter->pSeg->iSegid, pIter->iLeafPgno
    ));
    if( pNew ){
      if( pIter->iLeafPgno==pIter->iTermLeafPgno ){

        pIter->pLeaf = pNew;
        pIter->iLeafOffset = pIter->iTermLeafOffset;

      }else{
        int iRowidOff;
        iRowidOff = fts5LeafFirstRowidOff(pNew);
        if( iRowidOff ){
          pIter->pLeaf = pNew;
          pIter->iLeafOffset = iRowidOff;
        }
................................................................................
  Fts5SegWriter *pWriter,
  int nTerm, const u8 *pTerm 
){
  int nPrefix;                    /* Bytes of prefix compression for term */
  Fts5PageWriter *pPage = &pWriter->writer;
  Fts5Buffer *pPgidx = &pWriter->writer.pgidx;

  assert( p->rc==SQLITE_OK );
  assert( pPage->buf.n>=4 );
  assert( pPage->buf.n>4 || pWriter->bFirstTermInPage );

  /* If the current leaf page is full, flush it to disk. */
  if( (pPage->buf.n + pPgidx->n + nTerm + 2)>=p->pConfig->pgsz ){
    if( pPage->buf.n>4 ){
      fts5WriteFlushLeaf(p, pWriter);
................................................................................
  Fts5Index *p, 
  Fts5SegWriter *pWriter,         /* Writer object */
  int *pnLeaf                     /* OUT: Number of leaf pages in b-tree */
){
  int i;
  Fts5PageWriter *pLeaf = &pWriter->writer;
  if( p->rc==SQLITE_OK ){
    assert( pLeaf->pgno>=1 );


    if( pLeaf->buf.n>4 ){
      fts5WriteFlushLeaf(p, pWriter);
    }
    *pnLeaf = pLeaf->pgno-1;

    fts5WriteFlushBtree(p, pWriter);

  }
  fts5BufferFree(&pLeaf->term);
  fts5BufferFree(&pLeaf->buf);
  fts5BufferFree(&pLeaf->pgidx);
  fts5BufferFree(&pWriter->btterm);

  for(i=0; i<pWriter->nDlidx; i++){
................................................................................
      const u8 *pDoclist;         /* Pointer to doclist for this term */
      int nDoclist;               /* Size of doclist in bytes */

      /* Write the term for this entry to disk. */
      sqlite3Fts5HashScanEntry(pHash, &zTerm, &pDoclist, &nDoclist);
      fts5WriteAppendTerm(p, &writer, strlen(zTerm), (const u8*)zTerm);

      assert( writer.bFirstRowidInPage==0 );
      if( pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) ){

        /* The entire doclist will fit on the current leaf. */
        fts5BufferSafeAppendBlob(pBuf, pDoclist, nDoclist);
      }else{
        i64 iRowid = 0;
        i64 iDelta = 0;
        int iOff = 0;



        /* The entire doclist will not fit on this leaf. The following 
        ** loop iterates through the poslists that make up the current 
        ** doclist.  */
        while( p->rc==SQLITE_OK && iOff<nDoclist ){
          int nPos;
          int nCopy;
          int bDummy;
................................................................................
    for(fts5MultiIterNew(p, pStruct, 1, flags, pToken, nToken, -1, 0, &p1);
        fts5MultiIterEof(p, p1)==0;
        fts5MultiIterNext(p, p1, 0, 0)
    ){
      i64 iRowid = fts5MultiIterRowid(p1);
      int nTerm;
      const u8 *pTerm = fts5MultiIterTerm(p1, &nTerm);
      assert_nc( memcmp(pToken, pTerm, MIN(nToken, nTerm))<=0 );
      if( nTerm<nToken || memcmp(pToken, pTerm, nToken) ) break;

      if( doclist.n>0 && iRowid<=iLastRowid ){
        for(i=0; p->rc==SQLITE_OK && doclist.n; i++){
          assert( i<nBuf );
          if( aBuf[i].n==0 ){
            fts5BufferSwap(&doclist, &aBuf[i]);
................................................................................
    int nPos;
    int bDummy;
    iOff += fts5GetPoslistSize(&a[iOff], &nPos, &bDummy);
    iOff += fts5DecodePoslist(pRc, pBuf, &a[iOff], MIN(n-iOff, nPos));
    if( iOff<n ){
      i64 iDelta;
      iOff += sqlite3Fts5GetVarint(&a[iOff], (u64*)&iDelta);

      iDocid += iDelta;
      sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " id=%lld", iDocid);
    }
  }

  return iOff;
}

Changes to ext/fts5/test/fts5corrupt3.test.

19
20
21
22
23
24
25













26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
..
70
71
72
73
74
75
76
77













































































































































































78
79
80

# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
  finish_test
  return
}
sqlite3_fts5_may_be_corrupt 1














# Create a simple FTS5 table containing 100 documents. Each document 
# contains 10 terms, each of which start with the character "x".
#
expr srand(0)
db func rnddoc fts5_rnddoc
do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(x);
  INSERT INTO t1(t1, rank) VALUES('pgsz', 64);
  WITH ii(i) AS (SELECT 1 UNION SELECT i+1 FROM ii WHERE i<100)
  INSERT INTO t1 SELECT rnddoc(10) FROM ii;
}
set mask [expr 31 << 31]

do_test 1.1 {
  # Pick out the rowid of the right-most b-tree leaf in the new segment.
  set rowid [db one {
    SELECT max(rowid) FROM t1_data WHERE ((rowid>>31) & 0x0F)==1
  }]
  set L [db one {SELECT length(block) FROM t1_data WHERE rowid = $rowid}]
................................................................................
  UPDATE t2_data SET block = block || 'abcd' WHERE id=1;
  SELECT length(block) FROM t2_data WHERE id=1;
} {6}
do_execsql_test 2.2 {
  INSERT INTO t2 VALUES(rnddoc(10));
  SELECT length(block) FROM t2_data WHERE id=1;
} {2}














































































































































































sqlite3_fts5_may_be_corrupt 0
finish_test








>
>
>
>
>
>
>
>
>
>
>
>
>




<
<
|
<
<
<
<
<
<







 








>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>



19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42


43






44
45
46
47
48
49
50
..
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258

# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
  finish_test
  return
}
sqlite3_fts5_may_be_corrupt 1

proc create_t1 {} {
  expr srand(0)
  db func rnddoc fts5_rnddoc
  db eval {
    CREATE VIRTUAL TABLE t1 USING fts5(x);
    INSERT INTO t1(t1, rank) VALUES('pgsz', 64);
    WITH ii(i) AS (SELECT 1 UNION SELECT i+1 FROM ii WHERE i<100)
      INSERT INTO t1 SELECT rnddoc(10) FROM ii;
  }
}

if 1 {

# Create a simple FTS5 table containing 100 documents. Each document 
# contains 10 terms, each of which start with the character "x".
#


do_test 1.0 { create_t1 } {}







do_test 1.1 {
  # Pick out the rowid of the right-most b-tree leaf in the new segment.
  set rowid [db one {
    SELECT max(rowid) FROM t1_data WHERE ((rowid>>31) & 0x0F)==1
  }]
  set L [db one {SELECT length(block) FROM t1_data WHERE rowid = $rowid}]
................................................................................
  UPDATE t2_data SET block = block || 'abcd' WHERE id=1;
  SELECT length(block) FROM t2_data WHERE id=1;
} {6}
do_execsql_test 2.2 {
  INSERT INTO t2 VALUES(rnddoc(10));
  SELECT length(block) FROM t2_data WHERE id=1;
} {2}


#-------------------------------------------------------------------------
# Test that missing leaf pages are recognized as corruption.
#
reset_db
do_test 3.0 { create_t1 } {}

do_execsql_test 3.1 {
  SELECT count(*) FROM t1_data;
} {105}

proc do_3_test {tn} {
  set i 0
  foreach ::rowid [db eval "SELECT rowid FROM t1_data WHERE rowid>100"] {
    incr i
    do_test $tn.$i {
      db eval BEGIN
      db eval {DELETE FROM t1_data WHERE rowid = $::rowid}
      list [
        catch { db eval {SELECT rowid FROM t1 WHERE t1 MATCH 'x*'} } msg
      ] $msg
    } {1 {database disk image is malformed}}
    catch { db eval ROLLBACK }
  }
}

do_3_test 3.2

do_execsql_test 3.3 {
  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
  INSERT INTO t1 SELECT x FROM t1;
  INSERT INTO t1(t1) VALUES('optimize');
} {}

do_3_test 3.4

do_test 3.5 {
  execsql { 
    DELETE FROM t1;
    INSERT INTO t1(t1, rank) VALUES('pgsz', 40);
  }
  for {set i 0} {$i < 1000} {incr i} {
    set rnd [expr int(rand() * 1000)]
    set doc [string repeat "x$rnd " [expr int(rand() * 3) + 1]]
    execsql { INSERT INTO t1(rowid, x) VALUES($i, $doc) }
  }
} {}

do_3_test 3.6

do_test 3.7 {
  execsql {
    INSERT INTO t1(t1, rank) VALUES('pgsz', 40);
    INSERT INTO t1 SELECT x FROM t1;
    INSERT INTO t1(t1) VALUES('optimize');
  }
} {}

do_3_test 3.8

do_test 3.9 {
  execsql { 
    DELETE FROM t1;
    INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
  }
  for {set i 0} {$i < 100} {incr i} {
    set rnd [expr int(rand() * 100)]
    set doc "x[string repeat $rnd 20]"
    execsql { INSERT INTO t1(rowid, x) VALUES($i, $doc) }
  }
} {}

do_3_test 3.10

}

#-------------------------------------------------------------------------
# Test that segments that end unexpectedly are identified as corruption.
#
reset_db
do_test 4.0 {
  execsql { 
    CREATE VIRTUAL TABLE t1 USING fts5(x);
    INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
  }
  for {set i 0} {$i < 100} {incr i} {
    set rnd [expr int(rand() * 100)]
    set doc "x[string repeat $rnd 20]"
    execsql { INSERT INTO t1(rowid, x) VALUES($i, $doc) }
  }
  execsql { INSERT INTO t1(t1) VALUES('optimize') }
} {}

set nErr 0
for {set i 1} {1} {incr i} {
  set struct [db one {SELECT block FROM t1_data WHERE id=10}]
  binary scan $struct c* var
  set end [lindex $var end]
  if {$end<=$i} break
  lset var end [expr $end - $i]
  set struct [binary format c* $var]
  db eval {
    BEGIN;
    UPDATE t1_data SET block = $struct WHERE id=10;
  }
  do_test 4.1.$i {
    incr nErr [catch { db eval { SELECT rowid FROM t1 WHERE t1 MATCH 'x*' } }]
    set {} {}
  } {}
  catch { db eval ROLLBACK }
}
do_test 4.1.x { expr $nErr>45 } 1

#-------------------------------------------------------------------------
#

# The first argument passed to this command must be a binary blob 
# containing an FTS5 leaf page. This command returns a copy of this
# blob, with the pgidx of the leaf page replaced by a single varint
# containing value $iVal.
#
proc rewrite_pgidx {blob iVal} {
  binary scan $blob SS off1 szLeaf
  if {$iVal<0 || $iVal>=128} {
    error "$iVal out of range!"
  } else {
    set pgidx [binary format c $iVal]
  }

  binary format a${szLeaf}a* $blob $pgidx
}

reset_db
do_execsql_test 5.1 {
  CREATE VIRTUAL TABLE x1 USING fts5(x);
  INSERT INTO x1(x1, rank) VALUES('pgsz', 40);
  BEGIN;
  INSERT INTO x1 VALUES('xaaa xabb xccc xcdd xeee xeff xggg xghh xiii xijj');
  INSERT INTO x1 SELECT x FROM x1;
  INSERT INTO x1 SELECT x FROM x1;
  INSERT INTO x1 SELECT x FROM x1;
  INSERT INTO x1 SELECT x FROM x1;
  INSERT INTO x1(x1) VALUES('optimize');
  COMMIT;
}

#db eval { SELECT fts5_decode(id, block) b from x1_data } { puts $b }
#
db func rewrite_pgidx rewrite_pgidx  
set i 0
foreach rowid [db eval {SELECT rowid FROM x1_data WHERE rowid>100}] {
  foreach val {2 100} {
    do_test 5.2.$val.[incr i] {
      catchsql {
        BEGIN;
        UPDATE x1_data SET block=rewrite_pgidx(block, $val) WHERE id=$rowid;
        SELECT rowid FROM x1 WHERE x1 MATCH 'xa*';
        SELECT rowid FROM x1 WHERE x1 MATCH 'xb*';
        SELECT rowid FROM x1 WHERE x1 MATCH 'xc*';
        SELECT rowid FROM x1 WHERE x1 MATCH 'xd*';
        SELECT rowid FROM x1 WHERE x1 MATCH 'xe*';
        SELECT rowid FROM x1 WHERE x1 MATCH 'xf*';
        SELECT rowid FROM x1 WHERE x1 MATCH 'xg*';
        SELECT rowid FROM x1 WHERE x1 MATCH 'xh*';
        SELECT rowid FROM x1 WHERE x1 MATCH 'xi*';
      }
      set {} {}
    } {}
    catch { db eval ROLLBACK }
  }
}


sqlite3_fts5_may_be_corrupt 0
finish_test

Changes to ext/fts5/test/fts5dlidx.test.

21
22
23
24
25
26
27


28
29
30
31
32
33
34
...
123
124
125
126
127
128
129
130























































131
132
  return
}

if { $tcl_platform(wordSize)<8 } {
  finish_test
  return
}



proc do_fb_test {tn sql res} {
  set res2 [lsort -integer -decr $res]
  uplevel [list do_execsql_test $tn.1 $sql $res]
  uplevel [list do_execsql_test $tn.2 "$sql ORDER BY rowid DESC" $res2]
}

................................................................................
  breakpoint
  do_execsql_test $tn.2 {
    SELECT rowid FROM t1 WHERE t1 MATCH 'b AND a' ORDER BY rowid DESC
  } {1}
}

do_dlidx_test2 2.1 [expr 20] [expr 1<<57] [expr (1<<57) + 128]
























































finish_test








>
>







 








>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>


21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
...
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
  return
}

if { $tcl_platform(wordSize)<8 } {
  finish_test
  return
}

if 1 {

proc do_fb_test {tn sql res} {
  set res2 [lsort -integer -decr $res]
  uplevel [list do_execsql_test $tn.1 $sql $res]
  uplevel [list do_execsql_test $tn.2 "$sql ORDER BY rowid DESC" $res2]
}

................................................................................
  breakpoint
  do_execsql_test $tn.2 {
    SELECT rowid FROM t1 WHERE t1 MATCH 'b AND a' ORDER BY rowid DESC
  } {1}
}

do_dlidx_test2 2.1 [expr 20] [expr 1<<57] [expr (1<<57) + 128]

}

#--------------------------------------------------------------------
#
reset_db

set ::vocab [list \
  IteratorpItercurrentlypointstothefirstrowidofadoclist \
  Thereisadoclistindexassociatedwiththefinaltermonthecurrent \
  pageIfthecurrenttermisthelasttermonthepageloadthe \
  doclistindexfromdiskandinitializeaniteratoratpIterpDlidx \
  IteratorpItercurrentlypointstothefirstrowidofadoclist \
  Thereisadoclistindexassociatedwiththefinaltermonthecurrent \
  pageIfthecurrenttermisthelasttermonthepageloadthe \
  doclistindexfromdiskandinitializeaniteratoratpIterpDlidx \
]
proc rnddoc {} {
  global vocab
  set nVocab [llength $vocab]
  set ret [list]
  for {set i 0} {$i < 64} {incr i} {
    lappend ret [lindex $vocab [expr $i % $nVocab]]
  }
  set ret
}
db func rnddoc rnddoc

do_execsql_test 3.1 {
  CREATE VIRTUAL TABLE abc USING fts5(a);
  INSERT INTO abc(abc, rank) VALUES('pgsz', 32);

  INSERT INTO abc VALUES ( rnddoc() );
  INSERT INTO abc VALUES ( rnddoc() );
  INSERT INTO abc VALUES ( rnddoc() );
  INSERT INTO abc VALUES ( rnddoc() );

  INSERT INTO abc SELECT rnddoc() FROM abc;
  INSERT INTO abc SELECT rnddoc() FROM abc;
}

do_execsql_test 3.2 {
  INSERT INTO abc(abc) VALUES('integrity-check');
  INSERT INTO abc(abc) VALUES('optimize');
  INSERT INTO abc(abc) VALUES('integrity-check');
}

set v [lindex $vocab 0]
set i 0
foreach v $vocab {
  do_execsql_test 3.3.[incr i] {
    SELECT rowid FROM abc WHERE abc MATCH $v
  } {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16}
}


finish_test

Changes to ext/fts5/test/fts5fault7.test.

10
11
12
13
14
15
16
17
18
19
20
21
22
23


24
25
26
27
28
29
30
..
36
37
38
39
40
41
42
43












































44
45
#*************************************************************************
#
# This file is focused on OOM errors.
#

source [file join [file dirname [info script]] fts5_common.tcl]
source $testdir/malloc_common.tcl
set testprefix fts5fault2

# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts5 {
  finish_test
  return
}



#-------------------------------------------------------------------------
# Test fault-injection on a query that uses xColumnSize() on columnsize=0
# table.
#
do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize=0);
................................................................................

fts5_aux_test_functions db
do_faultsim_test 1 -faults oom* -body {
  execsql { SELECT fts5_test_columnsize(t1) FROM t1 WHERE t1 MATCH 'b' }
} -test {
  faultsim_test_result {0 {7 4 10}} {1 SQLITE_NOMEM}
}













































finish_test








|






>
>







 








>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>


10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
..
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
#*************************************************************************
#
# This file is focused on OOM errors.
#

source [file join [file dirname [info script]] fts5_common.tcl]
source $testdir/malloc_common.tcl
set testprefix fts5fault7

# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts5 {
  finish_test
  return
}

if 1 {

#-------------------------------------------------------------------------
# Test fault-injection on a query that uses xColumnSize() on columnsize=0
# table.
#
do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize=0);
................................................................................

fts5_aux_test_functions db
do_faultsim_test 1 -faults oom* -body {
  execsql { SELECT fts5_test_columnsize(t1) FROM t1 WHERE t1 MATCH 'b' }
} -test {
  faultsim_test_result {0 {7 4 10}} {1 SQLITE_NOMEM}
}

}

#-------------------------------------------------------------------------
# Test fault-injection when a segment is promoted.
#
do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE t2 USING fts5(a);
  INSERT INTO t2(t2, rank) VALUES('automerge', 0);
  INSERT INTO t2(t2, rank) VALUES('crisismerge', 4);
  INSERT INTO t2(t2, rank) VALUES('pgsz', 40);

  INSERT INTO t2 VALUES('a b c');
  INSERT INTO t2 VALUES('d e f');
  INSERT INTO t2 VALUES('f e d');
  INSERT INTO t2 VALUES('c b a');

  INSERT INTO t2 VALUES('a b c');
  INSERT INTO t2 VALUES('d e f');
  INSERT INTO t2 VALUES('f e d');
  INSERT INTO t2 VALUES('c b a');
} {}

faultsim_save_and_close
do_faultsim_test 1 -faults oom-t* -prep {
  faultsim_restore_and_reopen
  db eval {
    BEGIN;
    INSERT INTO t2 VALUES('c d c g g f');
    INSERT INTO t2 VALUES('c d g b f d');
    INSERT INTO t2 VALUES('c c f d e d');
    INSERT INTO t2 VALUES('e a f c e f');
    INSERT INTO t2 VALUES('c g f b b d');
    INSERT INTO t2 VALUES('d a g a b b');
    INSERT INTO t2 VALUES('e f a b c e');
    INSERT INTO t2 VALUES('e c a g c d');
    INSERT INTO t2 VALUES('g b d d e b');
    INSERT INTO t2 VALUES('e a d a e d');
  }
} -body {
  db eval COMMIT
} -test {
  faultsim_test_result {0 {}}
}

finish_test