SQLite

Check-in [09dabb3b9e]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add further tests for fts5 backend.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | fts5
Files: files | file ages | folders
SHA1: 09dabb3b9e140eec6cfda83bcb86b6b9f5cf54b6
User & Date: dan 2015-01-21 16:10:59.815
Context
2015-01-21
17:20
Merge trunk changes with this branch. (check-in: f8699a1a3b user: dan tags: fts5)
16:10
Add further tests for fts5 backend. (check-in: 09dabb3b9e user: dan tags: fts5)
06:36
Merge latest trunk changes with this branch. (check-in: b3348b1e07 user: dan tags: fts5)
Changes
Unified Diff Ignore Whitespace Patch
Changes to ext/fts5/fts5_index.c.
260
261
262
263
264
265
266







267
268
269
270
271
272
273
static int fts5Corrupt() { return SQLITE_CORRUPT_VTAB; }
# define FTS5_CORRUPT fts5Corrupt()
#else
# define FTS5_CORRUPT SQLITE_CORRUPT_VTAB
#endif









typedef struct Fts5BtreeIter Fts5BtreeIter;
typedef struct Fts5BtreeIterLevel Fts5BtreeIterLevel;
typedef struct Fts5ChunkIter Fts5ChunkIter;
typedef struct Fts5Data Fts5Data;
typedef struct Fts5DlidxIter Fts5DlidxIter;
typedef struct Fts5MultiSegIter Fts5MultiSegIter;
typedef struct Fts5NodeIter Fts5NodeIter;







>
>
>
>
>
>
>







260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
static int fts5Corrupt() { return SQLITE_CORRUPT_VTAB; }
# define FTS5_CORRUPT fts5Corrupt()
#else
# define FTS5_CORRUPT SQLITE_CORRUPT_VTAB
#endif


/*
** Each time a blob is read from the %_data table, it is padded with this
** many zero bytes. This makes it easier to decode the various record formats
** without overreading if the records are corrupt.
*/
#define FTS5_DATA_ZERO_PADDING 8

typedef struct Fts5BtreeIter Fts5BtreeIter;
typedef struct Fts5BtreeIterLevel Fts5BtreeIterLevel;
typedef struct Fts5ChunkIter Fts5ChunkIter;
typedef struct Fts5Data Fts5Data;
typedef struct Fts5DlidxIter Fts5DlidxIter;
typedef struct Fts5MultiSegIter Fts5MultiSegIter;
typedef struct Fts5NodeIter Fts5NodeIter;
712
713
714
715
716
717
718

719
720
721
722
723
724
725
726
      int nByte = sqlite3_blob_bytes(p->pReader);
      if( pBuf ){
        fts5BufferZero(pBuf);
        fts5BufferGrow(&rc, pBuf, nByte);
        aOut = pBuf->p;
        pBuf->n = nByte;
      }else{

        pRet = (Fts5Data*)sqlite3Fts5MallocZero(&rc, nByte+sizeof(Fts5Data));
        if( pRet ){
          pRet->n = nByte;
          aOut = pRet->p = (u8*)&pRet[1];
          pRet->nRef = 1;
        }
      }








>
|







719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
      int nByte = sqlite3_blob_bytes(p->pReader);
      if( pBuf ){
        fts5BufferZero(pBuf);
        fts5BufferGrow(&rc, pBuf, nByte);
        aOut = pBuf->p;
        pBuf->n = nByte;
      }else{
        int nSpace = nByte + FTS5_DATA_ZERO_PADDING;
        pRet = (Fts5Data*)sqlite3Fts5MallocZero(&rc, nSpace+sizeof(Fts5Data));
        if( pRet ){
          pRet->n = nByte;
          aOut = pRet->p = (u8*)&pRet[1];
          pRet->nRef = 1;
        }
      }

4535
4536
4537
4538
4539
4540
4541
4542

4543
4544

4545
4546
4547
4548
4549
4550





4551
4552
4553
4554
4555
4556
4557
static void fts5DecodeFunction(
  sqlite3_context *pCtx,          /* Function call context */
  int nArg,                       /* Number of args (always 2) */
  sqlite3_value **apVal           /* Function arguments */
){
  i64 iRowid;                     /* Rowid for record being decoded */
  int iIdx,iSegid,iHeight,iPgno;  /* Rowid components */
  const u8 *a; int n;             /* Record to decode */

  Fts5Buffer s;                   /* Build up text to return here */
  int rc = SQLITE_OK;             /* Return code */


  assert( nArg==2 );
  memset(&s, 0, sizeof(Fts5Buffer));
  iRowid = sqlite3_value_int64(apVal[0]);
  n = sqlite3_value_bytes(apVal[1]);
  a = sqlite3_value_blob(apVal[1]);





  fts5DecodeRowid(iRowid, &iIdx, &iSegid, &iHeight, &iPgno);

  fts5DebugRowid(&rc, &s, iRowid);
  if( iHeight==FTS5_SEGMENT_MAX_HEIGHT ){
    int i = 0;
    i64 iPrev;
    if( n>0 ){







|
>


>





|
>
>
>
>
>







4543
4544
4545
4546
4547
4548
4549
4550
4551
4552
4553
4554
4555
4556
4557
4558
4559
4560
4561
4562
4563
4564
4565
4566
4567
4568
4569
4570
4571
4572
static void fts5DecodeFunction(
  sqlite3_context *pCtx,          /* Function call context */
  int nArg,                       /* Number of args (always 2) */
  sqlite3_value **apVal           /* Function arguments */
){
  i64 iRowid;                     /* Rowid for record being decoded */
  int iIdx,iSegid,iHeight,iPgno;  /* Rowid components */
  const u8 *aBlob; int n;         /* Record to decode */
  u8 *a = 0;
  Fts5Buffer s;                   /* Build up text to return here */
  int rc = SQLITE_OK;             /* Return code */
  int nSpace = 0;

  assert( nArg==2 );
  memset(&s, 0, sizeof(Fts5Buffer));
  iRowid = sqlite3_value_int64(apVal[0]);
  n = sqlite3_value_bytes(apVal[1]);
  aBlob = sqlite3_value_blob(apVal[1]);

  nSpace = n + FTS5_DATA_ZERO_PADDING;
  a = (u8*)sqlite3Fts5MallocZero(&rc, nSpace);
  if( a==0 ) goto decode_out;
  memcpy(a, aBlob, n);
  fts5DecodeRowid(iRowid, &iIdx, &iSegid, &iHeight, &iPgno);

  fts5DebugRowid(&rc, &s, iRowid);
  if( iHeight==FTS5_SEGMENT_MAX_HEIGHT ){
    int i = 0;
    i64 iPrev;
    if( n>0 ){
4583
4584
4585
4586
4587
4588
4589

4590
4591




4592
4593
4594
4595
4596
4597
4598

    if( iHeight==0 ){
      int iTermOff = 0;
      int iRowidOff = 0;
      int iOff;
      int nKeep = 0;


      iRowidOff = fts5GetU16(&a[0]);
      iTermOff = fts5GetU16(&a[2]);





      if( iRowidOff ){
        iOff = iRowidOff;
      }else if( iTermOff ){
        iOff = iTermOff;
      }else{
        iOff = n;







>
|
|
>
>
>
>







4598
4599
4600
4601
4602
4603
4604
4605
4606
4607
4608
4609
4610
4611
4612
4613
4614
4615
4616
4617
4618

    if( iHeight==0 ){
      int iTermOff = 0;
      int iRowidOff = 0;
      int iOff;
      int nKeep = 0;

      if( n>=4 ){
        iRowidOff = fts5GetU16(&a[0]);
        iTermOff = fts5GetU16(&a[2]);
      }else{
        sqlite3Fts5BufferSet(&rc, &s, 8, (const u8*)"corrupt");
        goto decode_out;
      }

      if( iRowidOff ){
        iOff = iRowidOff;
      }else if( iTermOff ){
        iOff = iTermOff;
      }else{
        iOff = n;
4638
4639
4640
4641
4642
4643
4644


4645
4646
4647
4648
4649
4650
4651
          );
        }
      }
      fts5NodeIterFree(&ss);
    }
  }
  


  if( rc==SQLITE_OK ){
    sqlite3_result_text(pCtx, (const char*)s.p, s.n, SQLITE_TRANSIENT);
  }else{
    sqlite3_result_error_code(pCtx, rc);
  }
  fts5BufferFree(&s);
}







>
>







4658
4659
4660
4661
4662
4663
4664
4665
4666
4667
4668
4669
4670
4671
4672
4673
          );
        }
      }
      fts5NodeIterFree(&ss);
    }
  }
  
 decode_out:
  sqlite3_free(a);
  if( rc==SQLITE_OK ){
    sqlite3_result_text(pCtx, (const char*)s.p, s.n, SQLITE_TRANSIENT);
  }else{
    sqlite3_result_error_code(pCtx, rc);
  }
  fts5BufferFree(&s);
}
4689
4690
4691
4692
4693
4694
4695
4696
4697
4698
4699
4700
4701
4702
4703
        idx = sqlite3_value_int(apVal[1]);
        iRowid = FTS5_SEGMENT_ROWID(idx, 1, 0, 0);
        sqlite3_result_int64(pCtx, iRowid);
      }
    }else {
      sqlite3_result_error(pCtx, 
        "first arg to fts5_rowid() must be 'segment' "
        "or 'start-of-index' ..."
        , -1
      );
    }
  }
}

/*







|







4711
4712
4713
4714
4715
4716
4717
4718
4719
4720
4721
4722
4723
4724
4725
        idx = sqlite3_value_int(apVal[1]);
        iRowid = FTS5_SEGMENT_ROWID(idx, 1, 0, 0);
        sqlite3_result_int64(pCtx, iRowid);
      }
    }else {
      sqlite3_result_error(pCtx, 
        "first arg to fts5_rowid() must be 'segment' "
        "or 'start-of-index'"
        , -1
      );
    }
  }
}

/*
Changes to ext/fts5/test/fts5fault1.test.
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
#   1: CREATE VIRTUAL TABLE
#   2: INSERT statement
#   3: DELETE statement
#   4: MATCH expressions
#
#

if 1 {

faultsim_save_and_close
do_faultsim_test 1 -prep {
  faultsim_restore_and_reopen
} -body {
  execsql { CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix='1, 2, 3') }
} -test {







|







27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
#   1: CREATE VIRTUAL TABLE
#   2: INSERT statement
#   3: DELETE statement
#   4: MATCH expressions
#
#

if 0 {

faultsim_save_and_close
do_faultsim_test 1 -prep {
  faultsim_restore_and_reopen
} -body {
  execsql { CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix='1, 2, 3') }
} -test {
103
104
105
106
107
108
109



110
111
112
113
114
115
116
  } -body "
    execsql { SELECT rowid FROM t2 WHERE t2 MATCH '$expr' }
  " -test "
    faultsim_test_result {[list 0 $res]}
  "
}




#-------------------------------------------------------------------------
# The following tests use a larger database populated with random data.
#
# The database page size is set to 512 bytes and the FTS5 page size left
# at the default 1000 bytes. This means that reading a node may require
# pulling an overflow page from disk, which is an extra opportunity for
# an error to occur.







>
>
>







103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
  } -body "
    execsql { SELECT rowid FROM t2 WHERE t2 MATCH '$expr' }
  " -test "
    faultsim_test_result {[list 0 $res]}
  "
}


}

#-------------------------------------------------------------------------
# The following tests use a larger database populated with random data.
#
# The database page size is set to 512 bytes and the FTS5 page size left
# at the default 1000 bytes. This means that reading a node may require
# pulling an overflow page from disk, which is an extra opportunity for
# an error to occur.
172
173
174
175
176
177
178



179






























180
181
182
183
184
185
186
  sqlite3 db test.db
} -body {
  execsql { INSERT INTO x1 VALUES('a b c d', 'e f g h') }
} -test {
  faultsim_test_result [list 0 {}]
}




}































#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 6.0 {
  CREATE VIRTUAL TABLE x1 USING fts5(x);
  INSERT INTO x1(x1, rank) VALUES('automerge', 0);







>
>
>
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
  sqlite3 db test.db
} -body {
  execsql { INSERT INTO x1 VALUES('a b c d', 'e f g h') }
} -test {
  faultsim_test_result [list 0 {}]
}

do_faultsim_test 5.5.1 -faults oom* -body {
  execsql { 
    SELECT count(fts5_decode(rowid, block)) FROM x1_data WHERE rowid=1
  }
} -test {
  faultsim_test_result [list 0 1]
}
do_faultsim_test 5.5.2 -faults oom* -body {
  execsql { 
    SELECT count(fts5_decode(rowid, block)) FROM x1_data WHERE rowid=10
  }
} -test {
  faultsim_test_result [list 0 1]
}
do_faultsim_test 5.5.3 -faults oom* -body {
  execsql { 
    SELECT count(fts5_decode(rowid, block)) FROM x1_data WHERE rowid = (
      SELECT min(rowid) FROM x1_data WHERE rowid>20
    )
  }
} -test {
  faultsim_test_result [list 0 1]
}
do_faultsim_test 5.5.4 -faults oom* -body {
  execsql { 
    SELECT count(fts5_decode(rowid, block)) FROM x1_data WHERE rowid = (
      SELECT max(rowid) FROM x1_data 
    )
  }
} -test {
  faultsim_test_result [list 0 1]
}

finish_test

#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 6.0 {
  CREATE VIRTUAL TABLE x1 USING fts5(x);
  INSERT INTO x1(x1, rank) VALUES('automerge', 0);
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219










220
221
222
  INSERT INTO x1 VALUES('a b c'); -- 15

  SELECT count(*) FROM x1_data;
} {17}

faultsim_save_and_close

do_faultsim_test 6.1 -faults oom-tr* -prep {
  faultsim_restore_and_reopen
} -body {
  execsql { INSERT INTO x1 VALUES('d e f') }
} -test {
  faultsim_test_result [list 0 {}]
  if {$testrc==0} {
    set nCnt [db one {SELECT count(*) FROM x1_data}]
    if {$nCnt!=3} { error "expected 3 entries but there are $nCnt" }
  }
}











finish_test








|










>
>
>
>
>
>
>
>
>
>



238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
  INSERT INTO x1 VALUES('a b c'); -- 15

  SELECT count(*) FROM x1_data;
} {17}

faultsim_save_and_close

do_faultsim_test 6.1 -faults oom* -prep {
  faultsim_restore_and_reopen
} -body {
  execsql { INSERT INTO x1 VALUES('d e f') }
} -test {
  faultsim_test_result [list 0 {}]
  if {$testrc==0} {
    set nCnt [db one {SELECT count(*) FROM x1_data}]
    if {$nCnt!=3} { error "expected 3 entries but there are $nCnt" }
  }
}

#-------------------------------------------------------------------------
do_faultsim_test 7.0 -faults oom* -prep {
  catch { db close }
} -body {
  sqlite3 db test.db
} -test {
  faultsim_test_result [list 0 {}] [list 1 {}]
}


finish_test

Added ext/fts5/test/fts5rowid.test.
























































































































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
# 2014 Dec 20
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#
# Tests of the scalar fts5_rowid() and fts5_decode() functions.
#

source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5rowid

do_catchsql_test 1.1 {
  SELECT fts5_rowid()
} {1 {should be: fts5_rowid(subject, ....)}}

do_catchsql_test 1.2 {
  SELECT fts5_rowid('segment')
} {1 {should be: fts5_rowid('segment', idx, segid, height, pgno))}}

do_execsql_test 1.3 {
  SELECT fts5_rowid('segment', 1, 1, 1, 1)
} {4503670494330881}

do_catchsql_test 1.4 {
  SELECT fts5_rowid('start-of-index');
} {1 {should be: fts5_rowid('start-of-index', idx)}}

do_execsql_test 1.5 {
  SELECT fts5_rowid('start-of-index', 1);
} {4503668346847232}

do_catchsql_test 1.4 {
  SELECT fts5_rowid('nosucharg');
} {1 {first arg to fts5_rowid() must be 'segment' or 'start-of-index'}} 


#-------------------------------------------------------------------------
# Tests of the fts5_decode() function.
#
reset_db
do_execsql_test 2.1 { 
  CREATE VIRTUAL TABLE x1 USING fts5(a, b);
  INSERT INTO x1(x1, rank) VALUES('pgsz', 32);
} {}

proc rnddoc {n} {
  set map [list 0 a  1 b  2 c  3 d  4 e  5 f  6 g  7 h  8 i  9 j]
  set doc [list]
  for {set i 0} {$i < $n} {incr i} {
    lappend doc [string map $map [format %.3d [expr int(rand()*100)]]]
  }
  set doc
}
db func rnddoc rnddoc

do_execsql_test 2.2 {
  WITH r(a, b) AS (
    SELECT rnddoc(6), rnddoc(6) UNION ALL
    SELECT rnddoc(6), rnddoc(6) FROM r
  )
  INSERT INTO x1 SELECT * FROM r LIMIT 10000;
}

set res [db one {SELECT count(*) FROM x1_data}]
do_execsql_test 2.3 {
  SELECT count(fts5_decode(rowid, block)) FROM x1_data;
} $res
do_execsql_test 2.4 {
  UPDATE x1_data SET block = X'';
  SELECT count(fts5_decode(rowid, block)) FROM x1_data;
} $res

do_execsql_test 2.5 {
  INSERT INTO x1(x1, rank) VALUES('pgsz', 1024);
  INSERT INTO x1(x1) VALUES('rebuild');
}

set res [db one {SELECT count(*) FROM x1_data}]
do_execsql_test 2.6 {
  SELECT count(fts5_decode(rowid, block)) FROM x1_data;
} $res
do_execsql_test 2.7 {
  UPDATE x1_data SET block = X'';
  SELECT count(fts5_decode(rowid, block)) FROM x1_data;
} $res

#-------------------------------------------------------------------------
# Tests with very large tokens.
#
set strlist [list \
  "[string repeat x 400]"                       \
  "[string repeat x 300][string repeat w 100]"  \
  "[string repeat x 300][string repeat y 100]"  \
  "[string repeat x 300][string repeat z 600]"  \
]
do_test 3.0 {
  execsql {
    BEGIN;
    CREATE VIRTUAL TABLE x2 USING fts5(a);
  }
  foreach str $strlist { execsql { INSERT INTO x2 VALUES($str) } }
  execsql COMMIT
} {}

for {set tn 0} {$tn<[llength $strlist]} {incr tn} {
  set str [lindex $strlist $tn]
  do_execsql_test 3.1.$tn {
    SELECT rowid FROM x2 WHERE x2 MATCH $str
  } [expr $tn+1]
}

set res [db one {SELECT count(*) FROM x2_data}]
do_execsql_test 3.2 {
  SELECT count(fts5_decode(rowid, block)) FROM x2_data;
} $res

finish_test