/ Check-in [8d05cfd4]
Login
Home Timeline Branches Tags Tickets Wiki More...

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix an fts5 problem involving detail=none, "ORDER BY rowid DESC" and deleted items. Also add tests to verify that the documented operator precedences are correct.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts5-offsets
Files: files | file ages | folders
SHA1: 8d05cfd48db1462dfd3efd502d53bd33214403fb
User & Date: dan 2016-01-06 18:53:57
Context
2016-01-06
19:35
Merge latest trunk changes into this branch. check-in: e7dcd013 user: dan tags: fts5-offsets
18:53
Fix an fts5 problem involving detail=none, "ORDER BY rowid DESC" and deleted items. Also add tests to verify that the documented operator precedences are correct. check-in: 8d05cfd4 user: dan tags: fts5-offsets
10:17
Fix a problem with the xPhraseFirstColumn() API in detail=col mode. Remove some unused test code from fts5ac.test. check-in: a4e55f73 user: dan tags: fts5-offsets
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5_index.c. 

1760
1761
1762
1763
1764
1765
1766
1767


1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781

        u8 *a = pIter->pLeaf->p;
        int iOff;
        int nPos;
        int bDummy;
        i64 iDelta;

        pIter->iRowidOffset--;
        pIter->iLeafOffset = iOff = pIter->aRowidOffset[pIter->iRowidOffset];


        if( p->pConfig->eDetail!=FTS5_DETAIL_NONE ){
          iOff += fts5GetPoslistSize(&a[iOff], &nPos, &bDummy);
          iOff += nPos;
        }
        fts5GetVarint(&a[iOff], (u64*)&iDelta);
        pIter->iRowid -= iDelta;
        fts5SegIterLoadNPos(p, pIter);
      }else{
        fts5SegIterReverseNewPage(p, pIter);
      }
    }else{
      Fts5Data *pLeaf = pIter->pLeaf;
      int iOff;
      int bNewTerm = 0;








|
>
>

<
|



<








1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770

1771
1772
1773
1774

1775
1776
1777
1778
1779
1780
1781

        u8 *a = pIter->pLeaf->p;
        int iOff;
        int nPos;
        int bDummy;
        i64 iDelta;

        pIter->iRowidOffset--;
        pIter->iLeafOffset = pIter->aRowidOffset[pIter->iRowidOffset];
        fts5SegIterLoadNPos(p, pIter);
        iOff = pIter->iLeafOffset;
        if( p->pConfig->eDetail!=FTS5_DETAIL_NONE ){

          iOff += pIter->nPos;
        }
        fts5GetVarint(&a[iOff], (u64*)&iDelta);
        pIter->iRowid -= iDelta;

      }else{
        fts5SegIterReverseNewPage(p, pIter);
      }
    }else{
      Fts5Data *pLeaf = pIter->pLeaf;
      int iOff;
      int bNewTerm = 0;

Changes to ext/fts5/test/fts5dlidx.test. 

22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
..
73
74
75
76
77
78
79

80
81
82
83

84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
...
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
...
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
...
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
...
187
188
189
190
191
192
193



194
195
196
197

}

if { $tcl_platform(wordSize)<8 } {
  finish_test
  return
}

if 1 {

proc do_fb_test {tn sql res} {
  set res2 [lsort -integer -decr $res]
  uplevel [list do_execsql_test $tn.1 $sql $res]
  uplevel [list do_execsql_test $tn.2 "$sql ORDER BY rowid DESC" $res2]
}

# This test populates the FTS5 table containing $nEntry entries. Rows are 
# numbered from 0 to ($nEntry-1). The rowid for row $i is:
#
#   ($iFirst + $i*$nStep)
#
# Each document is of the form "a b c a b c a b c...". If the row number ($i)
# is an integer multiple of $spc1, then an "x" token is appended to the
# document. If it is *also* a multiple of $spc2, a "y" token is also appended.
................................................................................
  
  do_fb_test $tn.3.1 { SELECT rowid FROM t1 WHERE t1 MATCH 'a AND x' } $xdoc
  do_fb_test $tn.3.2 { SELECT rowid FROM t1 WHERE t1 MATCH 'x AND a' } $xdoc
  
  do_fb_test $tn.4.1 { SELECT rowid FROM t1 WHERE t1 MATCH 'a AND y' } $ydoc
  do_fb_test $tn.4.2 { SELECT rowid FROM t1 WHERE t1 MATCH 'y AND a' } $ydoc
  

  do_fb_test $tn.5.1 { 
    SELECT rowid FROM t1 WHERE t1 MATCH 'a + b + c + x' } $xdoc
  do_fb_test $tn.5.2 { 
    SELECT rowid FROM t1 WHERE t1 MATCH 'b + c + x + y' } $ydoc

}


foreach {tn pgsz} {
  1 32
  2 200
} {
  do_execsql_test $tn.0 { 
    DROP TABLE IF EXISTS t1;
    CREATE VIRTUAL TABLE t1 USING fts5(x);
    INSERT INTO t1(t1, rank) VALUES('pgsz', $pgsz);
  }

  do_dlidx_test1 1.$tn.1     10 100 10000 0 1000
  do_dlidx_test1 1.$tn.2     10 10  10000 0 128
  do_dlidx_test1 1.$tn.3     10 10  66    0 36028797018963970
  do_dlidx_test1 1.$tn.4     10 10  50    0 150000000000000000
................................................................................
}

proc do_dlidx_test2 {tn nEntry iFirst nStep} {
  set str [string repeat "a " 500]
  execsql {
    BEGIN;
    DROP TABLE IF EXISTS t1;
    CREATE VIRTUAL TABLE t1 USING fts5(x);
    INSERT INTO t1(t1, rank) VALUES('pgsz', 64);
    INSERT INTO t1 VALUES('b a');

    WITH iii(ii, i) AS (
      SELECT 1,     $iFirst UNION ALL 
      SELECT ii+1, i+$nStep FROM iii WHERE ii<$nEntry
    )
................................................................................
  do_execsql_test $tn.2 {
    SELECT rowid FROM t1 WHERE t1 MATCH 'b AND a' ORDER BY rowid DESC
  } {1}
}

do_dlidx_test2 2.1 [expr 20] [expr 1<<57] [expr (1<<57) + 128]

}

#--------------------------------------------------------------------
#
reset_db

set ::vocab [list \
  IteratorpItercurrentlypointstothefirstrowidofadoclist \
  Thereisadoclistindexassociatedwiththefinaltermonthecurrent \
................................................................................
    lappend ret [lindex $vocab [expr $i % $nVocab]]
  }
  set ret
}
db func rnddoc rnddoc

do_execsql_test 3.1 {
  CREATE VIRTUAL TABLE abc USING fts5(a);
  INSERT INTO abc(abc, rank) VALUES('pgsz', 32);

  INSERT INTO abc VALUES ( rnddoc() );
  INSERT INTO abc VALUES ( rnddoc() );
  INSERT INTO abc VALUES ( rnddoc() );
  INSERT INTO abc VALUES ( rnddoc() );

................................................................................
set v [lindex $vocab 0]
set i 0
foreach v $vocab {
  do_execsql_test 3.3.[incr i] {
    SELECT rowid FROM abc WHERE abc MATCH $v
  } {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16}
}





finish_test









|







|







 







>
|
|
|
|
>









|







 







|







 







<
<







 







|







 







>
>
>





22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
..
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
...
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
...
128
129
130
131
132
133
134


135
136
137
138
139
140
141
...
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
...
187
188
189
190
191
192
193
194
195
196
197
198
199
200

}

if { $tcl_platform(wordSize)<8 } {
  finish_test
  return
}

foreach_detail_mode $testprefix {

proc do_fb_test {tn sql res} {
  set res2 [lsort -integer -decr $res]
  uplevel [list do_execsql_test $tn.1 $sql $res]
  uplevel [list do_execsql_test $tn.2 "$sql ORDER BY rowid DESC" $res2]
}

# This test populates the FTS5 table with $nEntry entries. Rows are 
# numbered from 0 to ($nEntry-1). The rowid for row $i is:
#
#   ($iFirst + $i*$nStep)
#
# Each document is of the form "a b c a b c a b c...". If the row number ($i)
# is an integer multiple of $spc1, then an "x" token is appended to the
# document. If it is *also* a multiple of $spc2, a "y" token is also appended.
................................................................................
  
  do_fb_test $tn.3.1 { SELECT rowid FROM t1 WHERE t1 MATCH 'a AND x' } $xdoc
  do_fb_test $tn.3.2 { SELECT rowid FROM t1 WHERE t1 MATCH 'x AND a' } $xdoc
  
  do_fb_test $tn.4.1 { SELECT rowid FROM t1 WHERE t1 MATCH 'a AND y' } $ydoc
  do_fb_test $tn.4.2 { SELECT rowid FROM t1 WHERE t1 MATCH 'y AND a' } $ydoc
  
  if {[detail_is_full]} {
    do_fb_test $tn.5.1 { 
      SELECT rowid FROM t1 WHERE t1 MATCH 'a + b + c + x' } $xdoc
    do_fb_test $tn.5.2 { 
      SELECT rowid FROM t1 WHERE t1 MATCH 'b + c + x + y' } $ydoc
  }
}


foreach {tn pgsz} {
  1 32
  2 200
} {
  do_execsql_test $tn.0 { 
    DROP TABLE IF EXISTS t1;
    CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%);
    INSERT INTO t1(t1, rank) VALUES('pgsz', $pgsz);
  }

  do_dlidx_test1 1.$tn.1     10 100 10000 0 1000
  do_dlidx_test1 1.$tn.2     10 10  10000 0 128
  do_dlidx_test1 1.$tn.3     10 10  66    0 36028797018963970
  do_dlidx_test1 1.$tn.4     10 10  50    0 150000000000000000
................................................................................
}

proc do_dlidx_test2 {tn nEntry iFirst nStep} {
  set str [string repeat "a " 500]
  execsql {
    BEGIN;
    DROP TABLE IF EXISTS t1;
    CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%);
    INSERT INTO t1(t1, rank) VALUES('pgsz', 64);
    INSERT INTO t1 VALUES('b a');

    WITH iii(ii, i) AS (
      SELECT 1,     $iFirst UNION ALL 
      SELECT ii+1, i+$nStep FROM iii WHERE ii<$nEntry
    )
................................................................................
  do_execsql_test $tn.2 {
    SELECT rowid FROM t1 WHERE t1 MATCH 'b AND a' ORDER BY rowid DESC
  } {1}
}

do_dlidx_test2 2.1 [expr 20] [expr 1<<57] [expr (1<<57) + 128]



#--------------------------------------------------------------------
#
reset_db

set ::vocab [list \
  IteratorpItercurrentlypointstothefirstrowidofadoclist \
  Thereisadoclistindexassociatedwiththefinaltermonthecurrent \
................................................................................
    lappend ret [lindex $vocab [expr $i % $nVocab]]
  }
  set ret
}
db func rnddoc rnddoc

do_execsql_test 3.1 {
  CREATE VIRTUAL TABLE abc USING fts5(a, detail=%DETAIL%);
  INSERT INTO abc(abc, rank) VALUES('pgsz', 32);

  INSERT INTO abc VALUES ( rnddoc() );
  INSERT INTO abc VALUES ( rnddoc() );
  INSERT INTO abc VALUES ( rnddoc() );
  INSERT INTO abc VALUES ( rnddoc() );

................................................................................
set v [lindex $vocab 0]
set i 0
foreach v $vocab {
  do_execsql_test 3.3.[incr i] {
    SELECT rowid FROM abc WHERE abc MATCH $v
  } {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16}
}

} ;# foreach_detail_mode



finish_test

Changes to ext/fts5/test/fts5eb.test. 

38
39
40
41
42
43
44









45
46
47
48
49
50
51

  6  {abc AND ""}                     {"abc"}
  7  {"" OR abc}                      {"abc"}
  8  {"" NOT abc}                     {"abc"}
  9  {"" AND abc}                     {"abc"}
  10 {abc + "" + def}                 {"abc" + "def"}
  11 {abc "" def}                     {"abc" AND "def"}
  12 {r+e OR w}                       {"r" + "e" OR "w"}









} {
  do_execsql_test 1.$tn {SELECT fts5_expr($expr)} [list $res]
}

do_catchsql_test 2.1 {
  SELECT fts5_expr()
} {1 {wrong number of arguments to function fts5_expr}}








>
>
>
>
>
>
>
>
>








38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60

  6  {abc AND ""}                     {"abc"}
  7  {"" OR abc}                      {"abc"}
  8  {"" NOT abc}                     {"abc"}
  9  {"" AND abc}                     {"abc"}
  10 {abc + "" + def}                 {"abc" + "def"}
  11 {abc "" def}                     {"abc" AND "def"}
  12 {r+e OR w}                       {"r" + "e" OR "w"}

  13 {a AND b NOT c}                  {"a" AND ("b" NOT "c")}
  14 {a OR b NOT c}                   {"a" OR ("b" NOT "c")}
  15 {a NOT b AND c}                  {("a" NOT "b") AND "c"}
  16 {a NOT b OR c}                   {("a" NOT "b") OR "c"}

  17 {a AND b OR c}                   {("a" AND "b") OR "c"}
  18 {a OR b AND c}                   {"a" OR ("b" AND "c")}

} {
  do_execsql_test 1.$tn {SELECT fts5_expr($expr)} [list $res]
}

do_catchsql_test 2.1 {
  SELECT fts5_expr()
} {1 {wrong number of arguments to function fts5_expr}}

Changes to ext/fts5/test/fts5simple2.test. 

250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271

































272
273

  15  {a}
} {
  do_execsql_test 13.1.$rowid {
    INSERT INTO t1(rowid, a) VALUES($rowid, $a);
  }
}

}

#-------------------------------------------------------------------------
#
reset_db
fts5_aux_test_functions db
do_execsql_test 14.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
  INSERT INTO t1 VALUES('a b c d');
} {}

do_execsql_test 14.1 {
  SELECT fts5_test_poslist(t1) FROM t1('b') ORDER BY rank;
} {0.0.1}


































finish_test









<
<













>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>



250
251
252
253
254
255
256


257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304

  15  {a}
} {
  do_execsql_test 13.1.$rowid {
    INSERT INTO t1(rowid, a) VALUES($rowid, $a);
  }
}



#-------------------------------------------------------------------------
#
reset_db
fts5_aux_test_functions db
do_execsql_test 14.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
  INSERT INTO t1 VALUES('a b c d');
} {}

do_execsql_test 14.1 {
  SELECT fts5_test_poslist(t1) FROM t1('b') ORDER BY rank;
} {0.0.1}

}

#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 15.1 {
  CREATE VIRTUAL TABLE t1 USING fts5(x, detail=none);
  BEGIN;
    INSERT INTO t1(rowid, x) VALUES(1, 'sqlite');
    INSERT INTO t1(rowid, x) VALUES(2, 'sqlite'); 
  COMMIT;
} {}

do_test 15.1 {
  execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
} {}

do_test 15.2 {
  execsql { DELETE FROM t1 }
} {}

do_execsql_test 15.3.1 {
  SELECT rowid FROM t1('sqlite');
} {}

do_execsql_test 15.3.2 {
  SELECT rowid FROM t1('sqlite') ORDER BY rowid DESC;
} {}

do_test 15.4 {
  execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
} {}
  
finish_test

This page was generated in about 0.014s by Fossil 2.11 [eee161e823] 2020-01-22 19:08:54