SQLite

Check-in [b1df00f3f1]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add extra tests for fts with a non-zero languageid_bits setting. Fix querying by docid with the same.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | fts-languageid-bits
Files: files | file ages | folders
SHA1: b1df00f3f1eecffbe6c56cdcbb922922314604a0
User & Date: dan 2013-06-20 16:22:32.207
Context
2013-06-20
18:32
Fix some issues related to ORDER BY and fts tables with a non-zero languageid_bits setting. (check-in: 81527768ef user: dan tags: fts-languageid-bits)
16:22
Add extra tests for fts with a non-zero languageid_bits setting. Fix querying by docid with the same. (check-in: b1df00f3f1 user: dan tags: fts-languageid-bits)
11:48
Add tests (and a fix) for large and small fts docid values with various languageid_bits settings. (check-in: 8dc261b765 user: dan tags: fts-languageid-bits)
Changes
Unified Diff Ignore Whitespace Patch
Changes to ext/fts3/fts3.c.
1441
1442
1443
1444
1445
1446
1447

1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
  ** so search through the constraints to see if a more efficient 
  ** strategy is possible.
  */
  pInfo->idxNum = FTS3_FULLSCAN_SEARCH;
  pInfo->estimatedCost = 500000;
  for(i=0; i<pInfo->nConstraint; i++){
    struct sqlite3_index_constraint *pCons = &pInfo->aConstraint[i];

    if( pCons->usable==0 ) continue;

    /* A direct lookup on the rowid or docid column. Assign a cost of 1.0. */
    if( iCons<0 
     && pCons->op==SQLITE_INDEX_CONSTRAINT_EQ 
     && (pCons->iColumn<0 || pCons->iColumn==p->nColumn+1 )
    ){
      pInfo->idxNum = FTS3_DOCID_SEARCH;
      pInfo->estimatedCost = 1.0;
      iCons = i;
    }

    /* A MATCH constraint. Use a full-text search.







>



<
|
|







1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451

1452
1453
1454
1455
1456
1457
1458
1459
1460
  ** so search through the constraints to see if a more efficient 
  ** strategy is possible.
  */
  pInfo->idxNum = FTS3_FULLSCAN_SEARCH;
  pInfo->estimatedCost = 500000;
  for(i=0; i<pInfo->nConstraint; i++){
    struct sqlite3_index_constraint *pCons = &pInfo->aConstraint[i];
    int iCol = pCons->iColumn;
    if( pCons->usable==0 ) continue;

    /* A direct lookup on the rowid or docid column. Assign a cost of 1.0. */

    if( iCons<0 && pCons->op==SQLITE_INDEX_CONSTRAINT_EQ 
     && (iCol<0 || (iCol==p->nColumn+1 && p->nLanguageidBits==0))
    ){
      pInfo->idxNum = FTS3_DOCID_SEARCH;
      pInfo->estimatedCost = 1.0;
      iCons = i;
    }

    /* A MATCH constraint. Use a full-text search.
Changes to test/fts4langid2.test.
192
193
194
195
196
197
198


































































































199
200
201
    SELECT docid, lid, content FROM t1 WHERE t1 MATCH '1'
  } "
      $min_docid  $min_langid  {1 min min x}
  "

  do_execsql_test 5.$bits.5 { DROP TABLE t1 }
}



































































































finish_test








>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>



192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
    SELECT docid, lid, content FROM t1 WHERE t1 MATCH '1'
  } "
      $min_docid  $min_langid  {1 min min x}
  "

  do_execsql_test 5.$bits.5 { DROP TABLE t1 }
}

#-------------------------------------------------------------------------
# Tests for auxilliary functions with langaugeid_bits tables.
#
proc mit {blob} {
  set scan(littleEndian) i*
  set scan(bigEndian) I*
  binary scan $blob $scan($::tcl_platform(byteOrder)) r
  return $r
}
db func mit mit

do_execsql_test 6.1 {
  CREATE VIRTUAL TABLE t1 USING fts4(languageid_bits=4, languageid=lid);
  INSERT INTO t1(docid,lid,content) VALUES(1, 1, 'one two three four');
  INSERT INTO t1(docid,lid,content) VALUES(2, 1, 'two three four five');
  INSERT INTO t1(docid,lid,content) VALUES(3, 1, 'three four five six');
  INSERT INTO t1(docid,lid,content) VALUES(4, 1, 'four five six seven');

  INSERT INTO t1(docid,lid,content) VALUES(1, 2, 'four three two one');
  INSERT INTO t1(docid,lid,content) VALUES(2, 2, 'five four three two');
  INSERT INTO t1(docid,lid,content) VALUES(3, 2, 'six five four three');
  INSERT INTO t1(docid,lid,content) VALUES(4, 2, 'A B C D');
}

do_execsql_test 6.2.1 {
  SELECT docid, snippet(t1) FROM t1 WHERE t1 MATCH 'one' AND lid=1;
} {1 {<b>one</b> two three four}}
do_execsql_test 6.2.2 {
  SELECT docid, snippet(t1) FROM t1 WHERE t1 MATCH 'one' AND lid=2;
} {1 {four three two <b>one</b>}}

do_execsql_test 6.2.1 {
  SELECT docid, offsets(t1) FROM t1 WHERE t1 MATCH 'two' AND lid=1;
} {1 {0 0 4 3} 2 {0 0 0 3}}
do_execsql_test 6.2.2 {
  SELECT docid, offsets(t1) FROM t1 WHERE t1 MATCH 'two' AND lid=2;
} {1 {0 0 11 3} 2 {0 0 16 3}}

do_execsql_test 6.3.1 {
  SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'two' AND lid=1;
} {1 {1 1 1 2 2} 2 {1 1 1 2 2}}
do_execsql_test 6.3.2 {
  SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'two' AND lid=2;
} {1 {1 1 1 2 2} 2 {1 1 1 2 2}}
do_execsql_test 6.3.3 {
  SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'B' AND lid=1;
} {}
do_execsql_test 6.3.4 {
  SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'B' AND lid=2;
} {4 {1 1 1 1 1}}

do_execsql_test 6.4 {
  CREATE VIRTUAL TABLE t2 USING fts4(languageid_bits=1, languageid=lid);
  INSERT INTO t1(docid,lid,content) VALUES(-1, 0, 'A B C D');
  INSERT INTO t1(docid,lid,content) VALUES(-2, 0, 'D C B A');
  INSERT INTO t1(docid,lid,content) VALUES(-3, 0, 'C B D A');
  INSERT INTO t1(docid,lid,content) VALUES(-4, 0, 'A D B C');

  INSERT INTO t1(docid,lid,content) VALUES(-1, 1, 'A A A A');
  INSERT INTO t1(docid,lid,content) VALUES(-2, 1, 'B B B B');
  INSERT INTO t1(docid,lid,content) VALUES(-3, 1, 'C C C C');
  INSERT INTO t1(docid,lid,content) VALUES(-4, 1, 'D D D D');
}

do_execsql_test 6.4.1 {
  SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'B';
} {
  -4 {1 1 1 4 4}
  -3 {1 1 1 4 4}
  -2 {1 1 1 4 4}
  -1 {1 1 1 4 4}
}
do_execsql_test 6.4.2 {
  SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'B' AND lid=1;
} {-2 {1 1 4 4 1}}

do_execsql_test 6.5 {
  DROP TABLE t1;
  DROP TABLE t2;
}

#-------------------------------------------------------------------------
# Tests for querying by docid.
#

do_execsql_test 7.1 {
  CREATE VIRTUAL TABLE t1 USING fts4(languageid_bits=8, languageid=lid);
  INSERT INTO t1(docid,lid,content) VALUES(10, 10, 'abc def');
}

do_execsql_test 7.2 {
  SELECT docid,lid,content FROM t1 WHERE docid=10;
} {10 10 {abc def}}





finish_test

Changes to test/permutations.test.
189
190
191
192
193
194
195
196

197
198
199
200
201
202
203
  fts3expr3.test
  fts3near.test fts3query.test fts3shared.test fts3snippet.test 
  fts3sort.test
  fts3fault.test fts3malloc.test fts3matchinfo.test
  fts3aux1.test fts3comp1.test fts3auto.test
  fts4aa.test fts4content.test
  fts3conf.test fts3prefix.test fts3fault2.test fts3corrupt.test
  fts3corrupt2.test fts3first.test fts4langid.test fts4merge.test

  fts4check.test fts4unicode.test
}

test_suite "nofaultsim" -prefix "" -description {
  "Very" quick test suite. Runs in less than 5 minutes on a workstation. 
  This test suite is the same as the "quick" tests, except that some files
  that test malloc and IO errors are omitted.







|
>







189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
  fts3expr3.test
  fts3near.test fts3query.test fts3shared.test fts3snippet.test 
  fts3sort.test
  fts3fault.test fts3malloc.test fts3matchinfo.test
  fts3aux1.test fts3comp1.test fts3auto.test
  fts4aa.test fts4content.test
  fts3conf.test fts3prefix.test fts3fault2.test fts3corrupt.test
  fts3corrupt2.test fts3first.test fts4langid.test fts4langid2.test
  fts4merge.test
  fts4check.test fts4unicode.test
}

test_suite "nofaultsim" -prefix "" -description {
  "Very" quick test suite. Runs in less than 5 minutes on a workstation. 
  This test suite is the same as the "quick" tests, except that some files
  that test malloc and IO errors are omitted.