/ Check-in [81527768]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix some issues related to ORDER BY and fts tables with a non-zero languageid_bits setting.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts-languageid-bits
Files: files | file ages | folders
SHA1: 81527768ef9b39289cfda69d415069b7968379dd
User & Date: dan 2013-06-20 18:32:34
Context
2013-06-20
19:55
Add tests for modifying the docid and languageid fields of an fts table with a non-zero languageid_bits field. Closed-Leaf check-in: 949425d4 user: dan tags: fts-languageid-bits
18:32
Fix some issues related to ORDER BY and fts tables with a non-zero languageid_bits setting. check-in: 81527768 user: dan tags: fts-languageid-bits
16:22
Add extra tests for fts with a non-zero languageid_bits setting. Fix querying by docid with the same. check-in: b1df00f3 user: dan tags: fts-languageid-bits
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts3/fts3.c.

1489
1490
1491
1492
1493
1494
1495

1496
1497

1498


1499
1500
1501
1502
1503
1504
1505
  if( iLangidCons>=0 ){
    pInfo->aConstraintUsage[iLangidCons].argvIndex = 2;
  } 

  /* Regardless of the strategy selected, FTS can deliver rows in rowid (or
  ** docid) order. Both ascending and descending are possible. 
  */

  if( pInfo->nOrderBy==1 ){
    struct sqlite3_index_orderby *pOrder = &pInfo->aOrderBy[0];

    if( pOrder->iColumn<0 || pOrder->iColumn==p->nColumn+1 ){


      if( pOrder->desc ){
        pInfo->idxStr = "DESC";
      }else{
        pInfo->idxStr = "ASC";
      }
      pInfo->orderByConsumed = 1;
    }







>


>
|
>
>







1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
  if( iLangidCons>=0 ){
    pInfo->aConstraintUsage[iLangidCons].argvIndex = 2;
  } 

  /* Regardless of the strategy selected, FTS can deliver rows in rowid (or
  ** docid) order. Both ascending and descending are possible. 
  */
  assert( pInfo->orderByConsumed==0 );
  if( pInfo->nOrderBy==1 ){
    struct sqlite3_index_orderby *pOrder = &pInfo->aOrderBy[0];
    if( pOrder->iColumn<0 || (
          (pOrder->iColumn==p->nColumn+1)
       && (pInfo->idxNum>=FTS3_FULLTEXT_SEARCH || p->nLanguageidBits==0)
    )){
      if( pOrder->desc ){
        pInfo->idxStr = "DESC";
      }else{
        pInfo->idxStr = "ASC";
      }
      pInfo->orderByConsumed = 1;
    }

Changes to ext/fts3/fts3_write.c.

5260
5261
5262
5263
5264
5265
5266












5267
5268
5269

5270
5271

5272





5273
5274
5275




5276

5277










5278
5279
5280
5281
5282
5283
5284
  return rc;
}

/*
** Convert a docid (iDocid) and a language id (iLangid) to a rowid,
** according to the configured languageid_bits= value belonging to
** FTS table *p.












*/
i64 sqlite3Fts3DocidToRowid(Fts3Table *p, i64 iDocid, int iLangid){
  i64 iRowid = iDocid;

  if( p->nLanguageidBits ){
    iRowid = (iRowid << p->nLanguageidBits) + iLangid;

  }





  return iRowid;
}





i64 sqlite3Fts3RowidToDocid(Fts3Table *p, i64 iRowid){

  return (iRowid >> p->nLanguageidBits);










}

/*
** This function does the work for the xUpdate method of FTS3 virtual
** tables. The schema of the virtual table being:
**
**     CREATE TABLE <table name>( 







>
>
>
>
>
>
>
>
>
>
>
>


|
>

|
>
|
>
>
>
>
>
|


>
>
>
>

>
|
>
>
>
>
>
>
>
>
>
>







5260
5261
5262
5263
5264
5265
5266
5267
5268
5269
5270
5271
5272
5273
5274
5275
5276
5277
5278
5279
5280
5281
5282
5283
5284
5285
5286
5287
5288
5289
5290
5291
5292
5293
5294
5295
5296
5297
5298
5299
5300
5301
5302
5303
5304
5305
5306
5307
5308
5309
5310
5311
5312
5313
5314
5315
5316
5317
5318
  return rc;
}

/*
** Convert a docid (iDocid) and a language id (iLangid) to a rowid,
** according to the configured languageid_bits= value belonging to
** FTS table *p.
**
** The conversion is as follows:
**
**   * The sign bit of iDocid becomes the sign bit of the rowid.
**
**   * iLangid is converted to an unsigned integer and stored in
**     the next most significant Fts3Table.nLanguageidBits bits
**     of the returned rowid.
**
**   * The least signficant (63-nLanguageidBits) of iDocid are
**     copied to the (63-nLanguageidBits) least signifcant bits of
**     the returned rowid.
*/
i64 sqlite3Fts3DocidToRowid(Fts3Table *p, i64 iDocid, int iLangid){
  u64 iRet = iDocid;

  if( p->nLanguageidBits ){
    int iShift = (63 - p->nLanguageidBits);
    u64 mask = ((((u64)1 << p->nLanguageidBits) - 1) << iShift);

    iRet &= ~mask;
    iRet |= (u64)iLangid << iShift;
  }

  assert( sqlite3Fts3RowidToDocid(p, (i64)iRet)==iDocid );
  return (i64)iRet;
}

/*
** Convert a rowid (iRowid) to a docid according to the languageid_bits=
** value belonging to FTS table *p.
*/
i64 sqlite3Fts3RowidToDocid(Fts3Table *p, i64 iRowid){
  u64 iRet = iRowid;
  if( p->nLanguageidBits ){
    static const u64 signbit = ((u64)1 << 63);
    u64 mask = ((((u64)1 << p->nLanguageidBits)-1) << (63-p->nLanguageidBits));

    if( iRet & signbit ){
      iRet |= mask;
    }else{
      iRet &= ~mask;
    }
  }
  return (i64)iRet;
}

/*
** This function does the work for the xUpdate method of FTS3 virtual
** tables. The schema of the virtual table being:
**
**     CREATE TABLE <table name>( 

Changes to test/fts4langid2.test.

18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
...
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293



294


295
296

























297
298
299

# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts3 {
  finish_test
  return
}


#-------------------------------------------------------------------------
# Test out-of-range values for the languageid_bits= parameter.
#
do_catchsql_test 1.1 {
  CREATE VIRTUAL TABLE t1 USING fts4(languageid=lid, languageid_bits=31);
} {1 {languageid_bits parameter out of range}}

................................................................................
  SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'B' AND lid=1;
} {}
do_execsql_test 6.3.4 {
  SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'B' AND lid=2;
} {4 {1 1 1 1 1}}

do_execsql_test 6.4 {
  CREATE VIRTUAL TABLE t2 USING fts4(languageid_bits=1, languageid=lid);
  INSERT INTO t1(docid,lid,content) VALUES(-1, 0, 'A B C D');
  INSERT INTO t1(docid,lid,content) VALUES(-2, 0, 'D C B A');
  INSERT INTO t1(docid,lid,content) VALUES(-3, 0, 'C B D A');
  INSERT INTO t1(docid,lid,content) VALUES(-4, 0, 'A D B C');

  INSERT INTO t1(docid,lid,content) VALUES(-1, 1, 'A A A A');
  INSERT INTO t1(docid,lid,content) VALUES(-2, 1, 'B B B B');
  INSERT INTO t1(docid,lid,content) VALUES(-3, 1, 'C C C C');
  INSERT INTO t1(docid,lid,content) VALUES(-4, 1, 'D D D D');
}

do_execsql_test 6.4.1 {
  SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'B';
} {
  -4 {1 1 1 4 4}
  -3 {1 1 1 4 4}
  -2 {1 1 1 4 4}
  -1 {1 1 1 4 4}
}
do_execsql_test 6.4.2 {
  SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'B' AND lid=1;
} {-2 {1 1 4 4 1}}

do_execsql_test 6.5 {
  DROP TABLE t1;
  DROP TABLE t2;
}

#-------------------------------------------------------------------------
# Tests for querying by docid.
#

do_execsql_test 7.1 {
  CREATE VIRTUAL TABLE t1 USING fts4(languageid_bits=8, languageid=lid);
  INSERT INTO t1(docid,lid,content) VALUES(10, 10, 'abc def');
}

do_execsql_test 7.2 {
  SELECT docid,lid,content FROM t1 WHERE docid=10;
} {10 10 {abc def}}



































finish_test








<







 







|
|
|
|
|

|
|
|
|



|







|










<









>
>
>

>
>
|

>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>



18
19
20
21
22
23
24

25
26
27
28
29
30
31
...
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282

283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327

# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts3 {
  finish_test
  return
}


#-------------------------------------------------------------------------
# Test out-of-range values for the languageid_bits= parameter.
#
do_catchsql_test 1.1 {
  CREATE VIRTUAL TABLE t1 USING fts4(languageid=lid, languageid_bits=31);
} {1 {languageid_bits parameter out of range}}

................................................................................
  SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'B' AND lid=1;
} {}
do_execsql_test 6.3.4 {
  SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'B' AND lid=2;
} {4 {1 1 1 1 1}}

do_execsql_test 6.4 {
  CREATE VIRTUAL TABLE t2 USING fts4(languageid_bits=8, languageid=lid);
  INSERT INTO t2(docid,lid,content) VALUES(-1, 0, 'A B C D');
  INSERT INTO t2(docid,lid,content) VALUES(-2, 0, 'D C B A');
  INSERT INTO t2(docid,lid,content) VALUES(-3, 0, 'C B D A');
  INSERT INTO t2(docid,lid,content) VALUES(-4, 0, 'A D B C');

  INSERT INTO t2(docid,lid,content) VALUES(-1, 1, 'A A A A');
  INSERT INTO t2(docid,lid,content) VALUES(-2, 1, 'B B B B');
  INSERT INTO t2(docid,lid,content) VALUES(-3, 1, 'C C C C');
  INSERT INTO t2(docid,lid,content) VALUES(-4, 1, 'D D D D');
}

do_execsql_test 6.4.1 {
  SELECT docid, mit(matchinfo(t2)) FROM t2 WHERE t2 MATCH 'B';
} {
  -4 {1 1 1 4 4}
  -3 {1 1 1 4 4}
  -2 {1 1 1 4 4}
  -1 {1 1 1 4 4}
}
do_execsql_test 6.4.2 {
  SELECT docid, mit(matchinfo(t2)) FROM t2 WHERE t2 MATCH 'B' AND lid=1;
} {-2 {1 1 4 4 1}}

do_execsql_test 6.5 {
  DROP TABLE t1;
  DROP TABLE t2;
}

#-------------------------------------------------------------------------
# Tests for querying by docid.
#

do_execsql_test 7.1 {
  CREATE VIRTUAL TABLE t1 USING fts4(languageid_bits=8, languageid=lid);
  INSERT INTO t1(docid,lid,content) VALUES(10, 10, 'abc def');
}

do_execsql_test 7.2 {
  SELECT docid,lid,content FROM t1 WHERE docid=10;
} {10 10 {abc def}}

do_execsql_test 7.3 {
  SELECT docid,lid,content FROM t1 WHERE docid<11;
} {10 10 {abc def}}

do_execsql_test 7.4 {
  DROP TABLE t1;
}

#-------------------------------------------------------------------------
# Tests for sorting by docid.
#
do_execsql_test 8.1 {
  CREATE VIRTUAL TABLE t1 USING fts4(languageid_bits=6, languageid=lid);
  INSERT INTO t1 (docid,lid,content) VALUES(1, 0, 'abc def');
  INSERT INTO t1 (docid,lid,content) VALUES(3, 0, 'abc ghi');
  INSERT INTO t1 (docid,lid,content) VALUES(2, 0, 'def ghi');

  INSERT INTO t1 (docid,lid,content) VALUES(1, 5, 'A B');
  INSERT INTO t1 (docid,lid,content) VALUES(3, 5, 'A C');
  INSERT INTO t1 (docid,lid,content) VALUES(2, 5, 'B C');
}

do_execsql_test 8.2 {
  SELECT docid FROM t1 ORDER BY docid;
} {1 1 2 2 3 3}

do_execsql_test 8.2 {
  SELECT docid FROM t1 WHERE t1 MATCH 'ghi' ORDER BY docid;
} {2 3}

do_execsql_test 8.2 {
  SELECT docid FROM t1 WHERE t1 MATCH 'ghi' ORDER BY docid DESC;
} {3 2}

finish_test