/ Check-in [94eeb077]
Login
Home Timeline Branches Tags Tickets Wiki More...

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix minor problems in term matching.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts5
Files: files | file ages | folders
SHA1: 94eeb077d08a1d2607f3ff3a9fbf18229ba475bb
User & Date: dan 2014-06-26 12:31:41
Context
2014-07-01
20:45
Change the position list format so that its size in bytes is stored at the start of the list itself. check-in: 62f2ff20 user: dan tags: fts5
2014-06-26
12:31
Fix minor problems in term matching. check-in: 94eeb077 user: dan tags: fts5
2014-06-25
20:28
Begin adding query support to fts5. check-in: 47a9f3cc user: dan tags: fts5
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5_expr.c. 

457
458
459
460
461
462
463
464

465

466
467
468
469
470
471
472

    if( pNew==0 ) return SQLITE_NOMEM;
    pCtx->pPhrase = pPhrase = pNew;
    pNew->nTerm = nNew - SZALLOC;
  }

  pTerm = &pPhrase->aTerm[pPhrase->nTerm++];
  pTerm->bPrefix = 0;
  

  pTerm->zTerm = fts5Strdup(pToken, nToken);

  return pTerm->zTerm ? SQLITE_OK : SQLITE_NOMEM;
}


/*
** Free the phrase object passed as the only argument.
*/








<
>

>








457
458
459
460
461
462
463

464
465
466
467
468
469
470
471
472
473

    if( pNew==0 ) return SQLITE_NOMEM;
    pCtx->pPhrase = pPhrase = pNew;
    pNew->nTerm = nNew - SZALLOC;
  }

  pTerm = &pPhrase->aTerm[pPhrase->nTerm++];
  pTerm->bPrefix = 0;

  pTerm->pIter = 0;
  pTerm->zTerm = fts5Strdup(pToken, nToken);

  return pTerm->zTerm ? SQLITE_OK : SQLITE_NOMEM;
}


/*
** Free the phrase object passed as the only argument.
*/

Changes to ext/fts5/fts5_index.c. 

1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
....
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203


1204
1205
1206
1207
1208
1209
1210
....
1211
1212
1213
1214
1215
1216
1217
1218
1219


1220
1221









1222

1223

1224
1225
1226
1227
1228
1229
1230

*/
static void fts5SegIterNextPage(
  Fts5Index *p,                   /* FTS5 backend object */
  Fts5SegIter *pIter              /* Iterator to advance to next page */
){
  Fts5StructureSegment *pSeg = pIter->pSeg;
  if( pIter->pLeaf ) fts5DataRelease(pIter->pLeaf);
  if( pIter->iLeafPgno<pSeg->pgnoLast ){
    pIter->iLeafPgno++;
    pIter->pLeaf = fts5DataRead(p, 
        FTS5_SEGMENT_ROWID(pIter->iIdx, pSeg->iSegid, 0, pIter->iLeafPgno)
    );
  }else{
    pIter->pLeaf = 0;
  }
}
................................................................................
    if( pNode==0 ) break;

    fts5NodeIterInit(pNode->p, pNode->n, &node);
    assert( node.term.n==0 );

    iPg = node.iChild;
    for(fts5NodeIterNext(&p->rc, &node);
        node.aData && fts5BufferCompareBlob(&node.term, pTerm, nTerm)>=0;
        fts5NodeIterNext(&p->rc, &node)
    ){
      iPg = node.iChild;
    }


  }

  if( iPg>=pSeg->pgnoFirst ){
    int res;
    pIter->iLeafPgno = iPg - 1;
    fts5SegIterNextPage(p, pIter);
    if( pIter->pLeaf ){
................................................................................
      u8 *a = pIter->pLeaf->p;
      int n = pIter->pLeaf->n;

      pIter->iLeafOffset = fts5GetU16(&a[2]);
      fts5SegIterLoadTerm(p, pIter, 0);

      while( (res = fts5BufferCompareBlob(&pIter->term, pTerm, nTerm)) ){
        if( res<0 ){
          /* Search for the end of the position list within the current page. */


          int iOff;
          for(iOff=pIter->iLeafOffset; iOff<n && a[iOff]; iOff++);









          pIter->iLeafOffset = iOff+1;

          if( iOff<n ) continue;

        }

        /* No matching term on this page. Set the iterator to EOF. */
        fts5DataRelease(pIter->pLeaf);
        pIter->pLeaf = 0;
        break;
      }








|
|







 







|




>
>







 







|
|
>
>

|
>
>
>
>
>
>
>
>
>
|
>
|
>








1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
....
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
....
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245

*/
static void fts5SegIterNextPage(
  Fts5Index *p,                   /* FTS5 backend object */
  Fts5SegIter *pIter              /* Iterator to advance to next page */
){
  Fts5StructureSegment *pSeg = pIter->pSeg;
  if( pIter->pLeaf ) fts5DataRelease(pIter->pLeaf);
  pIter->iLeafPgno++;
  if( pIter->iLeafPgno<=pSeg->pgnoLast ){
    pIter->pLeaf = fts5DataRead(p, 
        FTS5_SEGMENT_ROWID(pIter->iIdx, pSeg->iSegid, 0, pIter->iLeafPgno)
    );
  }else{
    pIter->pLeaf = 0;
  }
}
................................................................................
    if( pNode==0 ) break;

    fts5NodeIterInit(pNode->p, pNode->n, &node);
    assert( node.term.n==0 );

    iPg = node.iChild;
    for(fts5NodeIterNext(&p->rc, &node);
        node.aData && fts5BufferCompareBlob(&node.term, pTerm, nTerm)<=0;
        fts5NodeIterNext(&p->rc, &node)
    ){
      iPg = node.iChild;
    }
    fts5NodeIterFree(&node);
    fts5DataRelease(pNode);
  }

  if( iPg>=pSeg->pgnoFirst ){
    int res;
    pIter->iLeafPgno = iPg - 1;
    fts5SegIterNextPage(p, pIter);
    if( pIter->pLeaf ){
................................................................................
      u8 *a = pIter->pLeaf->p;
      int n = pIter->pLeaf->n;

      pIter->iLeafOffset = fts5GetU16(&a[2]);
      fts5SegIterLoadTerm(p, pIter, 0);

      while( (res = fts5BufferCompareBlob(&pIter->term, pTerm, nTerm)) ){
        if( res<0 && pIter->iLeafPgno==iPg ){
          /* Search for the end of the current doclist within the current
          ** page. The end of a doclist is marked by a pair of successive
          ** 0x00 bytes.  */
          int iOff;
          for(iOff=pIter->iLeafOffset+1; iOff<n; iOff++){
            if( a[iOff]==0 && a[iOff-1]==0 ) break;
          }
          iOff++;

          /* If the iterator is not yet at the end of the next page, load
          ** the next term and jump to the next iteration of the while()
          ** loop.  */
          if( iOff<n ){
            int nKeep;
            pIter->iLeafOffset = iOff + getVarint32(&a[iOff], nKeep);
            fts5SegIterLoadTerm(p, pIter, nKeep);
            continue;
          }
        }

        /* No matching term on this page. Set the iterator to EOF. */
        fts5DataRelease(pIter->pLeaf);
        pIter->pLeaf = 0;
        break;
      }

Changes to test/fts5ab.test. 

55
56
57
58
59
60
61

62
63
64
65
66
67
68
69
70
71
72
73
74




75
76





















77


} {}

#-------------------------------------------------------------------------

reset_db
do_execsql_test 2.1 {
  CREATE VIRTUAL TABLE t1 USING fts5(x);

  INSERT INTO t1 VALUES('one');
  INSERT INTO t1 VALUES('two');
  INSERT INTO t1 VALUES('three');
}

do_catchsql_test 2.2 {
  SELECT rowid, * FROM t1 WHERE t1 MATCH 'AND AND'
} {1 {fts5: syntax error near "AND"}}

do_execsql_test 2.3 { SELECT rowid, * FROM t1 WHERE t1 MATCH 'two' } {2 two}
do_execsql_test 2.4 { SELECT rowid, * FROM t1 WHERE t1 MATCH 'three' } {3 three}
do_execsql_test 2.5 { SELECT rowid, * FROM t1 WHERE t1 MATCH 'one' } {1 one}




























finish_test









>













>
>
>
>
|

>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>

>

55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104

} {}

#-------------------------------------------------------------------------

reset_db
do_execsql_test 2.1 {
  CREATE VIRTUAL TABLE t1 USING fts5(x);
  INSERT INTO t1(t1) VALUES('pgsz=32');
  INSERT INTO t1 VALUES('one');
  INSERT INTO t1 VALUES('two');
  INSERT INTO t1 VALUES('three');
}

do_catchsql_test 2.2 {
  SELECT rowid, * FROM t1 WHERE t1 MATCH 'AND AND'
} {1 {fts5: syntax error near "AND"}}

do_execsql_test 2.3 { SELECT rowid, * FROM t1 WHERE t1 MATCH 'two' } {2 two}
do_execsql_test 2.4 { SELECT rowid, * FROM t1 WHERE t1 MATCH 'three' } {3 three}
do_execsql_test 2.5 { SELECT rowid, * FROM t1 WHERE t1 MATCH 'one' } {1 one}

do_execsql_test 2.6 {
  INSERT INTO t1 VALUES('a b c d e f g');
  INSERT INTO t1 VALUES('b d e a a a i');
  INSERT INTO t1 VALUES('x y z b c c c');
}

foreach {tn expr res} {
  1  a    {5 4}
  2  b    {6 5 4}
  3  c    {6 4}
  4  d    {5 4}
  5  e    {5 4}
  6  f    {4}
  7  g    {4}
  8  x    {6}
  9  y    {6}
  10 z    {6}
} {
  do_execsql_test 2.7.$tn { SELECT rowid FROM t1 WHERE t1 MATCH $expr } $res
}

#db eval {
#  SELECT fts5_decode(rowid, block) AS t FROM t1_data;
#} {
#  puts $t
#}

finish_test

This page was generated in about 0.02s by Fossil 2.11 [eee161e823] 2020-01-22 19:08:54