/ Check-in [ad0987d8]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix a problem with prefix queries on fts5 offsets=0 tables.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts5-offsets
Files: files | file ages | folders
SHA1: ad0987d83c252dd8d6a69321893629d7be805c28
User & Date: dan 2015-12-18 19:07:14
Context
2015-12-21
18:45
Fix an fts5 integrity-check problem that affects offsets=0 tables with prefix indexes. check-in: 609a0bc7 user: dan tags: fts5-offsets
2015-12-18
19:07
Fix a problem with prefix queries on fts5 offsets=0 tables. check-in: ad0987d8 user: dan tags: fts5-offsets
2015-12-17
20:36
Add the "offsets=0" option to fts5, to create a smaller index without term offset information. A few things are currently broken on this branch. check-in: 40b5bbf0 user: dan tags: fts5-offsets
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5_expr.c.

1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812











1813
1814
1815
1816
1817
1818
1819
      pRet->eType = eType;
      pRet->pNear = pNear;
      if( eType==FTS5_STRING ){
        int iPhrase;
        for(iPhrase=0; iPhrase<pNear->nPhrase; iPhrase++){
          pNear->apPhrase[iPhrase]->pNode = pRet;
        }
        if( pNear->nPhrase==1 
         && pNear->apPhrase[0]->nTerm==1 
         && pNear->apPhrase[0]->aTerm[0].pSynonym==0
        ){
          pRet->eType = FTS5_TERM;











        }
      }else{
        fts5ExprAddChildren(pRet, pLeft);
        fts5ExprAddChildren(pRet, pRight);
      }
    }
  }







<
|
|
<
|
>
>
>
>
>
>
>
>
>
>
>







1801
1802
1803
1804
1805
1806
1807

1808
1809

1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
      pRet->eType = eType;
      pRet->pNear = pNear;
      if( eType==FTS5_STRING ){
        int iPhrase;
        for(iPhrase=0; iPhrase<pNear->nPhrase; iPhrase++){
          pNear->apPhrase[iPhrase]->pNode = pRet;
        }

        if( pNear->nPhrase==1 && pNear->apPhrase[0]->nTerm==1 ){
          if( pNear->apPhrase[0]->aTerm[0].pSynonym==0 ){

            pRet->eType = FTS5_TERM;
          }
        }else if( pParse->pConfig->bOffsets==0 ){
          assert( pParse->rc==SQLITE_OK );
          pParse->rc = SQLITE_ERROR;
          assert( pParse->zErr==0 );
          pParse->zErr = sqlite3_mprintf(
              "fts5: %s queries are not supported (offsets=0)", 
              pNear->nPhrase==1 ? "phrase": "NEAR"
          );
          sqlite3_free(pRet);
          pRet = 0;
        }
      }else{
        fts5ExprAddChildren(pRet, pLeft);
        fts5ExprAddChildren(pRet, pRight);
      }
    }
  }

Changes to ext/fts5/fts5_index.c.

4115
4116
4117
4118
4119
4120
4121
4122
4123

4124
4125
4126
4127
4128
4129
4130
....
4188
4189
4190
4191
4192
4193
4194
4195

4196
4197
4198
4199
4200
4201
4202
4203
....
4234
4235
4236
4237
4238
4239
4240
4241
4242
4243
4244
4245
4246

4247
4248
4249
4250
4251
4252
4253
....
4386
4387
4388
4389
4390
4391
4392



4393
4394
4395
4396
4397
4398
4399
        sCtx.pBuf = pBuf;
        sCtx.pColset = pColset;
        fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistOffsetsCallback);
      }else{
        PoslistCallbackCtx sCtx;
        sCtx.pBuf = pBuf;
        sCtx.pColset = pColset;
        assert( sCtx.eState==0 || sCtx.eState==1 );
        sCtx.eState = fts5IndexColsetTest(pColset, 0);

        fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistFilterCallback);
      }
    }
  }
}

/*
................................................................................
  Fts5Buffer *pBuf
){
  if( p->rc==SQLITE_OK ){
    Fts5SegIter *pSeg = &pMulti->aSeg[ pMulti->aFirst[1].iFirst ];
    assert( fts5MultiIterEof(p, pMulti)==0 );
    assert( pSeg->nPos>0 );
    if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos+9+9) ){


      if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf 
       && (pColset==0 || pColset->nCol==1)
      ){
        const u8 *pPos = &pSeg->pLeaf->p[pSeg->iLeafOffset];
        int nPos;
        if( pColset ){
          nPos = fts5IndexExtractCol(&pPos, pSeg->nPos, pColset->aiCol[0]);
          if( nPos==0 ) return 1;
................................................................................
              int nReq = sqlite3Fts5GetVarintLen((u32)(nActual*2));
              while( iSv2<(iData-nReq) ){ pBuf->p[iSv2++] = 0x80; }
              sqlite3Fts5PutVarint(&pBuf->p[iSv2], nActual*2);
            }
          }
        }
      }

    }
  }

  return 0;
}


static void fts5DoclistIterNext(Fts5DoclistIter *pIter){
  u8 *p = pIter->aPoslist + pIter->nSize + pIter->nPoslist;

  assert( pIter->aPoslist );
  if( p>=pIter->aEof ){
    pIter->aPoslist = 0;
................................................................................

    fts5BufferSet(&p->rc, p1, out.n, out.p);
    fts5BufferFree(&tmp);
    fts5BufferFree(&out);
  }
}




static void fts5BufferSwap(Fts5Buffer *p1, Fts5Buffer *p2){
  Fts5Buffer tmp = *p1;
  *p1 = *p2;
  *p2 = tmp;
}

static void fts5SetupPrefixIter(







<

>







 







<
>
|







 







<





>







 







>
>
>







4115
4116
4117
4118
4119
4120
4121

4122
4123
4124
4125
4126
4127
4128
4129
4130
....
4188
4189
4190
4191
4192
4193
4194

4195
4196
4197
4198
4199
4200
4201
4202
4203
....
4234
4235
4236
4237
4238
4239
4240

4241
4242
4243
4244
4245
4246
4247
4248
4249
4250
4251
4252
4253
....
4386
4387
4388
4389
4390
4391
4392
4393
4394
4395
4396
4397
4398
4399
4400
4401
4402
        sCtx.pBuf = pBuf;
        sCtx.pColset = pColset;
        fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistOffsetsCallback);
      }else{
        PoslistCallbackCtx sCtx;
        sCtx.pBuf = pBuf;
        sCtx.pColset = pColset;

        sCtx.eState = fts5IndexColsetTest(pColset, 0);
        assert( sCtx.eState==0 || sCtx.eState==1 );
        fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistFilterCallback);
      }
    }
  }
}

/*
................................................................................
  Fts5Buffer *pBuf
){
  if( p->rc==SQLITE_OK ){
    Fts5SegIter *pSeg = &pMulti->aSeg[ pMulti->aFirst[1].iFirst ];
    assert( fts5MultiIterEof(p, pMulti)==0 );
    assert( pSeg->nPos>0 );
    if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos+9+9) ){

      if( p->pConfig->bOffsets
       && pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf 
       && (pColset==0 || pColset->nCol==1)
      ){
        const u8 *pPos = &pSeg->pLeaf->p[pSeg->iLeafOffset];
        int nPos;
        if( pColset ){
          nPos = fts5IndexExtractCol(&pPos, pSeg->nPos, pColset->aiCol[0]);
          if( nPos==0 ) return 1;
................................................................................
              int nReq = sqlite3Fts5GetVarintLen((u32)(nActual*2));
              while( iSv2<(iData-nReq) ){ pBuf->p[iSv2++] = 0x80; }
              sqlite3Fts5PutVarint(&pBuf->p[iSv2], nActual*2);
            }
          }
        }
      }

    }
  }

  return 0;
}


static void fts5DoclistIterNext(Fts5DoclistIter *pIter){
  u8 *p = pIter->aPoslist + pIter->nSize + pIter->nPoslist;

  assert( pIter->aPoslist );
  if( p>=pIter->aEof ){
    pIter->aPoslist = 0;
................................................................................

    fts5BufferSet(&p->rc, p1, out.n, out.p);
    fts5BufferFree(&tmp);
    fts5BufferFree(&out);
  }
}

/*
** Swap the contents of buffer *p1 with that of *p2.
*/
static void fts5BufferSwap(Fts5Buffer *p1, Fts5Buffer *p2){
  Fts5Buffer tmp = *p1;
  *p1 = *p2;
  *p2 = tmp;
}

static void fts5SetupPrefixIter(

Changes to ext/fts5/test/fts5offsets.test.

18
19
20
21
22
23
24



25
26
27
28
29
30
31
32
33
34
..
51
52
53
54
55
56
57








58
59
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
  finish_test
  return
}





do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, offsets=0);

  INSERT INTO t1 VALUES('h d g', 'j b b g b', 'i e i d h g g'); -- 1
  INSERT INTO t1 VALUES('h j d', 'j h d a h', 'f d d g g f b'); -- 2
  INSERT INTO t1 VALUES('j c i', 'f f h e f', 'c j i j c h f'); -- 3
  INSERT INTO t1 VALUES('e g g', 'g e d h i', 'e d b e g d c'); -- 4
  INSERT INTO t1 VALUES('b c c', 'd i h a f', 'd i j f a b c'); -- 5
  INSERT INTO t1 VALUES('e d e', 'b c j g d', 'a i f d h b d'); -- 6
  INSERT INTO t1 VALUES('g h e', 'b c d i d', 'e f c i f i c'); -- 7
................................................................................
  } $res

  do_execsql_test 1.2.$tn.2 {
    SELECT rowid FROM t1($match || '*');
  } $res
}









finish_test








>
>
>


<







 







>
>
>
>
>
>
>
>


18
19
20
21
22
23
24
25
26
27
28
29

30
31
32
33
34
35
36
..
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
  finish_test
  return
}


#--------------------------------------------------------------------------
# Simple tests.
#
do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, offsets=0);

  INSERT INTO t1 VALUES('h d g', 'j b b g b', 'i e i d h g g'); -- 1
  INSERT INTO t1 VALUES('h j d', 'j h d a h', 'f d d g g f b'); -- 2
  INSERT INTO t1 VALUES('j c i', 'f f h e f', 'c j i j c h f'); -- 3
  INSERT INTO t1 VALUES('e g g', 'g e d h i', 'e d b e g d c'); -- 4
  INSERT INTO t1 VALUES('b c c', 'd i h a f', 'd i j f a b c'); -- 5
  INSERT INTO t1 VALUES('e d e', 'b c j g d', 'a i f d h b d'); -- 6
  INSERT INTO t1 VALUES('g h e', 'b c d i d', 'e f c i f i c'); -- 7
................................................................................
  } $res

  do_execsql_test 1.2.$tn.2 {
    SELECT rowid FROM t1($match || '*');
  } $res
}

do_catchsql_test 1.3.1 {
  SELECT rowid FROM t1('h + d');
} {1 {fts5: phrase queries are not supported (offsets=0)}}

do_catchsql_test 1.3.2 {
  SELECT rowid FROM t1('NEAR(h d)');
} {1 {fts5: NEAR queries are not supported (offsets=0)}}

finish_test