SQLite

Check-in [ad0987d83c]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix a problem with prefix queries on fts5 offsets=0 tables.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | fts5-offsets
Files: files | file ages | folders
SHA1: ad0987d83c252dd8d6a69321893629d7be805c28
User & Date: dan 2015-12-18 19:07:14.984
Context
2015-12-21
18:45
Fix an fts5 integrity-check problem that affects offsets=0 tables with prefix indexes. (check-in: 609a0bc7f3 user: dan tags: fts5-offsets)
2015-12-18
19:07
Fix a problem with prefix queries on fts5 offsets=0 tables. (check-in: ad0987d83c user: dan tags: fts5-offsets)
2015-12-17
20:36
Add the "offsets=0" option to fts5, to create a smaller index without term offset information. A few things are currently broken on this branch. (check-in: 40b5bbf02a user: dan tags: fts5-offsets)
Changes
Unified Diff Ignore Whitespace Patch
Changes to ext/fts5/fts5_expr.c.
︙︙
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812











1813
1814
1815
1816
1817
1818
1819

      pRet->eType = eType;
      pRet->pNear = pNear;
      if( eType==FTS5_STRING ){
        int iPhrase;
        for(iPhrase=0; iPhrase<pNear->nPhrase; iPhrase++){
          pNear->apPhrase[iPhrase]->pNode = pRet;
        }
        if( pNear->nPhrase==1 
         && pNear->apPhrase[0]->nTerm==1 
         && pNear->apPhrase[0]->aTerm[0].pSynonym==0
        ){
          pRet->eType = FTS5_TERM;











        }
      }else{
        fts5ExprAddChildren(pRet, pLeft);
        fts5ExprAddChildren(pRet, pRight);
      }
    }
  }








|
<
|
<
|
>
>
>
>
>
>
>
>
>
>
>








1801
1802
1803
1804
1805
1806
1807
1808

1809

1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828

      pRet->eType = eType;
      pRet->pNear = pNear;
      if( eType==FTS5_STRING ){
        int iPhrase;
        for(iPhrase=0; iPhrase<pNear->nPhrase; iPhrase++){
          pNear->apPhrase[iPhrase]->pNode = pRet;
        }
        if( pNear->nPhrase==1 && pNear->apPhrase[0]->nTerm==1 ){

          if( pNear->apPhrase[0]->aTerm[0].pSynonym==0 ){

            pRet->eType = FTS5_TERM;
          }
        }else if( pParse->pConfig->bOffsets==0 ){
          assert( pParse->rc==SQLITE_OK );
          pParse->rc = SQLITE_ERROR;
          assert( pParse->zErr==0 );
          pParse->zErr = sqlite3_mprintf(
              "fts5: %s queries are not supported (offsets=0)", 
              pNear->nPhrase==1 ? "phrase": "NEAR"
          );
          sqlite3_free(pRet);
          pRet = 0;
        }
      }else{
        fts5ExprAddChildren(pRet, pLeft);
        fts5ExprAddChildren(pRet, pRight);
      }
    }
  }
︙︙
Changes to ext/fts5/fts5_index.c.
︙︙
4115
4116
4117
4118
4119
4120
4121
4122
4123

4124
4125
4126
4127
4128
4129
4130

        sCtx.pBuf = pBuf;
        sCtx.pColset = pColset;
        fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistOffsetsCallback);
      }else{
        PoslistCallbackCtx sCtx;
        sCtx.pBuf = pBuf;
        sCtx.pColset = pColset;
        assert( sCtx.eState==0 || sCtx.eState==1 );
        sCtx.eState = fts5IndexColsetTest(pColset, 0);

        fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistFilterCallback);
      }
    }
  }
}

/*








<

>








4115
4116
4117
4118
4119
4120
4121

4122
4123
4124
4125
4126
4127
4128
4129
4130

        sCtx.pBuf = pBuf;
        sCtx.pColset = pColset;
        fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistOffsetsCallback);
      }else{
        PoslistCallbackCtx sCtx;
        sCtx.pBuf = pBuf;
        sCtx.pColset = pColset;

        sCtx.eState = fts5IndexColsetTest(pColset, 0);
        assert( sCtx.eState==0 || sCtx.eState==1 );
        fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistFilterCallback);
      }
    }
  }
}

/*
︙︙
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203

  Fts5Buffer *pBuf
){
  if( p->rc==SQLITE_OK ){
    Fts5SegIter *pSeg = &pMulti->aSeg[ pMulti->aFirst[1].iFirst ];
    assert( fts5MultiIterEof(p, pMulti)==0 );
    assert( pSeg->nPos>0 );
    if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos+9+9) ){

      if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf 
       && (pColset==0 || pColset->nCol==1)
      ){
        const u8 *pPos = &pSeg->pLeaf->p[pSeg->iLeafOffset];
        int nPos;
        if( pColset ){
          nPos = fts5IndexExtractCol(&pPos, pSeg->nPos, pColset->aiCol[0]);
          if( nPos==0 ) return 1;








|
|








4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203

  Fts5Buffer *pBuf
){
  if( p->rc==SQLITE_OK ){
    Fts5SegIter *pSeg = &pMulti->aSeg[ pMulti->aFirst[1].iFirst ];
    assert( fts5MultiIterEof(p, pMulti)==0 );
    assert( pSeg->nPos>0 );
    if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos+9+9) ){
      if( p->pConfig->bOffsets
       && pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf 
       && (pColset==0 || pColset->nCol==1)
      ){
        const u8 *pPos = &pSeg->pLeaf->p[pSeg->iLeafOffset];
        int nPos;
        if( pColset ){
          nPos = fts5IndexExtractCol(&pPos, pSeg->nPos, pColset->aiCol[0]);
          if( nPos==0 ) return 1;
︙︙
4234
4235
4236
4237
4238
4239
4240
4241
4242
4243
4244
4245
4246

4247
4248
4249
4250
4251
4252
4253

              int nReq = sqlite3Fts5GetVarintLen((u32)(nActual*2));
              while( iSv2<(iData-nReq) ){ pBuf->p[iSv2++] = 0x80; }
              sqlite3Fts5PutVarint(&pBuf->p[iSv2], nActual*2);
            }
          }
        }
      }

    }
  }

  return 0;
}


static void fts5DoclistIterNext(Fts5DoclistIter *pIter){
  u8 *p = pIter->aPoslist + pIter->nSize + pIter->nPoslist;

  assert( pIter->aPoslist );
  if( p>=pIter->aEof ){
    pIter->aPoslist = 0;








<





>








4234
4235
4236
4237
4238
4239
4240

4241
4242
4243
4244
4245
4246
4247
4248
4249
4250
4251
4252
4253

              int nReq = sqlite3Fts5GetVarintLen((u32)(nActual*2));
              while( iSv2<(iData-nReq) ){ pBuf->p[iSv2++] = 0x80; }
              sqlite3Fts5PutVarint(&pBuf->p[iSv2], nActual*2);
            }
          }
        }
      }

    }
  }

  return 0;
}


static void fts5DoclistIterNext(Fts5DoclistIter *pIter){
  u8 *p = pIter->aPoslist + pIter->nSize + pIter->nPoslist;

  assert( pIter->aPoslist );
  if( p>=pIter->aEof ){
    pIter->aPoslist = 0;
︙︙
4386
4387
4388
4389
4390
4391
4392



4393
4394
4395
4396
4397
4398
4399


    fts5BufferSet(&p->rc, p1, out.n, out.p);
    fts5BufferFree(&tmp);
    fts5BufferFree(&out);
  }
}




static void fts5BufferSwap(Fts5Buffer *p1, Fts5Buffer *p2){
  Fts5Buffer tmp = *p1;
  *p1 = *p2;
  *p2 = tmp;
}

static void fts5SetupPrefixIter(








>
>
>








4386
4387
4388
4389
4390
4391
4392
4393
4394
4395
4396
4397
4398
4399
4400
4401
4402


    fts5BufferSet(&p->rc, p1, out.n, out.p);
    fts5BufferFree(&tmp);
    fts5BufferFree(&out);
  }
}

/*
** Swap the contents of buffer *p1 with that of *p2.
*/
static void fts5BufferSwap(Fts5Buffer *p1, Fts5Buffer *p2){
  Fts5Buffer tmp = *p1;
  *p1 = *p2;
  *p2 = tmp;
}

static void fts5SetupPrefixIter(
︙︙
Changes to ext/fts5/test/fts5offsets.test.
︙︙
18
19
20
21
22
23
24



25
26
27
28
29
30
31
32
33
34

# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
  finish_test
  return
}





do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, offsets=0);

  INSERT INTO t1 VALUES('h d g', 'j b b g b', 'i e i d h g g'); -- 1
  INSERT INTO t1 VALUES('h j d', 'j h d a h', 'f d d g g f b'); -- 2
  INSERT INTO t1 VALUES('j c i', 'f f h e f', 'c j i j c h f'); -- 3
  INSERT INTO t1 VALUES('e g g', 'g e d h i', 'e d b e g d c'); -- 4
  INSERT INTO t1 VALUES('b c c', 'd i h a f', 'd i j f a b c'); -- 5
  INSERT INTO t1 VALUES('e d e', 'b c j g d', 'a i f d h b d'); -- 6
  INSERT INTO t1 VALUES('g h e', 'b c d i d', 'e f c i f i c'); -- 7








>
>
>


<








18
19
20
21
22
23
24
25
26
27
28
29

30
31
32
33
34
35
36

# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
  finish_test
  return
}


#--------------------------------------------------------------------------
# Simple tests.
#
do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, offsets=0);

  INSERT INTO t1 VALUES('h d g', 'j b b g b', 'i e i d h g g'); -- 1
  INSERT INTO t1 VALUES('h j d', 'j h d a h', 'f d d g g f b'); -- 2
  INSERT INTO t1 VALUES('j c i', 'f f h e f', 'c j i j c h f'); -- 3
  INSERT INTO t1 VALUES('e g g', 'g e d h i', 'e d b e g d c'); -- 4
  INSERT INTO t1 VALUES('b c c', 'd i h a f', 'd i j f a b c'); -- 5
  INSERT INTO t1 VALUES('e d e', 'b c j g d', 'a i f d h b d'); -- 6
  INSERT INTO t1 VALUES('g h e', 'b c d i d', 'e f c i f i c'); -- 7
︙︙
51
52
53
54
55
56
57








58
59

  } $res

  do_execsql_test 1.2.$tn.2 {
    SELECT rowid FROM t1($match || '*');
  } $res
}









finish_test









>
>
>
>
>
>
>
>



53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69

  } $res

  do_execsql_test 1.2.$tn.2 {
    SELECT rowid FROM t1($match || '*');
  } $res
}

do_catchsql_test 1.3.1 {
  SELECT rowid FROM t1('h + d');
} {1 {fts5: phrase queries are not supported (offsets=0)}}

do_catchsql_test 1.3.2 {
  SELECT rowid FROM t1('NEAR(h d)');
} {1 {fts5: NEAR queries are not supported (offsets=0)}}

finish_test