SQLite

Check-in [f20f9f813f]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add tests for fts5 phrase queries with column filters.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: f20f9f813f00cefcd427e506a09b0b973c782e3f
User & Date: dan 2015-10-07 13:24:27.688
Context
2015-10-07
16:14
Adjustments to spellfix2.test so that it works reliably on mac. (check-in: d591e860d3 user: drh tags: trunk)
13:24
Add tests for fts5 phrase queries with column filters. (check-in: f20f9f813f user: dan tags: trunk)
09:02
Add further tests for fts5 prefix queries. (check-in: accdc98b12 user: dan tags: trunk)
Changes
Unified Diff Ignore Whitespace Patch
Changes to ext/fts5/fts5Int.h.
445
446
447
448
449
450
451









452
453
454
455
456
457
458
int sqlite3Fts5GetVarint32(const unsigned char *p, u32 *v);
int sqlite3Fts5GetVarintLen(u32 iVal);
u8 sqlite3Fts5GetVarint(const unsigned char*, u64*);
int sqlite3Fts5PutVarint(unsigned char *p, u64 v);

#define fts5GetVarint32(a,b) sqlite3Fts5GetVarint32(a,(u32*)&b)
#define fts5GetVarint    sqlite3Fts5GetVarint










/*
** End of interface to code in fts5_varint.c.
**************************************************************************/


/**************************************************************************







>
>
>
>
>
>
>
>
>







445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
int sqlite3Fts5GetVarint32(const unsigned char *p, u32 *v);
int sqlite3Fts5GetVarintLen(u32 iVal);
u8 sqlite3Fts5GetVarint(const unsigned char*, u64*);
int sqlite3Fts5PutVarint(unsigned char *p, u64 v);

#define fts5GetVarint32(a,b) sqlite3Fts5GetVarint32(a,(u32*)&b)
#define fts5GetVarint    sqlite3Fts5GetVarint

#define fts5FastGetVarint32(a, iOff, nVal) {      \
  nVal = (a)[iOff++];                             \
  if( nVal & 0x80 ){                              \
    iOff--;                                       \
    iOff += fts5GetVarint32(&(a)[iOff], nVal);    \
  }                                               \
}


/*
** End of interface to code in fts5_varint.c.
**************************************************************************/


/**************************************************************************
Changes to ext/fts5/fts5_buffer.c.
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
  if( i>=n ){
    /* EOF */
    *piOff = -1;
    return 1;  
  }else{
    i64 iOff = *piOff;
    int iVal;
    i += fts5GetVarint32(&a[i], iVal);
    if( iVal==1 ){
      i += fts5GetVarint32(&a[i], iVal);
      iOff = ((i64)iVal) << 32;
      i += fts5GetVarint32(&a[i], iVal);
    }
    *piOff = iOff + (iVal-2);
    *pi = i;
    return 0;
  }
}








|

|

|







181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
  if( i>=n ){
    /* EOF */
    *piOff = -1;
    return 1;  
  }else{
    i64 iOff = *piOff;
    int iVal;
    fts5FastGetVarint32(a, i, iVal);
    if( iVal==1 ){
      fts5FastGetVarint32(a, i, iVal);
      iOff = ((i64)iVal) << 32;
      fts5FastGetVarint32(a, i, iVal);
    }
    *piOff = iOff + (iVal-2);
    *pi = i;
    return 0;
  }
}

Changes to ext/fts5/fts5_index.c.
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
  ){
    return;
  }

  pIter->pDlidx = fts5DlidxIterInit(p, bRev, iSeg, pIter->iTermLeafPgno);
}

#define fts5IndexGetVarint32(a, iOff, nVal) {     \
  nVal = (a)[iOff++];                             \
  if( nVal & 0x80 ){                              \
    iOff--;                                       \
    iOff += fts5GetVarint32(&(a)[iOff], nVal);    \
  }                                               \
}

#define fts5IndexSkipVarint(a, iOff) {            \
  int iEnd = iOff+9;                              \
  while( (a[iOff++] & 0x80) && iOff<iEnd );       \
}

/*
** The iterator object passed as the second argument currently contains







<
<
<
<
<
<
<
<







1935
1936
1937
1938
1939
1940
1941








1942
1943
1944
1945
1946
1947
1948
  ){
    return;
  }

  pIter->pDlidx = fts5DlidxIterInit(p, bRev, iSeg, pIter->iTermLeafPgno);
}









#define fts5IndexSkipVarint(a, iOff) {            \
  int iEnd = iOff+9;                              \
  while( (a[iOff++] & 0x80) && iOff<iEnd );       \
}

/*
** The iterator object passed as the second argument currently contains
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
  iPgidx = szLeaf;
  iPgidx += fts5GetVarint32(&a[iPgidx], iTermOff);
  iOff = iTermOff;

  while( 1 ){

    /* Figure out how many new bytes are in this term */
    fts5IndexGetVarint32(a, iOff, nNew);
    if( nKeep<nMatch ){
      goto search_failed;
    }

    assert( nKeep>=nMatch );
    if( nKeep==nMatch ){
      int nCmp;







|







1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
  iPgidx = szLeaf;
  iPgidx += fts5GetVarint32(&a[iPgidx], iTermOff);
  iOff = iTermOff;

  while( 1 ){

    /* Figure out how many new bytes are in this term */
    fts5FastGetVarint32(a, iOff, nNew);
    if( nKeep<nMatch ){
      goto search_failed;
    }

    assert( nKeep>=nMatch );
    if( nKeep==nMatch ){
      int nCmp;
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
    }

    iPgidx += fts5GetVarint32(&a[iPgidx], nKeep);
    iTermOff += nKeep;
    iOff = iTermOff;

    /* Read the nKeep field of the next term. */
    fts5IndexGetVarint32(a, iOff, nKeep);
  }

 search_failed:
  if( bGe==0 ){
    fts5DataRelease(pIter->pLeaf);
    pIter->pLeaf = 0;
    return;







|







2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
    }

    iPgidx += fts5GetVarint32(&a[iPgidx], nKeep);
    iTermOff += nKeep;
    iOff = iTermOff;

    /* Read the nKeep field of the next term. */
    fts5FastGetVarint32(a, iOff, nKeep);
  }

 search_failed:
  if( bGe==0 ){
    fts5DataRelease(pIter->pLeaf);
    pIter->pLeaf = 0;
    return;
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
    /* Search through to find the first varint with value 1. This is the
    ** start of the next columns hits. */
    int i = 0;
    int iStart = 0;

    if( pCtx->eState==2 ){
      int iCol;
      fts5IndexGetVarint32(pChunk, i, iCol);
      if( fts5IndexColsetTest(pCtx->pColset, iCol) ){
        pCtx->eState = 1;
        fts5BufferAppendVarint(&p->rc, pCtx->pBuf, 1);
      }else{
        pCtx->eState = 0;
      }
    }







|







3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
    /* Search through to find the first varint with value 1. This is the
    ** start of the next columns hits. */
    int i = 0;
    int iStart = 0;

    if( pCtx->eState==2 ){
      int iCol;
      fts5FastGetVarint32(pChunk, i, iCol);
      if( fts5IndexColsetTest(pCtx->pColset, iCol) ){
        pCtx->eState = 1;
        fts5BufferAppendVarint(&p->rc, pCtx->pBuf, 1);
      }else{
        pCtx->eState = 0;
      }
    }
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
      if( i<nChunk ){
        int iCol;
        iStart = i;
        i++;
        if( i>=nChunk ){
          pCtx->eState = 2;
        }else{
          fts5IndexGetVarint32(pChunk, i, iCol);
          pCtx->eState = fts5IndexColsetTest(pCtx->pColset, iCol);
          if( pCtx->eState ){
            fts5BufferAppendBlob(&p->rc, pCtx->pBuf, i-iStart, &pChunk[iStart]);
            iStart = i;
          }
        }
      }







|







3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
      if( i<nChunk ){
        int iCol;
        iStart = i;
        i++;
        if( i>=nChunk ){
          pCtx->eState = 2;
        }else{
          fts5FastGetVarint32(pChunk, i, iCol);
          pCtx->eState = fts5IndexColsetTest(pCtx->pColset, iCol);
          if( pCtx->eState ){
            fts5BufferAppendBlob(&p->rc, pCtx->pBuf, i-iStart, &pChunk[iStart]);
            iStart = i;
          }
        }
      }
Added ext/fts5/test/fts5phrase.test.














































































































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
# 2014 Jan 08
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#
# Tests focused on phrase queries.
#

source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5phrase

# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
  finish_test
  return
}

do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE t3 USING fts5(a, b, c);
  INSERT INTO t3 VALUES('d e a', 'd i j j f', 'i j i e b f h'); -- 1
  INSERT INTO t3 VALUES('g a e', 'f g i g a', 'h d g i g h c'); -- 2
  INSERT INTO t3 VALUES('e a d', 'e i h a f', 'c e h i f b i'); -- 3
  INSERT INTO t3 VALUES('a g c', 'd j d j c', 'c d f j i g j'); -- 4
  INSERT INTO t3 VALUES('b c b', 'j g c d f', 'j c j d g f b'); -- 5
  INSERT INTO t3 VALUES('j a d', 'e b i h h', 'c c f g d i d'); -- 6
  INSERT INTO t3 VALUES('a d f', 'h g i i i', 'e a g c i f b'); -- 7
  INSERT INTO t3 VALUES('g f d', 'f c g b j', 'b b h h h j j'); -- 8
  INSERT INTO t3 VALUES('f h g', 'c j f g j', 'd h d f e b h'); -- 9
  INSERT INTO t3 VALUES('f h d', 'c i a d b', 'g b j b a d e'); -- 10
  INSERT INTO t3 VALUES('j h h', 'j i h a g', 'd e i e a g j'); -- 11
  INSERT INTO t3 VALUES('a b e', 'h g a g c', 'h c a a d e g'); -- 12
  INSERT INTO t3 VALUES('a j g', 'i h i f i', 'a g h j g i b'); -- 13
  INSERT INTO t3 VALUES('j h e', 'f e d i e', 'i d c f e d c'); -- 14
  INSERT INTO t3 VALUES('d j d', 'd b i a c', 'g d h i d b e'); -- 15
  INSERT INTO t3 VALUES('h j e', 'e b b c f', 'j a f g h d j'); -- 16
  INSERT INTO t3 VALUES('c b j', 'c a b a i', 'h f i d a d c'); -- 17
  INSERT INTO t3 VALUES('e e d', 'i d f c c', 'g i d a f e a'); -- 18
  INSERT INTO t3 VALUES('e i g', 'e a b i h', 'i f d d a d f'); -- 19
  INSERT INTO t3 VALUES('h g f', 'b h h j d', 'i f d e g j a'); -- 20
  INSERT INTO t3 VALUES('e h f', 'j c b c f', 'j a j g h a c'); -- 21
  INSERT INTO t3 VALUES('d c h', 'b g i c e', 'i i c d e h i'); -- 22
  INSERT INTO t3 VALUES('a h i', 'a g d f f', 'e f i i b b h'); -- 23
  INSERT INTO t3 VALUES('d d g', 'c c b c g', 'g c h e b c e'); -- 24
  INSERT INTO t3 VALUES('a b b', 'b f a d i', 'd a h a b c i'); -- 25
  INSERT INTO t3 VALUES('a f d', 'a j e a h', 'j i h j a i f'); -- 26
  INSERT INTO t3 VALUES('d j d', 'h a d i a', 'h h f j h g a'); -- 27
  INSERT INTO t3 VALUES('g a e', 'd g f a g', 'i d b c g g j'); -- 28
  INSERT INTO t3 VALUES('j e h', 'g h j h g', 'd a e j a a h'); -- 29
  INSERT INTO t3 VALUES('e j e', 'g e j g c', 'f c e b e e a'); -- 30
  INSERT INTO t3 VALUES('h f f', 'i j g e c', 'j j f c a i j'); -- 31
  INSERT INTO t3 VALUES('a g c', 'c g d b i', 'g h c b a a f'); -- 32
  INSERT INTO t3 VALUES('c h i', 'j d h e e', 'a h i d c c j'); -- 33
  INSERT INTO t3 VALUES('d a c', 'e d d b j', 'c e b b h i h'); -- 34
  INSERT INTO t3 VALUES('d f h', 'c a f c c', 'j b b c c j f'); -- 35
  INSERT INTO t3 VALUES('b g h', 'g c c c f', 'c g c f h e e'); -- 36
  INSERT INTO t3 VALUES('f e a', 'b h f j h', 'j g h f d g f'); -- 37
  INSERT INTO t3 VALUES('h f a', 'a e i j g', 'f d a f d f c'); -- 38
  INSERT INTO t3 VALUES('f i c', 'f i i i i', 'e c f d h j f'); -- 39
  INSERT INTO t3 VALUES('h h d', 'd i e d i', 'd f e i a h a'); -- 40
  INSERT INTO t3 VALUES('f g c', 'd a f c h', 'b b g j c e g'); -- 41
  INSERT INTO t3 VALUES('h i h', 'h d j d e', 'e d b b i e g'); -- 42
  INSERT INTO t3 VALUES('b h i', 'j e i d a', 'j j h e e c a'); -- 43
  INSERT INTO t3 VALUES('g i g', 'f c c f d', 'a c i c a d a'); -- 44
  INSERT INTO t3 VALUES('c c f', 'a b j d b', 'c a e g f e c'); -- 45
  INSERT INTO t3 VALUES('d h j', 'g c b j d', 'e a h f h j g'); -- 46
  INSERT INTO t3 VALUES('a a d', 'j e j a i', 'i d c f f f b'); -- 47
  INSERT INTO t3 VALUES('b g j', 'e c i h f', 'd d h b g a d'); -- 48
  INSERT INTO t3 VALUES('c i a', 'a c c c c', 'e h i e h i e'); -- 49
  INSERT INTO t3 VALUES('f f c', 'f f b i i', 'f f a j e c i'); -- 50
}

proc pmatch {col expr} {
  return [expr {[string first $expr $col]>=0}]
}
db func pmatch pmatch

foreach {tn cols tokens} {
  1 a         "c c"
  2 b         "c c"
  3 c         "c c"
  4 {a b c}   "c c"
  5 {a b c}   "b h"
  6 {a b}     "b h"
  7 {a c}     "b h"
  8 {c a}     "b h"
  9 {c}       "i e"
  10 {b}      "i e"
  11 {a}      "i e"
} {
  set fts   "{$cols}:[join $tokens +]"
  set where [list]
  foreach c $cols { lappend where "pmatch($c, '$tokens')" }
  set where [join $where " OR "]

  set res [db eval "SELECT rowid FROM t3 WHERE $where"]
  do_execsql_test "1.$tn.$fts->([llength $res] rows)" { 
    SELECT rowid FROM t3($fts) 
  } $res
}

do_execsql_test 2.0 {
  SELECT rowid,
    highlight(t3, 0, '*', '*'),
    highlight(t3, 1, '*', '*'),
    highlight(t3, 2, '*', '*')
  FROM t3('a:f+f')
} {
  31 {h *f f*} {i j g e c} {j j f c a i j} 
  50 {*f f* c} {f f b i i} {f f a j e c i}
}

finish_test