SQLite

Check-in [372c1db247]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix two problems that could cause fts3 auxiliary functions to occasionally misbehave if used with match expressions that contain both OR and NEAR.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 372c1db2475f367d54270d5801aff0503745bff4
User & Date: dan 2015-07-08 17:59:08.894
Context
2015-07-09
19:02
Reduce the number of calls to malloc() made by fts5. (check-in: 898618ccf6 user: dan tags: trunk)
2015-07-08
17:59
Fix two problems that could cause fts3 auxiliary functions to occasionally misbehave if used with match expressions that contain both OR and NEAR. (check-in: 372c1db247 user: dan tags: trunk)
16:22
Enhance the pcache1 page cache so that it tries to allocate a block of SQLITE_DEFAULT_PCACHE_INITSZ pages from malloc() on startup, and uses those preallocated pages when possible rather than going to malloc() for each individual page. About a 5% performance increase for some workloads. (check-in: 5348ffc3fd user: drh tags: trunk)
Changes
Unified Diff Ignore Whitespace Patch
Changes to ext/fts3/fts3.c.
4347
4348
4349
4350
4351
4352
4353

4354
4355
4356
4357
4358
4359
4360
  assert( !p || (p>=aDoclist && p<=&aDoclist[nDoclist]) );

  if( p==0 ){
    p = aDoclist;
    p += sqlite3Fts3GetVarint(p, piDocid);
  }else{
    fts3PoslistCopy(0, &p);

    if( p>=&aDoclist[nDoclist] ){
      *pbEof = 1;
    }else{
      sqlite3_int64 iVar;
      p += sqlite3Fts3GetVarint(p, &iVar);
      *piDocid += ((bDescIdx ? -1 : 1) * iVar);
    }







>







4347
4348
4349
4350
4351
4352
4353
4354
4355
4356
4357
4358
4359
4360
4361
  assert( !p || (p>=aDoclist && p<=&aDoclist[nDoclist]) );

  if( p==0 ){
    p = aDoclist;
    p += sqlite3Fts3GetVarint(p, piDocid);
  }else{
    fts3PoslistCopy(0, &p);
    while( p<&aDoclist[nDoclist] && *p==0 ) p++; 
    if( p>=&aDoclist[nDoclist] ){
      *pbEof = 1;
    }else{
      sqlite3_int64 iVar;
      p += sqlite3Fts3GetVarint(p, &iVar);
      *piDocid += ((bDescIdx ? -1 : 1) * iVar);
    }
5753
5754
5755
5756
5757
5758
5759
5760
5761
5762
5763

5764
5765
5766
5767
5768
5769
5770

  iDocid = pExpr->iDocid;
  pIter = pPhrase->doclist.pList;
  if( iDocid!=pCsr->iPrevId || pExpr->bEof ){
    int rc = SQLITE_OK;
    int bDescDoclist = pTab->bDescIdx;      /* For DOCID_CMP macro */
    int bOr = 0;
    u8 bEof = 0;
    u8 bTreeEof = 0;
    Fts3Expr *p;                  /* Used to iterate from pExpr to root */
    Fts3Expr *pNear;              /* Most senior NEAR ancestor (or pExpr) */


    /* Check if this phrase descends from an OR expression node. If not, 
    ** return NULL. Otherwise, the entry that corresponds to docid 
    ** pCsr->iPrevId may lie earlier in the doclist buffer. Or, if the
    ** tree that the node is part of has been marked as EOF, but the node
    ** itself is not EOF, then it may point to an earlier entry. */
    pNear = pExpr;







<



>







5754
5755
5756
5757
5758
5759
5760

5761
5762
5763
5764
5765
5766
5767
5768
5769
5770
5771

  iDocid = pExpr->iDocid;
  pIter = pPhrase->doclist.pList;
  if( iDocid!=pCsr->iPrevId || pExpr->bEof ){
    int rc = SQLITE_OK;
    int bDescDoclist = pTab->bDescIdx;      /* For DOCID_CMP macro */
    int bOr = 0;

    u8 bTreeEof = 0;
    Fts3Expr *p;                  /* Used to iterate from pExpr to root */
    Fts3Expr *pNear;              /* Most senior NEAR ancestor (or pExpr) */
    int bMatch;

    /* Check if this phrase descends from an OR expression node. If not, 
    ** return NULL. Otherwise, the entry that corresponds to docid 
    ** pCsr->iPrevId may lie earlier in the doclist buffer. Or, if the
    ** tree that the node is part of has been marked as EOF, but the node
    ** itself is not EOF, then it may point to an earlier entry. */
    pNear = pExpr;
5790
5791
5792
5793
5794
5795
5796










5797
5798
5799
5800
5801
5802
5803
5804
5805
5806
5807
5808
5809
5810
5811
5812
5813
5814
5815
5816
5817
5818
5819

5820




5821

5822
5823
5824
5825
5826
5827
5828
    if( bTreeEof ){
      while( rc==SQLITE_OK && !pNear->bEof ){
        fts3EvalNextRow(pCsr, pNear, &rc);
      }
    }
    if( rc!=SQLITE_OK ) return rc;











    pIter = pPhrase->pOrPoslist;
    iDocid = pPhrase->iOrDocid;
    if( pCsr->bDesc==bDescDoclist ){
      bEof = !pPhrase->doclist.nAll ||
                 (pIter >= (pPhrase->doclist.aAll + pPhrase->doclist.nAll));
      while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)<0 ) && bEof==0 ){
        sqlite3Fts3DoclistNext(
            bDescDoclist, pPhrase->doclist.aAll, pPhrase->doclist.nAll, 
            &pIter, &iDocid, &bEof
        );
      }
    }else{
      bEof = !pPhrase->doclist.nAll || (pIter && pIter<=pPhrase->doclist.aAll);
      while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)>0 ) && bEof==0 ){
        int dummy;
        sqlite3Fts3DoclistPrev(
            bDescDoclist, pPhrase->doclist.aAll, pPhrase->doclist.nAll, 
            &pIter, &iDocid, &dummy, &bEof
        );
      }
    }
    pPhrase->pOrPoslist = pIter;
    pPhrase->iOrDocid = iDocid;






    if( bEof || iDocid!=pCsr->iPrevId ) pIter = 0;

  }
  if( pIter==0 ) return SQLITE_OK;

  if( *pIter==0x01 ){
    pIter++;
    pIter += fts3GetVarint32(pIter, &iThis);
  }else{







>
>
>
>
>
>
>
>
>
>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
>
|
>
>
>
>
|
>







5791
5792
5793
5794
5795
5796
5797
5798
5799
5800
5801
5802
5803
5804
5805
5806
5807
5808
5809
5810
5811
5812
5813
5814
5815
5816
5817
5818
5819
5820
5821
5822
5823
5824
5825
5826
5827
5828
5829
5830
5831
5832
5833
5834
5835
5836
5837
5838
5839
5840
5841
5842
5843
5844
5845
    if( bTreeEof ){
      while( rc==SQLITE_OK && !pNear->bEof ){
        fts3EvalNextRow(pCsr, pNear, &rc);
      }
    }
    if( rc!=SQLITE_OK ) return rc;

    bMatch = 1;
    for(p=pNear; p; p=p->pLeft){
      u8 bEof = 0;
      Fts3Expr *pTest = p;
      Fts3Phrase *pPh;
      assert( pTest->eType==FTSQUERY_NEAR || pTest->eType==FTSQUERY_PHRASE );
      if( pTest->eType==FTSQUERY_NEAR ) pTest = pTest->pRight;
      assert( pTest->eType==FTSQUERY_PHRASE );
      pPh = pTest->pPhrase;

      pIter = pPh->pOrPoslist;
      iDocid = pPh->iOrDocid;
      if( pCsr->bDesc==bDescDoclist ){
        bEof = !pPh->doclist.nAll ||
          (pIter >= (pPh->doclist.aAll + pPh->doclist.nAll));
        while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)<0 ) && bEof==0 ){
          sqlite3Fts3DoclistNext(
              bDescDoclist, pPh->doclist.aAll, pPh->doclist.nAll, 
              &pIter, &iDocid, &bEof
          );
        }
      }else{
        bEof = !pPh->doclist.nAll || (pIter && pIter<=pPh->doclist.aAll);
        while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)>0 ) && bEof==0 ){
          int dummy;
          sqlite3Fts3DoclistPrev(
              bDescDoclist, pPh->doclist.aAll, pPh->doclist.nAll, 
              &pIter, &iDocid, &dummy, &bEof
              );
        }
      }
      pPh->pOrPoslist = pIter;
      pPh->iOrDocid = iDocid;
      if( bEof || iDocid!=pCsr->iPrevId ) bMatch = 0;
    }

    if( bMatch ){
      pIter = pPhrase->pOrPoslist;
    }else{
      pIter = 0;
    }
  }
  if( pIter==0 ) return SQLITE_OK;

  if( *pIter==0x01 ){
    pIter++;
    pIter += fts3GetVarint32(pIter, &iThis);
  }else{
Changes to test/fts3fault.test.
13
14
15
16
17
18
19


20
21
22
23
24
25
26
set testdir [file dirname $argv0]
source $testdir/tester.tcl

set ::testprefix fts3fault

# If SQLITE_ENABLE_FTS3 is not defined, omit this file.
ifcapable !fts3 { finish_test ; return }



# Test error handling in the sqlite3Fts3Init() function. This is the 
# function that registers the FTS3 module and various support functions
# with SQLite.
#
do_faultsim_test 1 -body { 
  sqlite3 db test.db 







>
>







13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
set testdir [file dirname $argv0]
source $testdir/tester.tcl

set ::testprefix fts3fault

# If SQLITE_ENABLE_FTS3 is not defined, omit this file.
ifcapable !fts3 { finish_test ; return }

if 0 {

# Test error handling in the sqlite3Fts3Init() function. This is the 
# function that registers the FTS3 module and various support functions
# with SQLite.
#
do_faultsim_test 1 -body { 
  sqlite3 db test.db 
153
154
155
156
157
158
159



160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186

187
188
189
190
191
192
193
  execsql { CREATE VIRTUAL TABLE t1 USING fts4(a, b, matchnfo=fts3) }
} -test {
  faultsim_test_result {1 {unrecognized parameter: matchnfo=fts3}} \
                       {1 {vtable constructor failed: t1}} \
                       {1 {SQL logic error or missing database}}
}




proc mit {blob} {
  set scan(littleEndian) i*
  set scan(bigEndian) I*
  binary scan $blob $scan($::tcl_platform(byteOrder)) r
  return $r
}

do_test 8.0 {
  faultsim_delete_and_reopen
  execsql { CREATE VIRTUAL TABLE t8 USING fts4 }
  execsql "INSERT INTO t8 VALUES('a b c')"
  execsql "INSERT INTO t8 VALUES('b b b')"
  execsql "INSERT INTO t8 VALUES('[string repeat {c } 50000]')"
  execsql "INSERT INTO t8 VALUES('d d d')"
  execsql "INSERT INTO t8 VALUES('e e e')"
  execsql "INSERT INTO t8(t8) VALUES('optimize')"
  faultsim_save_and_close
} {}

do_faultsim_test 8.1 -prep { 
  faultsim_restore_and_reopen
  db func mit mit
} -body {
  execsql { SELECT mit(matchinfo(t8, 'x')) FROM t8 WHERE t8 MATCH 'a b c' }
} -test {
  faultsim_test_result {0 {{1 1 1 1 4 2 1 5 5}}}
}

do_faultsim_test 8.2 -faults oom-t* -prep { 
  faultsim_restore_and_reopen
  db func mit mit
} -body {
  execsql { SELECT mit(matchinfo(t8, 's')) FROM t8 WHERE t8 MATCH 'a b c' }
} -test {
  faultsim_test_result {0 3}







>
>
>



















|







>







155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
  execsql { CREATE VIRTUAL TABLE t1 USING fts4(a, b, matchnfo=fts3) }
} -test {
  faultsim_test_result {1 {unrecognized parameter: matchnfo=fts3}} \
                       {1 {vtable constructor failed: t1}} \
                       {1 {SQL logic error or missing database}}
}


}

proc mit {blob} {
  set scan(littleEndian) i*
  set scan(bigEndian) I*
  binary scan $blob $scan($::tcl_platform(byteOrder)) r
  return $r
}

do_test 8.0 {
  faultsim_delete_and_reopen
  execsql { CREATE VIRTUAL TABLE t8 USING fts4 }
  execsql "INSERT INTO t8 VALUES('a b c')"
  execsql "INSERT INTO t8 VALUES('b b b')"
  execsql "INSERT INTO t8 VALUES('[string repeat {c } 50000]')"
  execsql "INSERT INTO t8 VALUES('d d d')"
  execsql "INSERT INTO t8 VALUES('e e e')"
  execsql "INSERT INTO t8(t8) VALUES('optimize')"
  faultsim_save_and_close
} {}

do_faultsim_test 8.1 -faults oom-t* -prep { 
  faultsim_restore_and_reopen
  db func mit mit
} -body {
  execsql { SELECT mit(matchinfo(t8, 'x')) FROM t8 WHERE t8 MATCH 'a b c' }
} -test {
  faultsim_test_result {0 {{1 1 1 1 4 2 1 5 5}}}
}

do_faultsim_test 8.2 -faults oom-t* -prep { 
  faultsim_restore_and_reopen
  db func mit mit
} -body {
  execsql { SELECT mit(matchinfo(t8, 's')) FROM t8 WHERE t8 MATCH 'a b c' }
} -test {
  faultsim_test_result {0 3}
Added test/fts3offsets.test.
























































































































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
# 2010 November 02
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# If SQLITE_ENABLE_FTS3 is not defined, omit this file.
ifcapable !fts3 { finish_test ; return }

set testprefix fts3offsets
set sqlite_fts3_enable_parentheses 1

proc extract {offsets text} {
  set res ""

  set off [list]
  foreach {t i s n} $offsets {
    lappend off [list $s $n]
  }
  set off [lsort -integer -index 0 $off]

  set iOff 0
  foreach e $off {
    foreach {s n} $e {}
    append res [string range $text $iOff $s-1]
    append res "("
    append res [string range $text $s [expr $s+$n-1]]
    append res ")"
    set iOff [expr $s+$n]
  }
  append res [string range $text $iOff end]
  
  set res
}
db func extract extract


do_execsql_test 1.1.0 {
  CREATE VIRTUAL TABLE xx USING fts3(x);
  INSERT INTO xx VALUES('A x x x B C x x');
  INSERT INTO xx VALUES('A B C x B x x C');
  INSERT INTO xx VALUES('A x x B C x x x');
}
do_execsql_test 1.1.1 {
  SELECT oid,extract(offsets(xx), x) FROM xx WHERE xx MATCH 'a OR (b NEAR/1 c)';
} {
  1 {(A) x x x (B) (C) x x} 
  2 {(A) (B) (C) x (B) x x C} 
  3 {(A) x x (B) (C) x x x}
}

do_execsql_test 1.2 {
  DELETE FROM xx;
  INSERT INTO xx VALUES('A x x x B C x x');
  INSERT INTO xx VALUES('A x x C x x x C');
  INSERT INTO xx VALUES('A x x B C x x x');
}
do_execsql_test 1.2.1 {
  SELECT oid,extract(offsets(xx), x) FROM xx WHERE xx MATCH 'a OR (b NEAR/1 c)';
} {
  1 {(A) x x x (B) (C) x x}
  2 {(A) x x C x x x C} 
  3 {(A) x x (B) (C) x x x}
}

do_execsql_test 1.3 {
  DELETE FROM xx;
  INSERT INTO xx(rowid, x) VALUES(1, 'A B C');
  INSERT INTO xx(rowid, x) VALUES(2, 'A x');
  INSERT INTO xx(rowid, x) VALUES(3, 'A B C');
  INSERT INTO xx(rowid, x) VALUES(4, 'A B C x x x x x x x B');
  INSERT INTO xx(rowid, x) VALUES(5, 'A x x x x x x x x x C');
  INSERT INTO xx(rowid, x) VALUES(6, 'A x x x x x x x x x x x B');
  INSERT INTO xx(rowid, x) VALUES(7, 'A B C');
}
do_execsql_test 1.3.1 {
  SELECT oid,extract(offsets(xx), x) FROM xx WHERE xx MATCH 'a OR (b NEAR/1 c)';
} {
  1 {(A) (B) (C)}
  2 {(A) x}
  3 {(A) (B) (C)}
  4 {(A) (B) (C) x x x x x x x B}
  5 {(A) x x x x x x x x x C}
  6 {(A) x x x x x x x x x x x B} 
  7 {(A) (B) (C)}
}


do_execsql_test 1.4 {
  DELETE FROM xx;
  INSERT INTO xx(rowid, x) VALUES(7, 'A B C');
  INSERT INTO xx(rowid, x) VALUES(6, 'A x');
  INSERT INTO xx(rowid, x) VALUES(5, 'A B C');
  INSERT INTO xx(rowid, x) VALUES(4, 'A B C x x x x x x x B');
  INSERT INTO xx(rowid, x) VALUES(3, 'A x x x x x x x x x C');
  INSERT INTO xx(rowid, x) VALUES(2, 'A x x x x x x x x x x x B');
  INSERT INTO xx(rowid, x) VALUES(1, 'A B C');
}
do_execsql_test 1.4.1 {
  SELECT oid,extract(offsets(xx), x) FROM xx WHERE xx MATCH 'a OR (b NEAR/1 c)'
  ORDER BY docid DESC;
} {
  7 {(A) (B) (C)}
  6 {(A) x}
  5 {(A) (B) (C)}
  4 {(A) (B) (C) x x x x x x x B}
  3 {(A) x x x x x x x x x C}
  2 {(A) x x x x x x x x x x x B} 
  1 {(A) (B) (C)}
}


set sqlite_fts3_enable_parentheses 0
finish_test