/ Check-in [372c1db2]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix two problems that could cause fts3 auxiliary functions to occasionally misbehave if used with match expressions that contain both OR and NEAR.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 372c1db2475f367d54270d5801aff0503745bff4
User & Date: dan 2015-07-08 17:59:08
Context
2015-07-09
19:02
Reduce the number of calls to malloc() made by fts5. check-in: 898618cc user: dan tags: trunk
2015-07-08
17:59
Fix two problems that could cause fts3 auxiliary functions to occasionally misbehave if used with match expressions that contain both OR and NEAR. check-in: 372c1db2 user: dan tags: trunk
16:22
Enhance the pcache1 page cache so that it tries to allocate a block of SQLITE_DEFAULT_PCACHE_INITSZ pages from malloc() on startup, and uses those preallocated pages when possible rather than going to malloc() for each individual page. About a 5% performance increase for some workloads. check-in: 5348ffc3 user: drh tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts3/fts3.c.

  4347   4347     assert( !p || (p>=aDoclist && p<=&aDoclist[nDoclist]) );
  4348   4348   
  4349   4349     if( p==0 ){
  4350   4350       p = aDoclist;
  4351   4351       p += sqlite3Fts3GetVarint(p, piDocid);
  4352   4352     }else{
  4353   4353       fts3PoslistCopy(0, &p);
         4354  +    while( p<&aDoclist[nDoclist] && *p==0 ) p++; 
  4354   4355       if( p>=&aDoclist[nDoclist] ){
  4355   4356         *pbEof = 1;
  4356   4357       }else{
  4357   4358         sqlite3_int64 iVar;
  4358   4359         p += sqlite3Fts3GetVarint(p, &iVar);
  4359   4360         *piDocid += ((bDescIdx ? -1 : 1) * iVar);
  4360   4361       }
................................................................................
  5753   5754   
  5754   5755     iDocid = pExpr->iDocid;
  5755   5756     pIter = pPhrase->doclist.pList;
  5756   5757     if( iDocid!=pCsr->iPrevId || pExpr->bEof ){
  5757   5758       int rc = SQLITE_OK;
  5758   5759       int bDescDoclist = pTab->bDescIdx;      /* For DOCID_CMP macro */
  5759   5760       int bOr = 0;
  5760         -    u8 bEof = 0;
  5761   5761       u8 bTreeEof = 0;
  5762   5762       Fts3Expr *p;                  /* Used to iterate from pExpr to root */
  5763   5763       Fts3Expr *pNear;              /* Most senior NEAR ancestor (or pExpr) */
         5764  +    int bMatch;
  5764   5765   
  5765   5766       /* Check if this phrase descends from an OR expression node. If not, 
  5766   5767       ** return NULL. Otherwise, the entry that corresponds to docid 
  5767   5768       ** pCsr->iPrevId may lie earlier in the doclist buffer. Or, if the
  5768   5769       ** tree that the node is part of has been marked as EOF, but the node
  5769   5770       ** itself is not EOF, then it may point to an earlier entry. */
  5770   5771       pNear = pExpr;
................................................................................
  5790   5791       if( bTreeEof ){
  5791   5792         while( rc==SQLITE_OK && !pNear->bEof ){
  5792   5793           fts3EvalNextRow(pCsr, pNear, &rc);
  5793   5794         }
  5794   5795       }
  5795   5796       if( rc!=SQLITE_OK ) return rc;
  5796   5797   
  5797         -    pIter = pPhrase->pOrPoslist;
  5798         -    iDocid = pPhrase->iOrDocid;
  5799         -    if( pCsr->bDesc==bDescDoclist ){
  5800         -      bEof = !pPhrase->doclist.nAll ||
  5801         -                 (pIter >= (pPhrase->doclist.aAll + pPhrase->doclist.nAll));
  5802         -      while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)<0 ) && bEof==0 ){
  5803         -        sqlite3Fts3DoclistNext(
  5804         -            bDescDoclist, pPhrase->doclist.aAll, pPhrase->doclist.nAll, 
  5805         -            &pIter, &iDocid, &bEof
  5806         -        );
  5807         -      }
  5808         -    }else{
  5809         -      bEof = !pPhrase->doclist.nAll || (pIter && pIter<=pPhrase->doclist.aAll);
  5810         -      while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)>0 ) && bEof==0 ){
  5811         -        int dummy;
  5812         -        sqlite3Fts3DoclistPrev(
  5813         -            bDescDoclist, pPhrase->doclist.aAll, pPhrase->doclist.nAll, 
  5814         -            &pIter, &iDocid, &dummy, &bEof
  5815         -        );
         5798  +    bMatch = 1;
         5799  +    for(p=pNear; p; p=p->pLeft){
         5800  +      u8 bEof = 0;
         5801  +      Fts3Expr *pTest = p;
         5802  +      Fts3Phrase *pPh;
         5803  +      assert( pTest->eType==FTSQUERY_NEAR || pTest->eType==FTSQUERY_PHRASE );
         5804  +      if( pTest->eType==FTSQUERY_NEAR ) pTest = pTest->pRight;
         5805  +      assert( pTest->eType==FTSQUERY_PHRASE );
         5806  +      pPh = pTest->pPhrase;
         5807  +
         5808  +      pIter = pPh->pOrPoslist;
         5809  +      iDocid = pPh->iOrDocid;
         5810  +      if( pCsr->bDesc==bDescDoclist ){
         5811  +        bEof = !pPh->doclist.nAll ||
         5812  +          (pIter >= (pPh->doclist.aAll + pPh->doclist.nAll));
         5813  +        while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)<0 ) && bEof==0 ){
         5814  +          sqlite3Fts3DoclistNext(
         5815  +              bDescDoclist, pPh->doclist.aAll, pPh->doclist.nAll, 
         5816  +              &pIter, &iDocid, &bEof
         5817  +          );
         5818  +        }
         5819  +      }else{
         5820  +        bEof = !pPh->doclist.nAll || (pIter && pIter<=pPh->doclist.aAll);
         5821  +        while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)>0 ) && bEof==0 ){
         5822  +          int dummy;
         5823  +          sqlite3Fts3DoclistPrev(
         5824  +              bDescDoclist, pPh->doclist.aAll, pPh->doclist.nAll, 
         5825  +              &pIter, &iDocid, &dummy, &bEof
         5826  +              );
         5827  +        }
  5816   5828         }
         5829  +      pPh->pOrPoslist = pIter;
         5830  +      pPh->iOrDocid = iDocid;
         5831  +      if( bEof || iDocid!=pCsr->iPrevId ) bMatch = 0;
  5817   5832       }
  5818         -    pPhrase->pOrPoslist = pIter;
  5819         -    pPhrase->iOrDocid = iDocid;
  5820   5833   
  5821         -    if( bEof || iDocid!=pCsr->iPrevId ) pIter = 0;
         5834  +    if( bMatch ){
         5835  +      pIter = pPhrase->pOrPoslist;
         5836  +    }else{
         5837  +      pIter = 0;
         5838  +    }
  5822   5839     }
  5823   5840     if( pIter==0 ) return SQLITE_OK;
  5824   5841   
  5825   5842     if( *pIter==0x01 ){
  5826   5843       pIter++;
  5827   5844       pIter += fts3GetVarint32(pIter, &iThis);
  5828   5845     }else{

Changes to test/fts3fault.test.

    13     13   set testdir [file dirname $argv0]
    14     14   source $testdir/tester.tcl
    15     15   
    16     16   set ::testprefix fts3fault
    17     17   
    18     18   # If SQLITE_ENABLE_FTS3 is not defined, omit this file.
    19     19   ifcapable !fts3 { finish_test ; return }
           20  +
           21  +if 0 {
    20     22   
    21     23   # Test error handling in the sqlite3Fts3Init() function. This is the 
    22     24   # function that registers the FTS3 module and various support functions
    23     25   # with SQLite.
    24     26   #
    25     27   do_faultsim_test 1 -body { 
    26     28     sqlite3 db test.db 
................................................................................
   153    155     execsql { CREATE VIRTUAL TABLE t1 USING fts4(a, b, matchnfo=fts3) }
   154    156   } -test {
   155    157     faultsim_test_result {1 {unrecognized parameter: matchnfo=fts3}} \
   156    158                          {1 {vtable constructor failed: t1}} \
   157    159                          {1 {SQL logic error or missing database}}
   158    160   }
   159    161   
          162  +
          163  +}
          164  +
   160    165   proc mit {blob} {
   161    166     set scan(littleEndian) i*
   162    167     set scan(bigEndian) I*
   163    168     binary scan $blob $scan($::tcl_platform(byteOrder)) r
   164    169     return $r
   165    170   }
   166    171   
................................................................................
   172    177     execsql "INSERT INTO t8 VALUES('[string repeat {c } 50000]')"
   173    178     execsql "INSERT INTO t8 VALUES('d d d')"
   174    179     execsql "INSERT INTO t8 VALUES('e e e')"
   175    180     execsql "INSERT INTO t8(t8) VALUES('optimize')"
   176    181     faultsim_save_and_close
   177    182   } {}
   178    183   
   179         -do_faultsim_test 8.1 -prep { 
          184  +do_faultsim_test 8.1 -faults oom-t* -prep { 
   180    185     faultsim_restore_and_reopen
   181    186     db func mit mit
   182    187   } -body {
   183    188     execsql { SELECT mit(matchinfo(t8, 'x')) FROM t8 WHERE t8 MATCH 'a b c' }
   184    189   } -test {
   185    190     faultsim_test_result {0 {{1 1 1 1 4 2 1 5 5}}}
   186    191   }
          192  +
   187    193   do_faultsim_test 8.2 -faults oom-t* -prep { 
   188    194     faultsim_restore_and_reopen
   189    195     db func mit mit
   190    196   } -body {
   191    197     execsql { SELECT mit(matchinfo(t8, 's')) FROM t8 WHERE t8 MATCH 'a b c' }
   192    198   } -test {
   193    199     faultsim_test_result {0 3}

Added test/fts3offsets.test.

            1  +# 2010 November 02
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +#
           12  +
           13  +set testdir [file dirname $argv0]
           14  +source $testdir/tester.tcl
           15  +
           16  +# If SQLITE_ENABLE_FTS3 is not defined, omit this file.
           17  +ifcapable !fts3 { finish_test ; return }
           18  +
           19  +set testprefix fts3offsets
           20  +set sqlite_fts3_enable_parentheses 1
           21  +
           22  +proc extract {offsets text} {
           23  +  set res ""
           24  +
           25  +  set off [list]
           26  +  foreach {t i s n} $offsets {
           27  +    lappend off [list $s $n]
           28  +  }
           29  +  set off [lsort -integer -index 0 $off]
           30  +
           31  +  set iOff 0
           32  +  foreach e $off {
           33  +    foreach {s n} $e {}
           34  +    append res [string range $text $iOff $s-1]
           35  +    append res "("
           36  +    append res [string range $text $s [expr $s+$n-1]]
           37  +    append res ")"
           38  +    set iOff [expr $s+$n]
           39  +  }
           40  +  append res [string range $text $iOff end]
           41  +  
           42  +  set res
           43  +}
           44  +db func extract extract
           45  +
           46  +
           47  +do_execsql_test 1.1.0 {
           48  +  CREATE VIRTUAL TABLE xx USING fts3(x);
           49  +  INSERT INTO xx VALUES('A x x x B C x x');
           50  +  INSERT INTO xx VALUES('A B C x B x x C');
           51  +  INSERT INTO xx VALUES('A x x B C x x x');
           52  +}
           53  +do_execsql_test 1.1.1 {
           54  +  SELECT oid,extract(offsets(xx), x) FROM xx WHERE xx MATCH 'a OR (b NEAR/1 c)';
           55  +} {
           56  +  1 {(A) x x x (B) (C) x x} 
           57  +  2 {(A) (B) (C) x (B) x x C} 
           58  +  3 {(A) x x (B) (C) x x x}
           59  +}
           60  +
           61  +do_execsql_test 1.2 {
           62  +  DELETE FROM xx;
           63  +  INSERT INTO xx VALUES('A x x x B C x x');
           64  +  INSERT INTO xx VALUES('A x x C x x x C');
           65  +  INSERT INTO xx VALUES('A x x B C x x x');
           66  +}
           67  +do_execsql_test 1.2.1 {
           68  +  SELECT oid,extract(offsets(xx), x) FROM xx WHERE xx MATCH 'a OR (b NEAR/1 c)';
           69  +} {
           70  +  1 {(A) x x x (B) (C) x x}
           71  +  2 {(A) x x C x x x C} 
           72  +  3 {(A) x x (B) (C) x x x}
           73  +}
           74  +
           75  +do_execsql_test 1.3 {
           76  +  DELETE FROM xx;
           77  +  INSERT INTO xx(rowid, x) VALUES(1, 'A B C');
           78  +  INSERT INTO xx(rowid, x) VALUES(2, 'A x');
           79  +  INSERT INTO xx(rowid, x) VALUES(3, 'A B C');
           80  +  INSERT INTO xx(rowid, x) VALUES(4, 'A B C x x x x x x x B');
           81  +  INSERT INTO xx(rowid, x) VALUES(5, 'A x x x x x x x x x C');
           82  +  INSERT INTO xx(rowid, x) VALUES(6, 'A x x x x x x x x x x x B');
           83  +  INSERT INTO xx(rowid, x) VALUES(7, 'A B C');
           84  +}
           85  +do_execsql_test 1.3.1 {
           86  +  SELECT oid,extract(offsets(xx), x) FROM xx WHERE xx MATCH 'a OR (b NEAR/1 c)';
           87  +} {
           88  +  1 {(A) (B) (C)}
           89  +  2 {(A) x}
           90  +  3 {(A) (B) (C)}
           91  +  4 {(A) (B) (C) x x x x x x x B}
           92  +  5 {(A) x x x x x x x x x C}
           93  +  6 {(A) x x x x x x x x x x x B} 
           94  +  7 {(A) (B) (C)}
           95  +}
           96  +
           97  +
           98  +do_execsql_test 1.4 {
           99  +  DELETE FROM xx;
          100  +  INSERT INTO xx(rowid, x) VALUES(7, 'A B C');
          101  +  INSERT INTO xx(rowid, x) VALUES(6, 'A x');
          102  +  INSERT INTO xx(rowid, x) VALUES(5, 'A B C');
          103  +  INSERT INTO xx(rowid, x) VALUES(4, 'A B C x x x x x x x B');
          104  +  INSERT INTO xx(rowid, x) VALUES(3, 'A x x x x x x x x x C');
          105  +  INSERT INTO xx(rowid, x) VALUES(2, 'A x x x x x x x x x x x B');
          106  +  INSERT INTO xx(rowid, x) VALUES(1, 'A B C');
          107  +}
          108  +do_execsql_test 1.4.1 {
          109  +  SELECT oid,extract(offsets(xx), x) FROM xx WHERE xx MATCH 'a OR (b NEAR/1 c)'
          110  +  ORDER BY docid DESC;
          111  +} {
          112  +  7 {(A) (B) (C)}
          113  +  6 {(A) x}
          114  +  5 {(A) (B) (C)}
          115  +  4 {(A) (B) (C) x x x x x x x B}
          116  +  3 {(A) x x x x x x x x x C}
          117  +  2 {(A) x x x x x x x x x x x B} 
          118  +  1 {(A) (B) (C)}
          119  +}
          120  +
          121  +
          122  +set sqlite_fts3_enable_parentheses 0
          123  +finish_test
          124  +