/ Check-in [2c03b24f]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix a problem in FTS to do with ^ tokens and the snippet() function.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts4-content
Files: files | file ages | folders
SHA1: 2c03b24f4cc6f2c28c9d5b9984320d41b8486c32
User & Date: dan 2011-10-19 09:40:49
Context
2011-10-19
10:18
Add tests for FTS ^ searches and matchinfo(). check-in: 92618c14 user: dan tags: fts4-content
09:40
Fix a problem in FTS to do with ^ tokens and the snippet() function. check-in: 2c03b24f user: dan tags: fts4-content
2011-10-18
19:39
If a token within an FTS query is prefixed with a '^' character, it must be the first token in a column of data to match. check-in: 63ac33c8 user: dan tags: fts4-content
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts3/fts3.c.

  2387   2387         p++;
  2388   2388         p += sqlite3Fts3GetVarint(p, &iCol);
  2389   2389         if( *p==0x02 ){
  2390   2390           if( bWritten==0 ){
  2391   2391             fts3PutDeltaVarint3(&pOut, bDescDoclist, &iPrev, &bFirstOut, iDoc); 
  2392   2392             bWritten = 1;
  2393   2393           }
         2394  +        *pOut++ = 0x01;
  2394   2395           pOut += sqlite3Fts3PutVarint(pOut, iCol);
  2395   2396           *pOut++ = 0x02;
  2396   2397         }
  2397   2398         fts3ColumnlistCopy(0, &p);
  2398   2399       }
  2399   2400       if( bWritten ){
  2400   2401         *pOut++ = 0x00;

Changes to ext/fts3/fts3_snippet.c.

   364    364     pPhrase->nToken = pExpr->pPhrase->nToken;
   365    365   
   366    366     pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol);
   367    367     if( pCsr ){
   368    368       int iFirst = 0;
   369    369       pPhrase->pList = pCsr;
   370    370       fts3GetDeltaPosition(&pCsr, &iFirst);
          371  +    assert( iFirst>=0 );
   371    372       pPhrase->pHead = pCsr;
   372    373       pPhrase->pTail = pCsr;
   373    374       pPhrase->iHead = iFirst;
   374    375       pPhrase->iTail = iFirst;
   375    376     }else{
   376    377       assert( pPhrase->pList==0 && pPhrase->pHead==0 && pPhrase->pTail==0 );
   377    378     }

Changes to test/fts3first.test.

    13     13   source $testdir/tester.tcl
    14     14   source $testdir/malloc_common.tcl
    15     15   
    16     16   ifcapable !fts3 {
    17     17     finish_test
    18     18     return
    19     19   }
           20  +
           21  +proc lreverse {L} {
           22  +  set res [list]
           23  +  for {set ii [expr [llength $L]-1]} {$ii>=0} {incr ii -1} {
           24  +    lappend res [lindex $L $ii]
           25  +  }
           26  +  set res
           27  +}
    20     28   
    21     29   do_execsql_test 1.0 {
    22     30     CREATE VIRTUAL TABLE x1 USING FTS4(a, b, c);
    23     31     INSERT INTO x1(docid,a,b,c) VALUES(0, 'K H D S T', 'V M N Y K', 'S Z N Q S');
    24     32     INSERT INTO x1(docid,a,b,c) VALUES(1, 'K N J L W', 'S Z W J Q', 'D U W S E');
    25     33     INSERT INTO x1(docid,a,b,c) VALUES(2, 'B P M O I', 'R P H W S', 'R J L L E');
    26     34     INSERT INTO x1(docid,a,b,c) VALUES(3, 'U R Q M L', 'M J K A V', 'Q W J T J');
................................................................................
    29     37     INSERT INTO x1(docid,a,b,c) VALUES(6, 'M R G U T', 'U V I Q P', 'X Y D L S');
    30     38     INSERT INTO x1(docid,a,b,c) VALUES(7, 'D Y P O I', 'X J P K R', 'V O T H V');
    31     39     INSERT INTO x1(docid,a,b,c) VALUES(8, 'R Y D L R', 'U U E S J', 'N W L M R');
    32     40     INSERT INTO x1(docid,a,b,c) VALUES(9, 'Z P F N P', 'W A X D U', 'V A E Q A');
    33     41     INSERT INTO x1(docid,a,b,c) VALUES(10, 'Q I A Q M', 'N D K H C', 'A H T Q Z');
    34     42     INSERT INTO x1(docid,a,b,c) VALUES(11, 'T E R Q B', 'C I B C B', 'F Z U W R');
    35     43     INSERT INTO x1(docid,a,b,c) VALUES(12, 'E S V U W', 'T P F W H', 'A M D J Q');
    36         -  INSERT INTO x1(docid,a,b,c) VALUES(13, 'X S B T Y', 'U D N D P', 'X Z Y G F');
           44  +  INSERT INTO x1(docid,a,b,c) VALUES(13, 'X S B X Y', 'U D N D P', 'X Z Y G F');
    37     45     INSERT INTO x1(docid,a,b,c) VALUES(14, 'K H A B L', 'S R C C Z', 'D W E H J');
    38     46     INSERT INTO x1(docid,a,b,c) VALUES(15, 'C E U C C', 'W F M N M', 'T Z U X T');
    39     47     INSERT INTO x1(docid,a,b,c) VALUES(16, 'Q G C G H', 'H N N B H', 'B Q I H Y');
    40     48     INSERT INTO x1(docid,a,b,c) VALUES(17, 'Q T S K B', 'W B D Y N', 'V J P E C');
    41     49     INSERT INTO x1(docid,a,b,c) VALUES(18, 'A J M O Q', 'L G Y Y A', 'G N M R N');
    42     50     INSERT INTO x1(docid,a,b,c) VALUES(19, 'T R Y P Y', 'N V Y B X', 'L Z T N T');
    43     51   
    44     52     CREATE VIRTUAL TABLE x2 USING FTS4(a, b, c, order=DESC);
    45     53     INSERT INTO x2(docid, a, b, c) SELECT docid, a, b, c FROM x1;
    46     54   }
           55  +
    47     56   
    48     57   foreach x {1 2} {
    49     58     foreach {tn match res} {
    50     59       1  "^K"              {0 1 14}
    51     60       2  "^S"              {0 1 14}
    52     61       3  "^W"              {9 15 17}
    53     62       4  "^J"              {}
................................................................................
    58     67       9  "-^E V"           {0 3 4 6 7 9 17 19}
    59     68       10 "V"               {0 3 4 6 7 9 12 17 19}
    60     69   
    61     70       11 {"^K H"}          {0 14}
    62     71       12 {"K H"}           {0 10 14}
    63     72       13 {"K ^H"}          {}
    64     73     } {
    65         -    set rev [list]
    66         -    for {set ii [expr [llength $res]-1]} {$ii>=0} {incr ii -1} {
    67         -      lappend rev [lindex $res $ii]
    68         -    }
           74  +    set rev [lreverse $res]
    69     75       do_execsql_test 1.$x.$tn.1 {SELECT docid FROM x1 WHERE x1 MATCH $match} $res
    70     76       do_execsql_test 1.$x.$tn.2 {SELECT docid FROM x2 WHERE x2 MATCH $match} $rev
    71     77     }
    72     78   
    73     79     do_execsql_test 1.$x.[expr $tn+1] { 
    74     80       INSERT INTO x1(x1) VALUES('optimize');
    75     81       INSERT INTO x2(x2) VALUES('optimize');
    76     82     } {}
    77     83   }
           84  +
           85  +foreach {tn match res} {
           86  +  1  {^K}    {{[K] H D S T} {[K] N J L W} {[K] H A B L}}
           87  +  2  {^X}    {{[X] Y D L S} {[X] J P K R} {[X] S B X Y}}
           88  +  3  {^X Y}  {{[X] [Y] D L S} {D [Y] P O I...[X] J P K R} {[X] S B X [Y]}}
           89  +} {
           90  +  set rev [lreverse $res]
           91  +
           92  +  do_execsql_test 1.3.$tn.1 {
           93  +    SELECT snippet(x1, '[', ']', '...') FROM x1 WHERE x1 MATCH $match
           94  +  } $res
           95  +  do_execsql_test 1.3.$tn.2 {
           96  +    SELECT snippet(x2, '[', ']', '...') FROM x2 WHERE x2 MATCH $match
           97  +  } $rev
           98  +}
    78     99   
    79    100   finish_test