SQLite

Check-in [2c03b24f4c]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix a problem in FTS to do with ^ tokens and the snippet() function.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | fts4-content
Files: files | file ages | folders
SHA1: 2c03b24f4cc6f2c28c9d5b9984320d41b8486c32
User & Date: dan 2011-10-19 09:40:49.185
Context
2011-10-19
10:18
Add tests for FTS ^ searches and matchinfo(). (check-in: 92618c1463 user: dan tags: fts4-content)
09:40
Fix a problem in FTS to do with ^ tokens and the snippet() function. (check-in: 2c03b24f4c user: dan tags: fts4-content)
2011-10-18
19:39
If a token within an FTS query is prefixed with a '^' character, it must be the first token in a column of data to match. (check-in: 63ac33c860 user: dan tags: fts4-content)
Changes
Unified Diff Ignore Whitespace Patch
Changes to ext/fts3/fts3.c.
2387
2388
2389
2390
2391
2392
2393

2394
2395
2396
2397
2398
2399
2400
      p++;
      p += sqlite3Fts3GetVarint(p, &iCol);
      if( *p==0x02 ){
        if( bWritten==0 ){
          fts3PutDeltaVarint3(&pOut, bDescDoclist, &iPrev, &bFirstOut, iDoc); 
          bWritten = 1;
        }

        pOut += sqlite3Fts3PutVarint(pOut, iCol);
        *pOut++ = 0x02;
      }
      fts3ColumnlistCopy(0, &p);
    }
    if( bWritten ){
      *pOut++ = 0x00;







>







2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
      p++;
      p += sqlite3Fts3GetVarint(p, &iCol);
      if( *p==0x02 ){
        if( bWritten==0 ){
          fts3PutDeltaVarint3(&pOut, bDescDoclist, &iPrev, &bFirstOut, iDoc); 
          bWritten = 1;
        }
        *pOut++ = 0x01;
        pOut += sqlite3Fts3PutVarint(pOut, iCol);
        *pOut++ = 0x02;
      }
      fts3ColumnlistCopy(0, &p);
    }
    if( bWritten ){
      *pOut++ = 0x00;
Changes to ext/fts3/fts3_snippet.c.
364
365
366
367
368
369
370

371
372
373
374
375
376
377
  pPhrase->nToken = pExpr->pPhrase->nToken;

  pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol);
  if( pCsr ){
    int iFirst = 0;
    pPhrase->pList = pCsr;
    fts3GetDeltaPosition(&pCsr, &iFirst);

    pPhrase->pHead = pCsr;
    pPhrase->pTail = pCsr;
    pPhrase->iHead = iFirst;
    pPhrase->iTail = iFirst;
  }else{
    assert( pPhrase->pList==0 && pPhrase->pHead==0 && pPhrase->pTail==0 );
  }







>







364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
  pPhrase->nToken = pExpr->pPhrase->nToken;

  pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol);
  if( pCsr ){
    int iFirst = 0;
    pPhrase->pList = pCsr;
    fts3GetDeltaPosition(&pCsr, &iFirst);
    assert( iFirst>=0 );
    pPhrase->pHead = pCsr;
    pPhrase->pTail = pCsr;
    pPhrase->iHead = iFirst;
    pPhrase->iTail = iFirst;
  }else{
    assert( pPhrase->pList==0 && pPhrase->pHead==0 && pPhrase->pTail==0 );
  }
Changes to test/fts3first.test.
13
14
15
16
17
18
19








20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46

47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78















79
source $testdir/tester.tcl
source $testdir/malloc_common.tcl

ifcapable !fts3 {
  finish_test
  return
}









do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE x1 USING FTS4(a, b, c);
  INSERT INTO x1(docid,a,b,c) VALUES(0, 'K H D S T', 'V M N Y K', 'S Z N Q S');
  INSERT INTO x1(docid,a,b,c) VALUES(1, 'K N J L W', 'S Z W J Q', 'D U W S E');
  INSERT INTO x1(docid,a,b,c) VALUES(2, 'B P M O I', 'R P H W S', 'R J L L E');
  INSERT INTO x1(docid,a,b,c) VALUES(3, 'U R Q M L', 'M J K A V', 'Q W J T J');
  INSERT INTO x1(docid,a,b,c) VALUES(4, 'N J C Y N', 'R U D X V', 'B O U A Q');
  INSERT INTO x1(docid,a,b,c) VALUES(5, 'Q L X L U', 'I F N X S', 'U Q A N Y');
  INSERT INTO x1(docid,a,b,c) VALUES(6, 'M R G U T', 'U V I Q P', 'X Y D L S');
  INSERT INTO x1(docid,a,b,c) VALUES(7, 'D Y P O I', 'X J P K R', 'V O T H V');
  INSERT INTO x1(docid,a,b,c) VALUES(8, 'R Y D L R', 'U U E S J', 'N W L M R');
  INSERT INTO x1(docid,a,b,c) VALUES(9, 'Z P F N P', 'W A X D U', 'V A E Q A');
  INSERT INTO x1(docid,a,b,c) VALUES(10, 'Q I A Q M', 'N D K H C', 'A H T Q Z');
  INSERT INTO x1(docid,a,b,c) VALUES(11, 'T E R Q B', 'C I B C B', 'F Z U W R');
  INSERT INTO x1(docid,a,b,c) VALUES(12, 'E S V U W', 'T P F W H', 'A M D J Q');
  INSERT INTO x1(docid,a,b,c) VALUES(13, 'X S B T Y', 'U D N D P', 'X Z Y G F');
  INSERT INTO x1(docid,a,b,c) VALUES(14, 'K H A B L', 'S R C C Z', 'D W E H J');
  INSERT INTO x1(docid,a,b,c) VALUES(15, 'C E U C C', 'W F M N M', 'T Z U X T');
  INSERT INTO x1(docid,a,b,c) VALUES(16, 'Q G C G H', 'H N N B H', 'B Q I H Y');
  INSERT INTO x1(docid,a,b,c) VALUES(17, 'Q T S K B', 'W B D Y N', 'V J P E C');
  INSERT INTO x1(docid,a,b,c) VALUES(18, 'A J M O Q', 'L G Y Y A', 'G N M R N');
  INSERT INTO x1(docid,a,b,c) VALUES(19, 'T R Y P Y', 'N V Y B X', 'L Z T N T');

  CREATE VIRTUAL TABLE x2 USING FTS4(a, b, c, order=DESC);
  INSERT INTO x2(docid, a, b, c) SELECT docid, a, b, c FROM x1;
}


foreach x {1 2} {
  foreach {tn match res} {
    1  "^K"              {0 1 14}
    2  "^S"              {0 1 14}
    3  "^W"              {9 15 17}
    4  "^J"              {}
    5  "^E"              {12}
    6  "V ^-E"           {0 3 4 6 7 9 17 19}
    7  "V -^E"           {0 3 4 6 7 9 17 19}
    8  "^-E V"           {0 3 4 6 7 9 17 19}
    9  "-^E V"           {0 3 4 6 7 9 17 19}
    10 "V"               {0 3 4 6 7 9 12 17 19}

    11 {"^K H"}          {0 14}
    12 {"K H"}           {0 10 14}
    13 {"K ^H"}          {}
  } {
    set rev [list]
    for {set ii [expr [llength $res]-1]} {$ii>=0} {incr ii -1} {
      lappend rev [lindex $res $ii]
    }
    do_execsql_test 1.$x.$tn.1 {SELECT docid FROM x1 WHERE x1 MATCH $match} $res
    do_execsql_test 1.$x.$tn.2 {SELECT docid FROM x2 WHERE x2 MATCH $match} $rev
  }

  do_execsql_test 1.$x.[expr $tn+1] { 
    INSERT INTO x1(x1) VALUES('optimize');
    INSERT INTO x2(x2) VALUES('optimize');
  } {}
}
















finish_test







>
>
>
>
>
>
>
>
















|










>


















|
<
<
<










>
>
>
>
>
>
>
>
>
>
>
>
>
>
>

13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74



75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
source $testdir/tester.tcl
source $testdir/malloc_common.tcl

ifcapable !fts3 {
  finish_test
  return
}

proc lreverse {L} {
  set res [list]
  for {set ii [expr [llength $L]-1]} {$ii>=0} {incr ii -1} {
    lappend res [lindex $L $ii]
  }
  set res
}

do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE x1 USING FTS4(a, b, c);
  INSERT INTO x1(docid,a,b,c) VALUES(0, 'K H D S T', 'V M N Y K', 'S Z N Q S');
  INSERT INTO x1(docid,a,b,c) VALUES(1, 'K N J L W', 'S Z W J Q', 'D U W S E');
  INSERT INTO x1(docid,a,b,c) VALUES(2, 'B P M O I', 'R P H W S', 'R J L L E');
  INSERT INTO x1(docid,a,b,c) VALUES(3, 'U R Q M L', 'M J K A V', 'Q W J T J');
  INSERT INTO x1(docid,a,b,c) VALUES(4, 'N J C Y N', 'R U D X V', 'B O U A Q');
  INSERT INTO x1(docid,a,b,c) VALUES(5, 'Q L X L U', 'I F N X S', 'U Q A N Y');
  INSERT INTO x1(docid,a,b,c) VALUES(6, 'M R G U T', 'U V I Q P', 'X Y D L S');
  INSERT INTO x1(docid,a,b,c) VALUES(7, 'D Y P O I', 'X J P K R', 'V O T H V');
  INSERT INTO x1(docid,a,b,c) VALUES(8, 'R Y D L R', 'U U E S J', 'N W L M R');
  INSERT INTO x1(docid,a,b,c) VALUES(9, 'Z P F N P', 'W A X D U', 'V A E Q A');
  INSERT INTO x1(docid,a,b,c) VALUES(10, 'Q I A Q M', 'N D K H C', 'A H T Q Z');
  INSERT INTO x1(docid,a,b,c) VALUES(11, 'T E R Q B', 'C I B C B', 'F Z U W R');
  INSERT INTO x1(docid,a,b,c) VALUES(12, 'E S V U W', 'T P F W H', 'A M D J Q');
  INSERT INTO x1(docid,a,b,c) VALUES(13, 'X S B X Y', 'U D N D P', 'X Z Y G F');
  INSERT INTO x1(docid,a,b,c) VALUES(14, 'K H A B L', 'S R C C Z', 'D W E H J');
  INSERT INTO x1(docid,a,b,c) VALUES(15, 'C E U C C', 'W F M N M', 'T Z U X T');
  INSERT INTO x1(docid,a,b,c) VALUES(16, 'Q G C G H', 'H N N B H', 'B Q I H Y');
  INSERT INTO x1(docid,a,b,c) VALUES(17, 'Q T S K B', 'W B D Y N', 'V J P E C');
  INSERT INTO x1(docid,a,b,c) VALUES(18, 'A J M O Q', 'L G Y Y A', 'G N M R N');
  INSERT INTO x1(docid,a,b,c) VALUES(19, 'T R Y P Y', 'N V Y B X', 'L Z T N T');

  CREATE VIRTUAL TABLE x2 USING FTS4(a, b, c, order=DESC);
  INSERT INTO x2(docid, a, b, c) SELECT docid, a, b, c FROM x1;
}


foreach x {1 2} {
  foreach {tn match res} {
    1  "^K"              {0 1 14}
    2  "^S"              {0 1 14}
    3  "^W"              {9 15 17}
    4  "^J"              {}
    5  "^E"              {12}
    6  "V ^-E"           {0 3 4 6 7 9 17 19}
    7  "V -^E"           {0 3 4 6 7 9 17 19}
    8  "^-E V"           {0 3 4 6 7 9 17 19}
    9  "-^E V"           {0 3 4 6 7 9 17 19}
    10 "V"               {0 3 4 6 7 9 12 17 19}

    11 {"^K H"}          {0 14}
    12 {"K H"}           {0 10 14}
    13 {"K ^H"}          {}
  } {
    set rev [lreverse $res]



    do_execsql_test 1.$x.$tn.1 {SELECT docid FROM x1 WHERE x1 MATCH $match} $res
    do_execsql_test 1.$x.$tn.2 {SELECT docid FROM x2 WHERE x2 MATCH $match} $rev
  }

  do_execsql_test 1.$x.[expr $tn+1] { 
    INSERT INTO x1(x1) VALUES('optimize');
    INSERT INTO x2(x2) VALUES('optimize');
  } {}
}

foreach {tn match res} {
  1  {^K}    {{[K] H D S T} {[K] N J L W} {[K] H A B L}}
  2  {^X}    {{[X] Y D L S} {[X] J P K R} {[X] S B X Y}}
  3  {^X Y}  {{[X] [Y] D L S} {D [Y] P O I...[X] J P K R} {[X] S B X [Y]}}
} {
  set rev [lreverse $res]

  do_execsql_test 1.3.$tn.1 {
    SELECT snippet(x1, '[', ']', '...') FROM x1 WHERE x1 MATCH $match
  } $res
  do_execsql_test 1.3.$tn.2 {
    SELECT snippet(x2, '[', ']', '...') FROM x2 WHERE x2 MATCH $match
  } $rev
}

finish_test