SQLite: Check-in [2c03b24f]

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview

Comment:	Fix a problem in FTS to do with ^ tokens and the snippet() function.
Downloads:	Tarball \| ZIP archive \| SQL archive
Timelines:	family \| ancestors \| descendants \| both \| fts4-content
Files:	files \| file ages \| folders
SHA1:	2c03b24f4cc6f2c28c9d5b9984320d41b8486c32
User & Date:	dan 2011-10-19 09:40:49

Context

2011-10-19
10:18		Add tests for FTS ^ searches and matchinfo(). check-in: 92618c14 user: dan tags: fts4-content
09:40		Fix a problem in FTS to do with ^ tokens and the snippet() function. check-in: 2c03b24f user: dan tags: fts4-content
2011-10-18
19:39		If a token within an FTS query is prefixed with a '^' character, it must be the first token in a column of data to match. check-in: 63ac33c8 user: dan tags: fts4-content

Changes

Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts3/fts3.c.

      p++;
      p += sqlite3Fts3GetVarint(p, &iCol);
      if( *p==0x02 ){
        if( bWritten==0 ){
          fts3PutDeltaVarint3(&pOut, bDescDoclist, &iPrev, &bFirstOut, iDoc); 
          bWritten = 1;
        }

        pOut += sqlite3Fts3PutVarint(pOut, iCol);
        *pOut++ = 0x02;
      }
      fts3ColumnlistCopy(0, &p);
    }
    if( bWritten ){
      *pOut++ = 0x00;

      p++;
      p += sqlite3Fts3GetVarint(p, &iCol);
      if( *p==0x02 ){
        if( bWritten==0 ){
          fts3PutDeltaVarint3(&pOut, bDescDoclist, &iPrev, &bFirstOut, iDoc); 
          bWritten = 1;
        }
        *pOut++ = 0x01;
        pOut += sqlite3Fts3PutVarint(pOut, iCol);
        *pOut++ = 0x02;
      }
      fts3ColumnlistCopy(0, &p);
    }
    if( bWritten ){
      *pOut++ = 0x00;

Changes to ext/fts3/fts3_snippet.c.

  pPhrase->nToken = pExpr->pPhrase->nToken;

  pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol);
  if( pCsr ){
    int iFirst = 0;
    pPhrase->pList = pCsr;
    fts3GetDeltaPosition(&pCsr, &iFirst);

    pPhrase->pHead = pCsr;
    pPhrase->pTail = pCsr;
    pPhrase->iHead = iFirst;
    pPhrase->iTail = iFirst;
  }else{
    assert( pPhrase->pList==0 && pPhrase->pHead==0 && pPhrase->pTail==0 );
  }

  pPhrase->nToken = pExpr->pPhrase->nToken;

  pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol);
  if( pCsr ){
    int iFirst = 0;
    pPhrase->pList = pCsr;
    fts3GetDeltaPosition(&pCsr, &iFirst);
    assert( iFirst>=0 );
    pPhrase->pHead = pCsr;
    pPhrase->pTail = pCsr;
    pPhrase->iHead = iFirst;
    pPhrase->iTail = iFirst;
  }else{
    assert( pPhrase->pList==0 && pPhrase->pHead==0 && pPhrase->pTail==0 );
  }

Changes to test/fts3first.test.

source $testdir/tester.tcl
source $testdir/malloc_common.tcl

ifcapable !fts3 {
  finish_test
  return
}









do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE x1 USING FTS4(a, b, c);
  INSERT INTO x1(docid,a,b,c) VALUES(0, 'K H D S T', 'V M N Y K', 'S Z N Q S');
  INSERT INTO x1(docid,a,b,c) VALUES(1, 'K N J L W', 'S Z W J Q', 'D U W S E');
  INSERT INTO x1(docid,a,b,c) VALUES(2, 'B P M O I', 'R P H W S', 'R J L L E');
  INSERT INTO x1(docid,a,b,c) VALUES(3, 'U R Q M L', 'M J K A V', 'Q W J T J');
................................................................................
  INSERT INTO x1(docid,a,b,c) VALUES(6, 'M R G U T', 'U V I Q P', 'X Y D L S');
  INSERT INTO x1(docid,a,b,c) VALUES(7, 'D Y P O I', 'X J P K R', 'V O T H V');
  INSERT INTO x1(docid,a,b,c) VALUES(8, 'R Y D L R', 'U U E S J', 'N W L M R');
  INSERT INTO x1(docid,a,b,c) VALUES(9, 'Z P F N P', 'W A X D U', 'V A E Q A');
  INSERT INTO x1(docid,a,b,c) VALUES(10, 'Q I A Q M', 'N D K H C', 'A H T Q Z');
  INSERT INTO x1(docid,a,b,c) VALUES(11, 'T E R Q B', 'C I B C B', 'F Z U W R');
  INSERT INTO x1(docid,a,b,c) VALUES(12, 'E S V U W', 'T P F W H', 'A M D J Q');
  INSERT INTO x1(docid,a,b,c) VALUES(13, 'X S B T Y', 'U D N D P', 'X Z Y G F');
  INSERT INTO x1(docid,a,b,c) VALUES(14, 'K H A B L', 'S R C C Z', 'D W E H J');
  INSERT INTO x1(docid,a,b,c) VALUES(15, 'C E U C C', 'W F M N M', 'T Z U X T');
  INSERT INTO x1(docid,a,b,c) VALUES(16, 'Q G C G H', 'H N N B H', 'B Q I H Y');
  INSERT INTO x1(docid,a,b,c) VALUES(17, 'Q T S K B', 'W B D Y N', 'V J P E C');
  INSERT INTO x1(docid,a,b,c) VALUES(18, 'A J M O Q', 'L G Y Y A', 'G N M R N');
  INSERT INTO x1(docid,a,b,c) VALUES(19, 'T R Y P Y', 'N V Y B X', 'L Z T N T');

  CREATE VIRTUAL TABLE x2 USING FTS4(a, b, c, order=DESC);
  INSERT INTO x2(docid, a, b, c) SELECT docid, a, b, c FROM x1;
}


foreach x {1 2} {
  foreach {tn match res} {
    1  "^K"              {0 1 14}
    2  "^S"              {0 1 14}
    3  "^W"              {9 15 17}
    4  "^J"              {}
................................................................................
    9  "-^E V"           {0 3 4 6 7 9 17 19}
    10 "V"               {0 3 4 6 7 9 12 17 19}

    11 {"^K H"}          {0 14}
    12 {"K H"}           {0 10 14}
    13 {"K ^H"}          {}
  } {
    set rev [list]
    for {set ii [expr [llength $res]-1]} {$ii>=0} {incr ii -1} {
      lappend rev [lindex $res $ii]
    }
    do_execsql_test 1.$x.$tn.1 {SELECT docid FROM x1 WHERE x1 MATCH $match} $res
    do_execsql_test 1.$x.$tn.2 {SELECT docid FROM x2 WHERE x2 MATCH $match} $rev
  }

  do_execsql_test 1.$x.[expr $tn+1] { 
    INSERT INTO x1(x1) VALUES('optimize');
    INSERT INTO x2(x2) VALUES('optimize');
  } {}
}
















finish_test








>
>
>
>
>
>
>
>







 







|










>







 







|
<
<
<










>
>
>
>
>
>
>
>
>
>
>
>
>
>
>

source $testdir/tester.tcl
source $testdir/malloc_common.tcl

ifcapable !fts3 {
  finish_test
  return
}

proc lreverse {L} {
  set res [list]
  for {set ii [expr [llength $L]-1]} {$ii>=0} {incr ii -1} {
    lappend res [lindex $L $ii]
  }
  set res
}

do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE x1 USING FTS4(a, b, c);
  INSERT INTO x1(docid,a,b,c) VALUES(0, 'K H D S T', 'V M N Y K', 'S Z N Q S');
  INSERT INTO x1(docid,a,b,c) VALUES(1, 'K N J L W', 'S Z W J Q', 'D U W S E');
  INSERT INTO x1(docid,a,b,c) VALUES(2, 'B P M O I', 'R P H W S', 'R J L L E');
  INSERT INTO x1(docid,a,b,c) VALUES(3, 'U R Q M L', 'M J K A V', 'Q W J T J');
................................................................................
  INSERT INTO x1(docid,a,b,c) VALUES(6, 'M R G U T', 'U V I Q P', 'X Y D L S');
  INSERT INTO x1(docid,a,b,c) VALUES(7, 'D Y P O I', 'X J P K R', 'V O T H V');
  INSERT INTO x1(docid,a,b,c) VALUES(8, 'R Y D L R', 'U U E S J', 'N W L M R');
  INSERT INTO x1(docid,a,b,c) VALUES(9, 'Z P F N P', 'W A X D U', 'V A E Q A');
  INSERT INTO x1(docid,a,b,c) VALUES(10, 'Q I A Q M', 'N D K H C', 'A H T Q Z');
  INSERT INTO x1(docid,a,b,c) VALUES(11, 'T E R Q B', 'C I B C B', 'F Z U W R');
  INSERT INTO x1(docid,a,b,c) VALUES(12, 'E S V U W', 'T P F W H', 'A M D J Q');
  INSERT INTO x1(docid,a,b,c) VALUES(13, 'X S B X Y', 'U D N D P', 'X Z Y G F');
  INSERT INTO x1(docid,a,b,c) VALUES(14, 'K H A B L', 'S R C C Z', 'D W E H J');
  INSERT INTO x1(docid,a,b,c) VALUES(15, 'C E U C C', 'W F M N M', 'T Z U X T');
  INSERT INTO x1(docid,a,b,c) VALUES(16, 'Q G C G H', 'H N N B H', 'B Q I H Y');
  INSERT INTO x1(docid,a,b,c) VALUES(17, 'Q T S K B', 'W B D Y N', 'V J P E C');
  INSERT INTO x1(docid,a,b,c) VALUES(18, 'A J M O Q', 'L G Y Y A', 'G N M R N');
  INSERT INTO x1(docid,a,b,c) VALUES(19, 'T R Y P Y', 'N V Y B X', 'L Z T N T');

  CREATE VIRTUAL TABLE x2 USING FTS4(a, b, c, order=DESC);
  INSERT INTO x2(docid, a, b, c) SELECT docid, a, b, c FROM x1;
}


foreach x {1 2} {
  foreach {tn match res} {
    1  "^K"              {0 1 14}
    2  "^S"              {0 1 14}
    3  "^W"              {9 15 17}
    4  "^J"              {}
................................................................................
    9  "-^E V"           {0 3 4 6 7 9 17 19}
    10 "V"               {0 3 4 6 7 9 12 17 19}

    11 {"^K H"}          {0 14}
    12 {"K H"}           {0 10 14}
    13 {"K ^H"}          {}
  } {
    set rev [lreverse $res]



    do_execsql_test 1.$x.$tn.1 {SELECT docid FROM x1 WHERE x1 MATCH $match} $res
    do_execsql_test 1.$x.$tn.2 {SELECT docid FROM x2 WHERE x2 MATCH $match} $rev
  }

  do_execsql_test 1.$x.[expr $tn+1] { 
    INSERT INTO x1(x1) VALUES('optimize');
    INSERT INTO x2(x2) VALUES('optimize');
  } {}
}

foreach {tn match res} {
  1  {^K}    {{[K] H D S T} {[K] N J L W} {[K] H A B L}}
  2  {^X}    {{[X] Y D L S} {[X] J P K R} {[X] S B X Y}}
  3  {^X Y}  {{[X] [Y] D L S} {D [Y] P O I...[X] J P K R} {[X] S B X [Y]}}
} {
  set rev [lreverse $res]

  do_execsql_test 1.3.$tn.1 {
    SELECT snippet(x1, '[', ']', '...') FROM x1 WHERE x1 MATCH $match
  } $res
  do_execsql_test 1.3.$tn.2 {
    SELECT snippet(x2, '[', ']', '...') FROM x2 WHERE x2 MATCH $match
  } $rev
}

finish_test