SQLite

Check-in [3bb13a0652]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix another bug in 3-way NEAR queries.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 3bb13a06521b54194c9f3eb44e0dc42bacf016a4
User & Date: dan 2009-12-05 14:29:23.000
Context
2009-12-05
18:03
Catch an obscure case of database corruption. (check-in: 0f9b17a6da user: dan tags: trunk)
14:29
Fix another bug in 3-way NEAR queries. (check-in: 3bb13a0652 user: dan tags: trunk)
11:37
Fix some problems with FTS3 and 3-way NEAR queries. (check-in: 23dc6fb5b2 user: dan tags: trunk)
Changes
Unified Diff Ignore Whitespace Patch
Changes to ext/fts3/fts3.c.
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129

1130



1131


1132
1133
1134
1135
1136
1137
1138
  char **pp1,                     /* Left input list */
  char **pp2                      /* Right input list */
){
  char *p = *pp;
  char *p1 = *pp1;
  char *p2 = *pp2;

  while( *p1 && *p2 ){
    int iCol1 = 0;
    int iCol2 = 0;

    if( *p1==0x01 ) sqlite3Fts3GetVarint32(&p1[1], &iCol1);



    if( *p2==0x01 ) sqlite3Fts3GetVarint32(&p2[1], &iCol2);



    if( iCol1==iCol2 ){
      sqlite3_int64 i1 = 0;
      sqlite3_int64 i2 = 0;
      sqlite3_int64 iPrev = 0;
      int n = fts3PutColNumber(&p, iCol1);
      p1 += n;







|
|
|
>

>
>
>

>
>







1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
  char **pp1,                     /* Left input list */
  char **pp2                      /* Right input list */
){
  char *p = *pp;
  char *p1 = *pp1;
  char *p2 = *pp2;

  while( *p1 || *p2 ){
    int iCol1;
    int iCol2;

    if( *p1==0x01 ) sqlite3Fts3GetVarint32(&p1[1], &iCol1);
    else if( *p1==0x00 ) iCol1 = OFFSET_LIST_END;
    else iCol1 = 0;

    if( *p2==0x01 ) sqlite3Fts3GetVarint32(&p2[1], &iCol2);
    else if( *p2==0x00 ) iCol2 = OFFSET_LIST_END;
    else iCol2 = 0;

    if( iCol1==iCol2 ){
      sqlite3_int64 i1 = 0;
      sqlite3_int64 i2 = 0;
      sqlite3_int64 iPrev = 0;
      int n = fts3PutColNumber(&p, iCol1);
      p1 += n;
1376
1377
1378
1379
1380
1381
1382




1383
1384
1385
1386
1387
1388
1389
       || mergetype==MERGE_AND    || mergetype==MERGE_NOT
       || mergetype==MERGE_PHRASE || mergetype==MERGE_POS_PHRASE
       || mergetype==MERGE_NEAR   || mergetype==MERGE_POS_NEAR
  );

  if( !aBuffer ){
    return SQLITE_NOMEM;




  }

  /* Read the first docid from each doclist */
  fts3GetDeltaVarint2(&p1, pEnd1, &i1);
  fts3GetDeltaVarint2(&p2, pEnd2, &i2);

  switch( mergetype ){







>
>
>
>







1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
       || mergetype==MERGE_AND    || mergetype==MERGE_NOT
       || mergetype==MERGE_PHRASE || mergetype==MERGE_POS_PHRASE
       || mergetype==MERGE_NEAR   || mergetype==MERGE_POS_NEAR
  );

  if( !aBuffer ){
    return SQLITE_NOMEM;
  }
  if( n1==0 && n2==0 ){
    *pnBuffer = 0;
    return SQLITE_OK;
  }

  /* Read the first docid from each doclist */
  fts3GetDeltaVarint2(&p1, pEnd1, &i1);
  fts3GetDeltaVarint2(&p2, pEnd2, &i2);

  switch( mergetype ){
Changes to test/fts3rnd.test.
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109

110
111
112
113
114
115
116
117
118

119
120
121
122
123
124
125
  }
  lsort -integer $ret
}

proc simple_near {termlist nNear} {
  set ret [list]

  # List of nNear+1 elements that contains no terms.
  #
  set dummyfields "[string repeat "X " $nNear]X"

  foreach {key value} [array get ::t1] {
    foreach v $value {
      set v [concat $dummyfields $v $dummyfields]

      set l [lsearch -exact -all $v [lindex $termlist 0]]
      foreach T [lrange $termlist 1 end] {
        set l2 [list]
        foreach i $l {
          set iStart [expr $i - $nNear - 1]
          set iEnd [expr $i + $nNear + 1]

          foreach i2 [lsearch -exact -all [lrange $v $iStart $iEnd] $T] {
            incr i2 $iStart
            if {$i2 != $i} { lappend l2 $i2 } 
          }
        }
        set l $l2
      }

      if {[llength $l]} {

        lappend ret $key
      } 
    }
  }

  lsort -unique -integer $ret
}







<
<
<
<


<







>





|



>







89
90
91
92
93
94
95




96
97

98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
  }
  lsort -integer $ret
}

proc simple_near {termlist nNear} {
  set ret [list]





  foreach {key value} [array get ::t1] {
    foreach v $value {


      set l [lsearch -exact -all $v [lindex $termlist 0]]
      foreach T [lrange $termlist 1 end] {
        set l2 [list]
        foreach i $l {
          set iStart [expr $i - $nNear - 1]
          set iEnd [expr $i + $nNear + 1]
          if {$iStart < 0} {set iStart 0}
          foreach i2 [lsearch -exact -all [lrange $v $iStart $iEnd] $T] {
            incr i2 $iStart
            if {$i2 != $i} { lappend l2 $i2 } 
          }
        }
        set l [lsort -uniq -integer $l2]
      }

      if {[llength $l]} {
#puts "MATCH($key): $v"
        lappend ret $key
      } 
    }
  }

  lsort -unique -integer $ret
}
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
      set iInsert [expr {int(rand()*1000000)}]
    }
    db transaction {
      insert_row $iInsert
      update_row $iUpdate
      delete_row $iDelete
    }
  
    # Pick 10 terms from the vocabulary. Check that the results of querying
    # the database for the set of documents containing each of these terms
    # is the same as the result obtained by scanning the contents of the Tcl 
    # array for each term.
    #
    for {set i 0} {$i < 10} {incr i} {
      set term [random_term]







|







148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
      set iInsert [expr {int(rand()*1000000)}]
    }
    db transaction {
      insert_row $iInsert
      update_row $iUpdate
      delete_row $iDelete
    }

    # Pick 10 terms from the vocabulary. Check that the results of querying
    # the database for the set of documents containing each of these terms
    # is the same as the result obtained by scanning the contents of the Tcl 
    # array for each term.
    #
    for {set i 0} {$i < 10} {incr i} {
      set term [random_term]
206
207
208
209
210
211
212
213
214
215
216
217

218


219
220
221

222
223
224
225
226
227
      set terms [list [random_term] [random_term]]
      set match [join $terms " NEAR "]
      do_test fts3rnd-1.$nodesize.$iTest.5.$i.$match {
        execsql { SELECT docid FROM t1 WHERE t1 MATCH $match }
      } [simple_near $terms 10]
    }
 
if 0 {
    # A 3-way NEAR query with terms as the arguments.
    #
    for {set i 0} {$i < 10} {incr i} {
      set terms [list [random_term] [random_term] [random_term]]

      set match [join $terms " NEAR "]


      do_test fts3rnd-1.$nodesize.$iTest.5.$i.$match {
        execsql { SELECT docid FROM t1 WHERE t1 MATCH $match }
      } [simple_near $terms 10]

    }
}
  }
}

finish_test







<




>
|
>
>


|
>

<




203
204
205
206
207
208
209

210
211
212
213
214
215
216
217
218
219
220
221
222

223
224
225
226
      set terms [list [random_term] [random_term]]
      set match [join $terms " NEAR "]
      do_test fts3rnd-1.$nodesize.$iTest.5.$i.$match {
        execsql { SELECT docid FROM t1 WHERE t1 MATCH $match }
      } [simple_near $terms 10]
    }
 

    # A 3-way NEAR query with terms as the arguments.
    #
    for {set i 0} {$i < 10} {incr i} {
      set terms [list [random_term] [random_term] [random_term]]
      set nNear 11
      set match [join $terms " NEAR/$nNear "]
      set fts3 [execsql { SELECT docid FROM t1 WHERE t1 MATCH $match }]
      set tcl [simple_near $terms $nNear]
      do_test fts3rnd-1.$nodesize.$iTest.5.$i.$match {
        execsql { SELECT docid FROM t1 WHERE t1 MATCH $match }
      } [simple_near $terms $nNear]
if {$::nErr} {exit -1}
    }

  }
}

finish_test