/ Check-in [b9fb69e5]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add tests for deferred tokens to fts3auto.test. Fix a problem with OR queries and deferred tokens.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts3-prefix-search
Files: files | file ages | folders
SHA1:b9fb69e55bb05a8819688ee63987f2a05c33d59b
User & Date: dan 2011-06-13 17:00:12
Context
2011-06-13
18:21
Update trace2.test to account for new FTS queries. check-in: 2c201292 user: dan tags: fts3-prefix-search
17:00
Add tests for deferred tokens to fts3auto.test. Fix a problem with OR queries and deferred tokens. check-in: b9fb69e5 user: dan tags: fts3-prefix-search
13:48
Changes to fts3auto.test to test OR, AND and NOT operations. check-in: e4ab6cdb user: dan tags: fts3-prefix-search
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts3/fts3.c.

3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579





3580
3581
3582
3583
3584
3585
3586
....
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
....
4306
4307
4308
4309
4310
4311
4312
4313
4314
4315
4316
4317
4318
4319
4320
        pTC->pRoot = pRoot;
        pTC->pToken = &pPhrase->aToken[i];
        pTC->iCol = pPhrase->iColumn;
        *pRc = sqlite3Fts3MsrOvfl(pCsr, pTC->pToken->pSegcsr, &pTC->nOvfl);
      }
    }else if( pExpr->eType!=FTSQUERY_NOT ){
      if( pExpr->eType==FTSQUERY_OR ){
        pRoot = pExpr;
        **ppOr = pExpr;
        (*ppOr)++;
      }
      fts3EvalTokenCosts(pCsr, pRoot, pExpr->pLeft, ppTC, ppOr, pRc);





      fts3EvalTokenCosts(pCsr, pRoot, pExpr->pRight, ppTC, ppOr, pRc);
    }
  }
}

static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){
  if( pCsr->nRowAvg==0 ){
................................................................................
  /* If bOptOk is true, check if there are any tokens that should be deferred.
  */
  if( rc==SQLITE_OK && bOptOk && nToken>1 && pTab->bHasStat ){
    Fts3TokenAndCost *aTC;
    Fts3Expr **apOr;
    aTC = (Fts3TokenAndCost *)sqlite3_malloc(
        sizeof(Fts3TokenAndCost) * nToken
      + sizeof(Fts3Expr *) * nOr
    );
    apOr = (Fts3Expr **)&aTC[nToken];

    if( !aTC ){
      rc = SQLITE_NOMEM;
    }else{
      int ii;
................................................................................
  Fts3Expr *pExpr,                /* Phrase expression */
  u32 *aiOut                      /* Array to write results into (see above) */
){
  Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
  int rc = SQLITE_OK;
  int iCol;

  if( pExpr->bDeferred ){
    assert( pCsr->nDoc>0 );
    for(iCol=0; iCol<pTab->nColumn; iCol++){
      aiOut[iCol*3 + 1] = pCsr->nDoc;
      aiOut[iCol*3 + 2] = pCsr->nDoc;
    }
  }else{
    rc = fts3EvalGatherStats(pCsr, pExpr);







|
|



>
>
>
>
>







 







|







 







|







3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
....
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
....
4311
4312
4313
4314
4315
4316
4317
4318
4319
4320
4321
4322
4323
4324
4325
        pTC->pRoot = pRoot;
        pTC->pToken = &pPhrase->aToken[i];
        pTC->iCol = pPhrase->iColumn;
        *pRc = sqlite3Fts3MsrOvfl(pCsr, pTC->pToken->pSegcsr, &pTC->nOvfl);
      }
    }else if( pExpr->eType!=FTSQUERY_NOT ){
      if( pExpr->eType==FTSQUERY_OR ){
        pRoot = pExpr->pLeft;
        **ppOr = pRoot;
        (*ppOr)++;
      }
      fts3EvalTokenCosts(pCsr, pRoot, pExpr->pLeft, ppTC, ppOr, pRc);
      if( pExpr->eType==FTSQUERY_OR ){
        pRoot = pExpr->pRight;
        **ppOr = pRoot;
        (*ppOr)++;
      }
      fts3EvalTokenCosts(pCsr, pRoot, pExpr->pRight, ppTC, ppOr, pRc);
    }
  }
}

static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){
  if( pCsr->nRowAvg==0 ){
................................................................................
  /* If bOptOk is true, check if there are any tokens that should be deferred.
  */
  if( rc==SQLITE_OK && bOptOk && nToken>1 && pTab->bHasStat ){
    Fts3TokenAndCost *aTC;
    Fts3Expr **apOr;
    aTC = (Fts3TokenAndCost *)sqlite3_malloc(
        sizeof(Fts3TokenAndCost) * nToken
      + sizeof(Fts3Expr *) * nOr * 2
    );
    apOr = (Fts3Expr **)&aTC[nToken];

    if( !aTC ){
      rc = SQLITE_NOMEM;
    }else{
      int ii;
................................................................................
  Fts3Expr *pExpr,                /* Phrase expression */
  u32 *aiOut                      /* Array to write results into (see above) */
){
  Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
  int rc = SQLITE_OK;
  int iCol;

  if( pExpr->bDeferred && pExpr->pParent->eType!=FTSQUERY_NEAR ){
    assert( pCsr->nDoc>0 );
    for(iCol=0; iCol<pTab->nColumn; iCol++){
      aiOut[iCol*3 + 1] = pCsr->nDoc;
      aiOut[iCol*3 + 2] = pCsr->nDoc;
    }
  }else{
    rc = fts3EvalGatherStats(pCsr, pExpr);

Changes to test/fts3auto.test.

89
90
91
92
93
94
95
96
97
98
99
100

































101
102
103
104
105
106
107
108
109
110
111

112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
...
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
...
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298

299
300
301

302
303
304
305
306
307
308
309
310
311

312
313
314
315
316
317
318
...
323
324
325
326
327
328
329
330
331
332
333






























































334
335
336
      set nDoc($iPhrase,$iCol) 0
    }
  }
  foreach key [array names counts] {
    set iCol [lindex [split $key ,] 1]
    set iPhrase 0
    foreach c $counts($key) {
      if {$c>0} { incr nHit($iPhrase,$iCol) 1 }
      incr nDoc($iPhrase,$iCol) $c
      incr iPhrase
    }
  }


































  # Set up the aMatchinfo array. For each document, set aMatchinfo($docid) to
  # contain the output of matchinfo('x') for the document.
  #
  foreach docid [array names hits] {
    set mi [list]
    for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
      for {set iCol 0} {$iCol<$nCol} {incr iCol} {
        lappend mi [lindex $counts($docid,$iCol) $iPhrase]
        lappend mi $nDoc($iPhrase,$iCol)
        lappend mi $nHit($iPhrase,$iCol)

      }
    }
    set aMatchinfo($docid) $mi
  }

  # Set up the nullentry output.
  #
  set nullentry [list]
  for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
    for {set iCol 0} {$iCol<$nCol} {incr iCol} {
      lappend nullentry 0 $nDoc($iPhrase,$iCol) $nHit($iPhrase,$iCol)
    }
  }
}


proc matching_brackets {expr} {
  if {[string range $expr 0 0]!="(" || [string range $expr end end] !=")"} { 
................................................................................

        set nullentry [concat $nullleft $nullright]
      }
    }
  }
}

proc do_near_test {tn tbl expr} {

  get_near_results $tbl $expr aMatchinfo
  set match $expr

  set matchinfo_asc [list]
  foreach docid [lsort -integer -incr [array names aMatchinfo]] {
    lappend matchinfo_asc $docid $aMatchinfo($docid)
................................................................................

set A "a b c d e f g h i j k l m n o p q r s t u v w x y z"
test_fts3_near_match 1.3.1 $A {"c d" 5 "i j" 1 "e f"}      {0 0 0}
test_fts3_near_match 1.3.2 $A {"c d" 5 "i j" 2 "e f"}      {1 1 1}

#--------------------------------------------------------------------------
# Test cases fts3auto-2.* run some simple tests using the 
# [do_near_test] proc.
#
foreach {tn create} {
  1    "CREATE VIRTUAL TABLE t1 USING fts4(a, b)"
  2    "CREATE VIRTUAL TABLE t1 USING fts4(a, b, order=DESC)"
  3    "CREATE VIRTUAL TABLE t1 USING fts4(a, b, order=ASC)"
  4    "CREATE VIRTUAL TABLE t1 USING fts4(a, b, prefix=1)"
  5    "CREATE VIRTUAL TABLE t1 USING fts4(a, b, order=DESC, prefix=1)"
  6    "CREATE VIRTUAL TABLE t1 USING fts4(a, b, order=ASC, prefix=1)"
} {

  do_test 2.$tn.1 {
    catchsql { DROP TABLE t1 }
    execsql  $create

    for {set i 0} {$i<32} {incr i} {
      set doc [list]
      if {$i&0x01} {lappend doc one}
      if {$i&0x02} {lappend doc two}
      if {$i&0x04} {lappend doc three}
      if {$i&0x08} {lappend doc four}
      if {$i&0x10} {lappend doc five}
      execsql { INSERT INTO t1 VALUES($doc, null) }
    }
  } {}

  foreach {tn2 expr} {
    1     {one}
    2     {one NEAR/1 five}
    3     {t*}
    4     {t* NEAR/0 five}
    5     {o* NEAR/1 f*}
    6     {one NEAR five NEAR two NEAR four NEAR three}
................................................................................
    11    {one AND two OR three}
    12    {three OR one AND two}
    13    {(three OR one) AND two}
    14    {(three OR one) AND two NOT (five NOT four)}
    15    {"one two"}
    16    {"one two" NOT "three four"}
  } {
    do_near_test 2.$tn.2.$tn2 t1 $expr
  }
}































































set sqlite_fts3_enable_parentheses $sfep
finish_test








|
|



>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>









<

>










|







 







|







 







|


|
|
|
|
|
|

>


<
>










>







 







|



>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>



89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142

143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
...
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
...
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334

335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
...
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
      set nDoc($iPhrase,$iCol) 0
    }
  }
  foreach key [array names counts] {
    set iCol [lindex [split $key ,] 1]
    set iPhrase 0
    foreach c $counts($key) {
      if {$c>0} { incr nDoc($iPhrase,$iCol) 1 }
      incr nHit($iPhrase,$iCol) $c
      incr iPhrase
    }
  }

  if {[info exists ::fts3_deferred] && [llength $expr]==1} {
    set phrase [lindex $expr 0]
    set rewritten [list]
    set partial 0
    foreach tok $phrase {
      if {[lsearch $::fts3_deferred $tok]>=0} {
        lappend rewritten *
      } else {
        lappend rewritten $tok
        set partial 1
      }
    }
    if {$partial==0} {
      set tblsize [db one "SELECT count(*) FROM $tbl"]
      for {set iCol 0} {$iCol < $nCol} {incr iCol} {
        set nHit(0,$iCol) $tblsize
        set nDoc(0,$iCol) $tblsize
      }
    } elseif {$rewritten != $phrase} {
      while {[lindex $rewritten end] == "*"} {
        set rewritten [lrange $rewritten 0 end-1]
      }
      while {[lindex $rewritten 0] == "*"} {
        set rewritten [lrange $rewritten 1 end]
      }
      get_single_near_results $tbl [list $rewritten] aRewrite nullentry
      foreach docid [array names hits] {
        set aMatchinfo($docid) $aRewrite($docid)
      }
      return
    }
  }

  # Set up the aMatchinfo array. For each document, set aMatchinfo($docid) to
  # contain the output of matchinfo('x') for the document.
  #
  foreach docid [array names hits] {
    set mi [list]
    for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
      for {set iCol 0} {$iCol<$nCol} {incr iCol} {
        lappend mi [lindex $counts($docid,$iCol) $iPhrase]

        lappend mi $nHit($iPhrase,$iCol)
        lappend mi $nDoc($iPhrase,$iCol)
      }
    }
    set aMatchinfo($docid) $mi
  }

  # Set up the nullentry output.
  #
  set nullentry [list]
  for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
    for {set iCol 0} {$iCol<$nCol} {incr iCol} {
      lappend nullentry 0 $nHit($iPhrase,$iCol) $nDoc($iPhrase,$iCol)
    }
  }
}


proc matching_brackets {expr} {
  if {[string range $expr 0 0]!="(" || [string range $expr end end] !=")"} { 
................................................................................

        set nullentry [concat $nullleft $nullright]
      }
    }
  }
}

proc do_fts3query_test {tn tbl expr} {

  get_near_results $tbl $expr aMatchinfo
  set match $expr

  set matchinfo_asc [list]
  foreach docid [lsort -integer -incr [array names aMatchinfo]] {
    lappend matchinfo_asc $docid $aMatchinfo($docid)
................................................................................

set A "a b c d e f g h i j k l m n o p q r s t u v w x y z"
test_fts3_near_match 1.3.1 $A {"c d" 5 "i j" 1 "e f"}      {0 0 0}
test_fts3_near_match 1.3.2 $A {"c d" 5 "i j" 2 "e f"}      {1 1 1}

#--------------------------------------------------------------------------
# Test cases fts3auto-2.* run some simple tests using the 
# [do_fts3query_test] proc.
#
foreach {tn create} {
  1    "fts4(a, b)"
  2    "fts4(a, b, order=DESC)"
  3    "fts4(a, b, order=ASC)"
  4    "fts4(a, b, prefix=1)"
  5    "fts4(a, b, order=DESC, prefix=1)"
  6    "fts4(a, b, order=ASC, prefix=1)"
} {
break
  do_test 2.$tn.1 {
    catchsql { DROP TABLE t1 }

    execsql  "CREATE VIRTUAL TABLE t1 USING $create"
    for {set i 0} {$i<32} {incr i} {
      set doc [list]
      if {$i&0x01} {lappend doc one}
      if {$i&0x02} {lappend doc two}
      if {$i&0x04} {lappend doc three}
      if {$i&0x08} {lappend doc four}
      if {$i&0x10} {lappend doc five}
      execsql { INSERT INTO t1 VALUES($doc, null) }
    }
  } {}

  foreach {tn2 expr} {
    1     {one}
    2     {one NEAR/1 five}
    3     {t*}
    4     {t* NEAR/0 five}
    5     {o* NEAR/1 f*}
    6     {one NEAR five NEAR two NEAR four NEAR three}
................................................................................
    11    {one AND two OR three}
    12    {three OR one AND two}
    13    {(three OR one) AND two}
    14    {(three OR one) AND two NOT (five NOT four)}
    15    {"one two"}
    16    {"one two" NOT "three four"}
  } {
    do_fts3query_test 2.$tn.2.$tn2 t1 $expr
  }
}

#--------------------------------------------------------------------------
# Some test cases involving deferred tokens.
#
proc make_token_deferrable {tbl token} {
  set nRow [db one "SELECT count(*) FROM $tbl"]
  set pgsz [db one "PRAGMA page_size"]

  execsql "INSERT INTO $tbl ($tbl) VALUES('maxpending=100000000')"
  execsql BEGIN
  for {set i 0} {$i < ($nRow * $pgsz * 1.2)/100} {incr i} {
    set doc [string repeat "$token " 100]
    execsql "INSERT INTO $tbl VALUES(\$doc)"
  }
  execsql "INSERT INTO $tbl VALUES('aaaaaaa ${token}aaaaa')"
  execsql COMMIT
}

foreach {tn create} {
  1    "fts4(x)"
  2    "fts4(x, order=DESC)"
} {
  catchsql { DROP TABLE t1 }
  execsql  "CREATE VIRTUAL TABLE t1 USING $create"
  do_execsql_test 3.$tn.1 {
    INSERT INTO t1 VALUES('a b c d e f g h i j k');
    INSERT INTO t1 VALUES('b c d e f g h i j k a');
    INSERT INTO t1 VALUES('c d e f g h i j k a b');
    INSERT INTO t1 VALUES('d e f g h i j k a b c');
    INSERT INTO t1 VALUES('e f g h i j k a b c d');
    INSERT INTO t1 VALUES('f g h i j k a b c d e');
    INSERT INTO t1 VALUES('a c e g i k');
    INSERT INTO t1 VALUES('a d g j');
    INSERT INTO t1 VALUES('c a b');
  }

  make_token_deferrable t1 c

  set ::fts3_deferred [list]
  foreach {tn2 expr} {
    1     {a OR c}
  } {
    do_fts3query_test 3.$tn.2.$tn2 t1 $expr
  }

  set ::fts3_deferred [list c]
  execsql { 
    UPDATE t1_segments 
    SET block = zeroblob(length(block)) 
    WHERE length(block)>10000 AND 0
  }
  foreach {tn2 expr} {
    1     {a NEAR c}
    2     {a AND c}
    3     {"a c"}
    4     {"c a"}
    5     {"a c" NEAR/1 g}
    6     {"a c" NEAR/0 g}
  } {
    do_fts3query_test 3.$tn.2.$tn2 t1 $expr
  }
}

set sqlite_fts3_enable_parentheses $sfep
finish_test

Changes to test/fts3defer2.test.

52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
  SELECT content FROM t1 WHERE t1 MATCH 'f (e a)';
} {{a b c d e f a x y}}

do_execsql_test 1.2.1 {
  SELECT content FROM t1 WHERE t1 MATCH 'f (e NEAR/2 a)';
} {{a b c d e f a x y}}

breakpoint
do_execsql_test 1.2.2 {
  SELECT snippet(t1, '[', ']'), offsets(t1), mit(matchinfo(t1, 'pcxnal'))
  FROM t1 WHERE t1 MATCH 'f (e NEAR/2 a)';
} [list                              \
   {a b c d [e] [f] [a] x y}         \
   {0 1 8 1 0 0 10 1 0 2 12 1}       \
   [list 3 1   1 1 1   1 1 1   1 8 8   8 5001 9]
]

do_execsql_test 1.2.3 {
  SELECT snippet(t1, '[', ']'), offsets(t1), mit(matchinfo(t1, 'pcxnal'))
  FROM t1 WHERE t1 MATCH 'f (e NEAR/3 a)';
} [list                                 \
   {[a] b c d [e] [f] [a] x y}          \
   {0 2 0 1 0 1 8 1 0 0 10 1 0 2 12 1}  \
   [list 3 1   1 1 1   1 1 1   2 8 8   8 5001 9]
]

do_execsql_test 1.3.1 { DROP TABLE t1 }

#-----------------------------------------------------------------------------
# Test cases fts3defer2-2.* focus specifically on the matchinfo function.
# 







<






|








|







52
53
54
55
56
57
58

59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
  SELECT content FROM t1 WHERE t1 MATCH 'f (e a)';
} {{a b c d e f a x y}}

do_execsql_test 1.2.1 {
  SELECT content FROM t1 WHERE t1 MATCH 'f (e NEAR/2 a)';
} {{a b c d e f a x y}}


do_execsql_test 1.2.2 {
  SELECT snippet(t1, '[', ']'), offsets(t1), mit(matchinfo(t1, 'pcxnal'))
  FROM t1 WHERE t1 MATCH 'f (e NEAR/2 a)';
} [list                              \
   {a b c d [e] [f] [a] x y}         \
   {0 1 8 1 0 0 10 1 0 2 12 1}       \
   [list 3 1   1 1 1   1 1 1   1 1 1   8 5001 9]
]

do_execsql_test 1.2.3 {
  SELECT snippet(t1, '[', ']'), offsets(t1), mit(matchinfo(t1, 'pcxnal'))
  FROM t1 WHERE t1 MATCH 'f (e NEAR/3 a)';
} [list                                 \
   {[a] b c d [e] [f] [a] x y}          \
   {0 2 0 1 0 1 8 1 0 0 10 1 0 2 12 1}  \
   [list 3 1   1 1 1   1 1 1   2 2 1   8 5001 9]
]

do_execsql_test 1.3.1 { DROP TABLE t1 }

#-----------------------------------------------------------------------------
# Test cases fts3defer2-2.* focus specifically on the matchinfo function.
#