SQLite

Check-in [5e9d8ccae9]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add coverage tests for fts3_snippet.c. Also fixes related to the same.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 5e9d8ccae9731f380527463ef87ddcd216b4b721
User & Date: dan 2010-01-15 17:25:53.000
Context
2010-01-15
17:33
Fix test numbering in fts3snippet.test. (check-in: 34f6854685 user: dan tags: trunk)
17:25
Add coverage tests for fts3_snippet.c. Also fixes related to the same. (check-in: 5e9d8ccae9 user: dan tags: trunk)
2010-01-14
11:45
Add a test to e_fts3.test for the matchinfo example in fts3.html. (check-in: e5336edc55 user: dan tags: trunk)
Changes
Unified Diff Ignore Whitespace Patch
Changes to ext/fts3/fts3_snippet.c.
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
*/
static int fts3ExprNearTrim(Fts3Expr *pExpr){
  int rc = SQLITE_OK;
  Fts3Expr *pParent = pExpr->pParent;

  assert( pExpr->eType==FTSQUERY_PHRASE );
  while( rc==SQLITE_OK
   && pExpr->aDoclist && pParent 
   && pParent->eType==FTSQUERY_NEAR 
   && pParent->pRight==pExpr 
  ){
    /* This expression (pExpr) is the right-hand-side of a NEAR operator. 
    ** Find the expression to the left of the same operator.
    */
    int nNear = pParent->nNear;







|







175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
*/
static int fts3ExprNearTrim(Fts3Expr *pExpr){
  int rc = SQLITE_OK;
  Fts3Expr *pParent = pExpr->pParent;

  assert( pExpr->eType==FTSQUERY_PHRASE );
  while( rc==SQLITE_OK
   && pParent 
   && pParent->eType==FTSQUERY_NEAR 
   && pParent->pRight==pExpr 
  ){
    /* This expression (pExpr) is the right-hand-side of a NEAR operator. 
    ** Find the expression to the left of the same operator.
    */
    int nNear = pParent->nNear;
764
765
766
767
768
769
770

771
772
773
774
775
776
777
778
779
    int nHit;
    sqlite3_int64 iCol = 0;
    if( *pCsr==0x01 ){
      pCsr++;
      pCsr += sqlite3Fts3GetVarint(pCsr, &iCol);
    }
    nHit = fts3ColumnlistCount(&pCsr);

    if( isGlobal ){
      if( nHit ) aOut[iCol*3+1]++;
    }
    aOut[iCol*3] += nHit;
  }
  pCsr++;
  *pp = pCsr;
}








>

|







764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
    int nHit;
    sqlite3_int64 iCol = 0;
    if( *pCsr==0x01 ){
      pCsr++;
      pCsr += sqlite3Fts3GetVarint(pCsr, &iCol);
    }
    nHit = fts3ColumnlistCount(&pCsr);
    assert( nHit>0 );
    if( isGlobal ){
      aOut[iCol*3+1]++;
    }
    aOut[iCol*3] += nHit;
  }
  pCsr++;
  *pp = pCsr;
}

910
911
912
913
914
915
916

917
918
919
920
921
922
923



924
925
926
927
928
929
930
  ** cannot be found, the second iteration of the loop attempts to locate
  ** a pair of fragments, and so on.
  */
  int nSnippet = 0;               /* Number of fragments in this snippet */
  SnippetFragment aSnippet[4];    /* Maximum of 4 fragments per snippet */
  int nFToken = -1;               /* Number of tokens in each fragment */


  do {
    int iSnip;                    /* Loop counter 0..nSnippet-1 */
    u64 mCovered = 0;             /* Bitmask of phrases covered by snippet */
    u64 mSeen = 0;                /* Bitmask of phrases seen by BestSnippet() */

    nSnippet++;
    nFToken = (nToken+nSnippet-1) / nSnippet;




    for(iSnip=0; iSnip<nSnippet; iSnip++){
      int iBestScore = -1;        /* Best score of columns checked so far */
      int iRead;                  /* Used to iterate through columns */
      SnippetFragment *pFragment = &aSnippet[iSnip];

      memset(pFragment, 0, sizeof(*pFragment));







>
|




|
|
>
>
>







911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
  ** cannot be found, the second iteration of the loop attempts to locate
  ** a pair of fragments, and so on.
  */
  int nSnippet = 0;               /* Number of fragments in this snippet */
  SnippetFragment aSnippet[4];    /* Maximum of 4 fragments per snippet */
  int nFToken = -1;               /* Number of tokens in each fragment */

  for(nSnippet=1; 1; nSnippet++){

    int iSnip;                    /* Loop counter 0..nSnippet-1 */
    u64 mCovered = 0;             /* Bitmask of phrases covered by snippet */
    u64 mSeen = 0;                /* Bitmask of phrases seen by BestSnippet() */

    if( nToken>=0 ){
      nFToken = (nToken+nSnippet-1) / nSnippet;
    }else{
      nFToken = -1 * nToken;
    }

    for(iSnip=0; iSnip<nSnippet; iSnip++){
      int iBestScore = -1;        /* Best score of columns checked so far */
      int iRead;                  /* Used to iterate through columns */
      SnippetFragment *pFragment = &aSnippet[iSnip];

      memset(pFragment, 0, sizeof(*pFragment));
952
953
954
955
956
957
958
959
960

961
962
963
964
965
966
967
      mCovered |= pFragment->covered;
    }

    /* If all query phrases seen by fts3BestSnippet() are present in at least
    ** one of the nSnippet snippet fragments, break out of the loop.
    */
    assert( (mCovered&mSeen)==mCovered );
    if( mSeen==mCovered ) break;
  }while( nSnippet<SizeofArray(aSnippet) );


  assert( nFToken>0 );

  for(i=0; i<nSnippet && rc==SQLITE_OK; i++){
    rc = fts3SnippetText(pCsr, &aSnippet[i], 
        i, (i==nSnippet-1), nFToken, zStart, zEnd, zEllipsis, &res
    );







|
<
>







957
958
959
960
961
962
963
964

965
966
967
968
969
970
971
972
      mCovered |= pFragment->covered;
    }

    /* If all query phrases seen by fts3BestSnippet() are present in at least
    ** one of the nSnippet snippet fragments, break out of the loop.
    */
    assert( (mCovered&mSeen)==mCovered );
    if( mSeen==mCovered || nSnippet==SizeofArray(aSnippet) ) break;

  }

  assert( nFToken>0 );

  for(i=0; i<nSnippet && rc==SQLITE_OK; i++){
    rc = fts3SnippetText(pCsr, &aSnippet[i], 
        i, (i==nSnippet-1), nFToken, zStart, zEnd, zEllipsis, &res
    );
1059
1060
1061
1062
1063
1064
1065
1066



1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
    sqlite3_tokenizer_cursor *pC; /* Tokenizer cursor */
    int iStart;
    int iEnd;
    int iCurrent;
    const char *zDoc;
    int nDoc;

    /* Initialize the contents of sCtx.aTerm[] for column iCol. */



    sCtx.iCol = iCol;
    sCtx.iTerm = 0;
    rc = fts3ExprIterate(pCsr->pExpr, fts3ExprTermOffsetInit, (void *)&sCtx);
    if( rc!=SQLITE_OK ) goto offsets_out;

    /* Retreive the text stored in column iCol. If an SQL NULL is stored 
    ** in column iCol, jump immediately to the next iteration of the loop.
    ** If an OOM occurs while retrieving the data (this can happen if SQLite
    ** needs to transform the data from utf-16 to utf-8), return SQLITE_NOMEM 
    ** to the caller. 
    */







|
>
>
>


|
<







1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077

1078
1079
1080
1081
1082
1083
1084
    sqlite3_tokenizer_cursor *pC; /* Tokenizer cursor */
    int iStart;
    int iEnd;
    int iCurrent;
    const char *zDoc;
    int nDoc;

    /* Initialize the contents of sCtx.aTerm[] for column iCol. There is 
    ** no way that this operation can fail, so the return code from
    ** fts3ExprIterate() can be discarded.
    */
    sCtx.iCol = iCol;
    sCtx.iTerm = 0;
    (void)fts3ExprIterate(pCsr->pExpr, fts3ExprTermOffsetInit, (void *)&sCtx);


    /* Retreive the text stored in column iCol. If an SQL NULL is stored 
    ** in column iCol, jump immediately to the next iteration of the loop.
    ** If an OOM occurs while retrieving the data (this can happen if SQLite
    ** needs to transform the data from utf-16 to utf-8), return SQLITE_NOMEM 
    ** to the caller. 
    */
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
              "%d %d %d %d ", iCol, pTerm-sCtx.aTerm, iStart, iEnd-iStart
          );
          rc = fts3StringAppend(&res, aBuffer, -1);
        }
      }
    }
    if( rc==SQLITE_DONE ){
      rc = SQLITE_ERROR;
    }

    pMod->xClose(pC);
    if( rc!=SQLITE_OK ) goto offsets_out;
  }

 offsets_out:







|







1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
              "%d %d %d %d ", iCol, pTerm-sCtx.aTerm, iStart, iEnd-iStart
          );
          rc = fts3StringAppend(&res, aBuffer, -1);
        }
      }
    }
    if( rc==SQLITE_DONE ){
      rc = SQLITE_CORRUPT;
    }

    pMod->xClose(pC);
    if( rc!=SQLITE_OK ) goto offsets_out;
  }

 offsets_out:
Changes to test/fts3_common.tcl.
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
}

proc doPassiveTest {isRestart name sql catchres} {
  if {![info exists ::DO_MALLOC_TEST]} { set ::DO_MALLOC_TEST 1 }

  switch $::DO_MALLOC_TEST {
    0 { # No malloc failures.
      do_test $name [list catchsql $sql] $catchres
      return
    }
    1 { # Simulate transient failures.
      set nRepeat 1
      set zName "transient"
      set nStartLimit 100000
      set nBackup 1







|







322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
}

proc doPassiveTest {isRestart name sql catchres} {
  if {![info exists ::DO_MALLOC_TEST]} { set ::DO_MALLOC_TEST 1 }

  switch $::DO_MALLOC_TEST {
    0 { # No malloc failures.
      do_test $name [list set {} [uplevel [list catchsql $sql]]] $catchres
      return
    }
    1 { # Simulate transient failures.
      set nRepeat 1
      set zName "transient"
      set nStartLimit 100000
      set nBackup 1
Changes to test/fts3snippet.test.
1
2
3
4
5
6
7
8
9
10
11




12
13
14
15
16
17





18
19
20
21
22
23
24
25
26
27








































































































28





29
30
31
32
33
34
35
36
37



38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59

60
61
62

63





64
65
66
67
68
69

70
71
72
73
74
75
76
77
78








79
80
81
82
83
84
85
86
87










88



89



90






91





92














93










94






95








96






97
98


99






100













101





102



103





































104
105







106











107



108



















































# 2010 January 07
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#*************************************************************************
#





set testdir [file dirname $argv0]
source $testdir/tester.tcl

# If SQLITE_ENABLE_FTS3 is not defined, omit this file.
ifcapable !fts3 { finish_test ; return }






# Transform the list $L to its "normal" form. So that it can be compared to
# another list with the same set of elements using [string compare].
#
proc normalize {L} {
  set ret [list]
  foreach l $L {lappend ret $l}
  return $ret
}









































































































do_test fts3snippet-1.1 {





  execsql {
    CREATE VIRTUAL TABLE ft USING fts3;
    INSERT INTO ft VALUES('xxx xxx xxx xxx');
  }
} {}

do_test fts3snippet-1.2 {
  execsql { SELECT offsets(ft) FROM ft WHERE ft MATCH 'xxx' }
} {{0 0 0 3 0 0 4 3 0 0 8 3 0 0 12 3}}




do_test fts3snippet-1.3 {
  execsql { SELECT offsets(ft) FROM ft WHERE ft MATCH '"xxx xxx"' }
} [list [normalize {
    0 0  0 3 
    0 0  4 3 
    0 1  4 3 
    0 0  8 3 
    0 1  8 3 
    0 1 12 3
}]]


do_test fts3snippet-1.4 {
  execsql { SELECT offsets(ft) FROM ft WHERE ft MATCH '"xxx xxx" xxx' }
} [list [normalize {
    0 0  0 3 
    0 2  0 3 
    0 0  4 3 
    0 1  4 3 
    0 2  4 3 
    0 0  8 3 

    0 1  8 3 
    0 2  8 3 
    0 1 12 3

    0 2 12 3





}]]

do_test fts3snippet-1.5 {
  execsql { SELECT offsets(ft) FROM ft WHERE ft MATCH 'xxx "xxx xxx"' }
} [list [normalize {
    0 0  0 3 

    0 1  0 3 
    0 0  4 3 
    0 1  4 3 
    0 2  4 3 
    0 0  8 3 
    0 1  8 3 
    0 2  8 3 
    0 0 12 3
    0 2 12 3








}]]

do_test fts3snippet-2.1 {
  execsql {
    DROP TABLE IF EXISTS ft;
    CREATE VIRTUAL TABLE ft USING fts3;
    INSERT INTO ft VALUES('one two three four five six seven eight nine ten');
  }
} {}










foreach {tn expr res} {



   1 one       "[one] two three four five..."



   2 two       "one [two] three four five..."






   3 three     "one two [three] four five..."





   4 four      "...two three [four] five six..."














   5 five      "...three four [five] six seven..."










   6 six       "...four five [six] seven eight..."






   7 seven     "...five six [seven] eight nine..."








   8 eight     "...six seven [eight] nine ten"






   9 nine      "...six seven eight [nine] ten"
  10 ten       "...six seven eight nine [ten]"


} {






  do_test fts3snippet-2.2.$tn {













    execsql {





      SELECT snippet(ft, '[', ']', '...', 0, 5) FROM ft WHERE ft MATCH $expr



    }





































  } [list $res]
}



















finish_test


































































>
>
>
>






>
>
>
>
>










>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|
>
>
>
>
>
|
|
|
|
|
<
<
<
|
>
>
>
|
<
|
<
|
<
<
|
<
|
<
|
|
<
<
<
|
<
<
<
|
|
>
|
<
|
>
|
>
>
>
>
>
|

|
<
|
|
>
|
|
|
|
|
|
|
|
|
>
>
>
>
>
>
>
>
|
|
|
|
|
|
|
|
|
>
>
>
>
>
>
>
>
>
>
|
>
>
>
|
>
>
>
|
>
>
>
>
>
>
|
>
>
>
>
>
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|
>
>
>
>
>
>
>
>
>
>
|
>
>
>
>
>
>
|
>
>
>
>
>
>
>
>
|
>
>
>
>
>
>
|
|
>
>
|
>
>
>
>
>
>
|
>
>
>
>
>
>
>
>
>
>
>
>
>

>
>
>
>
>
|
>
>
>
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|
|
>
>
>
>
>
>
>
|
>
>
>
>
>
>
>
>
>
>
>
|
>
>
>
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151



152
153
154
155
156

157

158


159

160

161
162



163



164
165
166
167

168
169
170
171
172
173
174
175
176
177
178

179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
# 2010 January 07
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#*************************************************************************
#
# The tests in this file test the FTS3 auxillary functions offsets(), 
# snippet() and matchinfo() work. At time of writing, running this file 
# provides full coverage of fts3_snippet.c.
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# If SQLITE_ENABLE_FTS3 is not defined, omit this file.
ifcapable !fts3 { finish_test ; return }
source $testdir/fts3_common.tcl
source $testdir/malloc_common.tcl

set sqlite_fts3_enable_parentheses 1
set DO_MALLOC_TEST 0

# Transform the list $L to its "normal" form. So that it can be compared to
# another list with the same set of elements using [string compare].
#
proc normalize {L} {
  set ret [list]
  foreach l $L {lappend ret $l}
  return $ret
}

proc do_offsets_test {name expr args} {
  set result [list]
  foreach a $args {
    lappend result [normalize $a]
  }
  do_select_test $name {
    SELECT offsets(ft) FROM ft WHERE ft MATCH $expr
  } $result
}
  
# Document text used by a few tests. Contains the English names of all
# integers between 1 and 300.
#
set numbers [normalize {
  one two three four five six seven eight nine ten eleven twelve thirteen
  fourteen fifteen sixteen seventeen eighteen nineteen twenty twentyone
  twentytwo twentythree twentyfour twentyfive twentysix twentyseven
  twentyeight twentynine thirty thirtyone thirtytwo thirtythree thirtyfour
  thirtyfive thirtysix thirtyseven thirtyeight thirtynine forty fortyone
  fortytwo fortythree fortyfour fortyfive fortysix fortyseven fortyeight
  fortynine fifty fiftyone fiftytwo fiftythree fiftyfour fiftyfive fiftysix
  fiftyseven fiftyeight fiftynine sixty sixtyone sixtytwo sixtythree sixtyfour
  sixtyfive sixtysix sixtyseven sixtyeight sixtynine seventy seventyone
  seventytwo seventythree seventyfour seventyfive seventysix seventyseven
  seventyeight seventynine eighty eightyone eightytwo eightythree eightyfour
  eightyfive eightysix eightyseven eightyeight eightynine ninety ninetyone
  ninetytwo ninetythree ninetyfour ninetyfive ninetysix ninetyseven
  ninetyeight ninetynine onehundred onehundredone onehundredtwo
  onehundredthree onehundredfour onehundredfive onehundredsix onehundredseven
  onehundredeight onehundrednine onehundredten onehundredeleven
  onehundredtwelve onehundredthirteen onehundredfourteen onehundredfifteen
  onehundredsixteen onehundredseventeen onehundredeighteen onehundrednineteen
  onehundredtwenty onehundredtwentyone onehundredtwentytwo
  onehundredtwentythree onehundredtwentyfour onehundredtwentyfive
  onehundredtwentysix onehundredtwentyseven onehundredtwentyeight
  onehundredtwentynine onehundredthirty onehundredthirtyone
  onehundredthirtytwo onehundredthirtythree onehundredthirtyfour
  onehundredthirtyfive onehundredthirtysix onehundredthirtyseven
  onehundredthirtyeight onehundredthirtynine onehundredforty
  onehundredfortyone onehundredfortytwo onehundredfortythree
  onehundredfortyfour onehundredfortyfive onehundredfortysix
  onehundredfortyseven onehundredfortyeight onehundredfortynine
  onehundredfifty onehundredfiftyone onehundredfiftytwo onehundredfiftythree
  onehundredfiftyfour onehundredfiftyfive onehundredfiftysix
  onehundredfiftyseven onehundredfiftyeight onehundredfiftynine
  onehundredsixty onehundredsixtyone onehundredsixtytwo onehundredsixtythree
  onehundredsixtyfour onehundredsixtyfive onehundredsixtysix
  onehundredsixtyseven onehundredsixtyeight onehundredsixtynine
  onehundredseventy onehundredseventyone onehundredseventytwo
  onehundredseventythree onehundredseventyfour onehundredseventyfive
  onehundredseventysix onehundredseventyseven onehundredseventyeight
  onehundredseventynine onehundredeighty onehundredeightyone
  onehundredeightytwo onehundredeightythree onehundredeightyfour
  onehundredeightyfive onehundredeightysix onehundredeightyseven
  onehundredeightyeight onehundredeightynine onehundredninety
  onehundredninetyone onehundredninetytwo onehundredninetythree
  onehundredninetyfour onehundredninetyfive onehundredninetysix
  onehundredninetyseven onehundredninetyeight onehundredninetynine twohundred
  twohundredone twohundredtwo twohundredthree twohundredfour twohundredfive
  twohundredsix twohundredseven twohundredeight twohundrednine twohundredten
  twohundredeleven twohundredtwelve twohundredthirteen twohundredfourteen
  twohundredfifteen twohundredsixteen twohundredseventeen twohundredeighteen
  twohundrednineteen twohundredtwenty twohundredtwentyone twohundredtwentytwo
  twohundredtwentythree twohundredtwentyfour twohundredtwentyfive
  twohundredtwentysix twohundredtwentyseven twohundredtwentyeight
  twohundredtwentynine twohundredthirty twohundredthirtyone
  twohundredthirtytwo twohundredthirtythree twohundredthirtyfour
  twohundredthirtyfive twohundredthirtysix twohundredthirtyseven
  twohundredthirtyeight twohundredthirtynine twohundredforty
  twohundredfortyone twohundredfortytwo twohundredfortythree
  twohundredfortyfour twohundredfortyfive twohundredfortysix
  twohundredfortyseven twohundredfortyeight twohundredfortynine
  twohundredfifty twohundredfiftyone twohundredfiftytwo twohundredfiftythree
  twohundredfiftyfour twohundredfiftyfive twohundredfiftysix
  twohundredfiftyseven twohundredfiftyeight twohundredfiftynine
  twohundredsixty twohundredsixtyone twohundredsixtytwo twohundredsixtythree
  twohundredsixtyfour twohundredsixtyfive twohundredsixtysix
  twohundredsixtyseven twohundredsixtyeight twohundredsixtynine
  twohundredseventy twohundredseventyone twohundredseventytwo
  twohundredseventythree twohundredseventyfour twohundredseventyfive
  twohundredseventysix twohundredseventyseven twohundredseventyeight
  twohundredseventynine twohundredeighty twohundredeightyone
  twohundredeightytwo twohundredeightythree twohundredeightyfour
  twohundredeightyfive twohundredeightysix twohundredeightyseven
  twohundredeightyeight twohundredeightynine twohundredninety
  twohundredninetyone twohundredninetytwo twohundredninetythree
  twohundredninetyfour twohundredninetyfive twohundredninetysix
  twohundredninetyseven twohundredninetyeight twohundredninetynine
  threehundred
}]

foreach {DO_MALLOC_TEST enc} {
  0 utf8
  1 utf8
  1 utf16
} {

  db close
  file delete -force test.db
  sqlite3 db test.db
  db eval "PRAGMA encoding = \"$enc\""

  # Set variable $T to the test name prefix for this iteration of the loop.
  #
  set T "fts3snippet-$enc"

  ##########################################################################
  # Test the offset function.
  #
  do_test $T.1.1 {
    execsql {
      CREATE VIRTUAL TABLE ft USING fts3;
      INSERT INTO ft VALUES('xxx xxx xxx xxx');
    }
  } {}



  do_offsets_test $T.1.2 {xxx} {0 0 0 3 0 0 4 3 0 0 8 3 0 0 12 3}
  do_offsets_test $T.1.3 {"xxx xxx"} {
      0 0  0 3     0 0  4 3     0 1  4 3     0 0  8 3 
      0 1  8 3     0 1 12 3
  }

  do_offsets_test $T.1.4 {"xxx xxx" xxx} {

      0 0  0 3     0 2  0 3     0 0  4 3     0 1  4 3 


      0 2  4 3     0 0  8 3     0 1  8 3     0 2  8 3 

      0 1 12 3     0 2 12 3

  }
  do_offsets_test $T.1.5 {xxx "xxx xxx"} {



      0 0  0 3     0 1  0 3     0 0  4 3     0 1  4 3 



      0 2  4 3     0 0  8 3     0 1  8 3     0 2  8 3 
      0 0 12 3     0 2 12 3
  }


  do_test $T.9.1 {
    set v1 [lrange $numbers 0 99]
    execsql {
      DROP TABLE IF EXISTS ft;
      CREATE VIRTUAL TABLE ft USING fts3(a, b);
      INSERT INTO ft VALUES($v1, $numbers);
      INSERT INTO ft VALUES($v1, NULL);
    }
  } {}

  set off [string first "twohundred " $numbers]

  do_offsets_test $T.9.1 {twohundred} [list 1 0 $off 10]

  set off [string first "onehundred " $numbers]
  do_offsets_test $T.9.2 {onehundred} \
    [list 0 0 $off 10 1 0 $off 10] [list 0 0 $off 10]

  # Test a corruption case:
  execsql { UPDATE ft_content SET c1b = 'hello world' WHERE c1b = $numbers }
  do_error_test $T.9.3 {
    SELECT offsets(ft) FROM ft WHERE ft MATCH 'onehundred'
  } {database disk image is malformed}
  
  ##########################################################################
  # Test the snippet function.
  #
  proc do_snippet_test {name expr iCol nTok args} {
    set res [list]
    foreach a $args { lappend res [string trim $a] }
    do_select_test $name {
      SELECT snippet(ft,'{','}','...',$iCol,$nTok) FROM ft WHERE ft MATCH $expr
    } $res
  }
  do_test $T.2.1 {
    execsql {
      DROP TABLE IF EXISTS ft;
      CREATE VIRTUAL TABLE ft USING fts3;
      INSERT INTO ft VALUES('one two three four five six seven eight nine ten');
    }
  } {}
  do_snippet_test $T.2.2  one    0 5 "{one} two three four five..."
  do_snippet_test $T.2.3  two    0 5 "one {two} three four five..."
  do_snippet_test $T.2.4  three  0 5 "one two {three} four five..."
  do_snippet_test $T.2.5  four   0 5 "...two three {four} five six..."
  do_snippet_test $T.2.6  five   0 5 "...three four {five} six seven..."
  do_snippet_test $T.2.7  six    0 5 "...four five {six} seven eight..."
  do_snippet_test $T.2.8  seven  0 5 "...five six {seven} eight nine..."
  do_snippet_test $T.2.9  eight  0 5 "...six seven {eight} nine ten"
  do_snippet_test $T.2.10 nine   0 5 "...six seven eight {nine} ten"
  do_snippet_test $T.2.11 ten    0 5 "...six seven eight nine {ten}"
  
  do_test $T.3.1 {
    execsql {
      INSERT INTO ft VALUES(
           'one two three four five '
        || 'six seven eight nine ten '
        || 'eleven twelve thirteen fourteen fifteen '
        || 'sixteen seventeen eighteen nineteen twenty '
        || 'one two three four five '
        || 'six seven eight nine ten '
        || 'eleven twelve thirteen fourteen fifteen '
        || 'sixteen seventeen eighteen nineteen twenty'
      );
    }
  } {}
  
  do_snippet_test $T.3.2 {one nine} 0 5 {
     {one} two three...eight {nine} ten
  } {
     {one} two three...eight {nine} ten...
  }
  
  do_snippet_test $T.3.3 {one nine} 0 -5 {
     {one} two three four five...six seven eight {nine} ten
  } {
     {one} two three four five...seven eight {nine} ten eleven...
  }
  do_snippet_test $T.3.3 {one nineteen} 0 -5 {
     ...eighteen {nineteen} twenty {one} two...
  }
  do_snippet_test $T.3.4 {two nineteen} 0 -5 {
     ...eighteen {nineteen} twenty one {two}...
  }
  do_snippet_test $T.3.5 {three nineteen} 0 -5 {
     ...{nineteen} twenty one two {three}...
  }
  
  do_snippet_test $T.3.6 {four nineteen} 0 -5 {
     ...two three {four} five six...seventeen eighteen {nineteen} twenty one...
  }
  do_snippet_test $T.3.7 {four NEAR nineteen} 0 -5 {
     ...seventeen eighteen {nineteen} twenty one...two three {four} five six...
  }
  
  do_snippet_test $T.3.8 {four nineteen} 0 5 {
     ...three {four} five...eighteen {nineteen} twenty...
  }
  do_snippet_test $T.3.9 {four NEAR nineteen} 0 5 {
     ...eighteen {nineteen} twenty...three {four} five...
  }
  do_snippet_test $T.3.10 {four NEAR nineteen} 0 -5 {
     ...seventeen eighteen {nineteen} twenty one...two three {four} five six...
  }
  do_snippet_test $T.3.11 {four NOT (nineteen twentyone)} 0 5 {
     ...two three {four} five six...
  } {
     ...two three {four} five six...
  }
  do_snippet_test $T.3.12 {four OR nineteen NEAR twentyone} 0 5 {
     ...two three {four} five six...
  } {
     ...two three {four} five six...
  }
  
  do_test $T.5.1 {
    execsql {
      DROP TABLE IF EXISTS ft;
      CREATE VIRTUAL TABLE ft USING fts3(a, b, c);
      INSERT INTO ft VALUES(
        'one two three four five', 
        'four five six seven eight', 
        'seven eight nine ten eleven'
      );
    }
  } {}
  
  do_snippet_test $T.5.2 {five} -1 3 {...three four {five}}
  do_snippet_test $T.5.3 {five}  0 3 {...three four {five}}
  do_snippet_test $T.5.4 {five}  1 3 {four {five} six...}
  do_snippet_test $T.5.5 {five}  2 3 {seven eight nine...}
  
  do_test $T.5.6 {
    execsql { UPDATE ft SET b = NULL }
  } {}
  
  do_snippet_test $T.5.7  {five} -1 3 {...three four {five}}
  do_snippet_test $T.5.8  {five}  0 3 {...three four {five}}
  do_snippet_test $T.5.9  {five}  1 3 {}
  do_snippet_test $T.5.10 {five}  2 3 {seven eight nine...}
  
  do_snippet_test $T.5.11 {one "seven eight nine"} -1 -3 {
    {one} two three...{seven} {eight} {nine}...
  }

  do_test $T.6.1 {
    execsql {
      DROP TABLE IF EXISTS ft;
      CREATE VIRTUAL TABLE ft USING fts3(x);
      INSERT INTO ft VALUES($numbers);
    }
  } {}
  do_snippet_test $T.6.2 {
    one fifty onehundred onehundredfifty twohundredfifty threehundred
  } -1 4 {
    {one}...{fifty}...{onehundred}...{onehundredfifty}...
  }
  do_snippet_test $T.6.3 {
    one fifty onehundred onehundredfifty twohundredfifty threehundred
  } -1 -4 {
    {one} two three four...fortyeight fortynine {fifty} fiftyone...ninetyeight ninetynine {onehundred} onehundredone...onehundredfortyeight onehundredfortynine {onehundredfifty} onehundredfiftyone...
  }

  do_test $T.7.1 {
    execsql {
      BEGIN;
        DROP TABLE IF EXISTS ft;
        CREATE VIRTUAL TABLE ft USING fts3(x);
    }
    set testresults [list]
    for {set i 1} {$i < 150} {incr i} {
      set commas [string repeat , $i]
      execsql {INSERT INTO ft VALUES('one' || $commas || 'two')}
      lappend testresults "{one}$commas{two}"
    }
    execsql COMMIT
  } {}
  do_snippet_test $T.7.2 {one two} -1 3 {*}$testresults
  
  ##########################################################################
  # Test the matchinfo function.
  #
  proc mit {blob} {
    set scan(littleEndian) i*
    set scan(bigEndian) I*
    binary scan $blob $scan($::tcl_platform(byteOrder)) r
    return $r
  }
  db func mit mit
  proc do_matchinfo_test {name expr args} {
    set res [list]
    foreach a $args { lappend res [normalize $a] }
    do_select_test $name {
      SELECT mit(matchinfo(ft)) FROM ft WHERE ft MATCH $expr
    } $res
  }
  do_test $T.4.1 {
    set ten {one two three four five six seven eight nine ten}
    execsql {
      DROP TABLE IF EXISTS ft;
      CREATE VIRTUAL TABLE ft USING fts3;
      INSERT INTO ft VALUES($ten);
      INSERT INTO ft VALUES($ten || ' ' || $ten);
    }
  } {}
  
  do_matchinfo_test $T.4.2 "one" {1 1  1 3 2} {1 1  2 3 2}
  do_matchinfo_test $T.4.3 "one NEAR/3 ten" {2 1  1 1 1 1 1 1}
  do_matchinfo_test $T.4.4 "five NEAR/4 ten" \
    {2 1  1 3 2  1 3 2} {2 1  2 3 2  2 3 2}
  do_matchinfo_test $T.4.5 "six NEAR/3 ten NEAR/3 two" \
    {3 1  1 1 1  1 1 1  1 1 1}
  do_matchinfo_test $T.4.6 "five NEAR/4 ten NEAR/3 two" \
    {3 1  2 2 1  1 1 1  1 1 1}

  do_test $T.8.1 {
    execsql {
      DROP TABLE IF EXISTS ft;
      CREATE VIRTUAL TABLE ft USING fts3(x, y);
    }
    foreach n {1 2 3} {
      set v1 [lrange $numbers 0 [expr $n*100]]
      set v2 [string trim [string repeat "$numbers " $n]]
      set docid [expr $n * 1000000]
      execsql { INSERT INTO ft(docid, x, y) VALUES($docid, $v1, $v2) }
    }
  } {}
  do_matchinfo_test $T.8.2 {two*}     \
    { 1 2    1   105 3   101 606 3}   \
    { 1 2    3   105 3   202 606 3}   \
    { 1 2    101 105 3   303 606 3}

  do_matchinfo_test $T.8.4 {"one* two*"}  \
    { 1 2    1 5 3   2 12 3}              \
    { 1 2    2 5 3   4 12 3}              \
    { 1 2    2 5 3   6 12 3}

  do_matchinfo_test $T.8.5 {twohundredfifty}  \
    { 1 2    0 1 1   1 6 3}                   \
    { 1 2    0 1 1   2 6 3}                   \
    { 1 2    1 1 1   3 6 3}

  do_matchinfo_test $T.8.6 {"threehundred one"} \
    { 1 2    0 0 0   1 3 2}                     \
    { 1 2    0 0 0   2 3 2}

  do_matchinfo_test $T.8.7 {one OR fivehundred} \
    { 2 2    1 3 3   1 6 3   0 0 0   0 0 0 }    \
    { 2 2    1 3 3   2 6 3   0 0 0   0 0 0 }    \
    { 2 2    1 3 3   3 6 3   0 0 0   0 0 0 }

  do_matchinfo_test $T.8.8 {two OR "threehundred one"} \
    { 2 2    1 3 3   1 6 3   0 0 0   0 3 2 }           \
    { 2 2    1 3 3   2 6 3   0 0 0   1 3 2 }           \
    { 2 2    1 3 3   3 6 3   0 0 0   2 3 2 }

  do_select_test $T.8.9 {
    SELECT mit(matchinfo(ft)), mit(matchinfo(ft))
    FROM ft WHERE ft MATCH 'two OR "threehundred one"' 
  } [normalize {
    {2 2 1 3 3 1 6 3 0 0 0 0 3 2}
    {2 2 1 3 3 1 6 3 0 0 0 0 3 2}
    {2 2 1 3 3 2 6 3 0 0 0 1 3 2}
    {2 2 1 3 3 2 6 3 0 0 0 1 3 2}
    {2 2 1 3 3 3 6 3 0 0 0 2 3 2}          
    {2 2 1 3 3 3 6 3 0 0 0 2 3 2}
  }]
}

set sqlite_fts3_enable_parentheses 0
finish_test