/ Check-in [c27d46b3]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Further OOM testing for fts3 code. Add Tcl code implementing an integrity-check for fts3.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts3-refactor
Files: files | file ages | folders
SHA1:c27d46b33e8596b45c562c2742b05030e8899092
User & Date: dan 2009-11-14 18:41:01
Original User & Date: dan 2009-11-14 11:41:01
Context
2009-11-15
06:50
Fixes to fts3 integrity check code. check-in: d3cae986 user: dan tags: fts3-refactor
2009-11-14
18:41
Further OOM testing for fts3 code. Add Tcl code implementing an integrity-check for fts3. check-in: c27d46b3 user: dan tags: fts3-refactor
2009-11-13
17:36
Start reworking fts3 code to match the rest of SQLite (code conventions, malloc-failure handling etc.). check-in: 30a92f11 user: dan tags: fts3-refactor
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts3/fts3.c.

274
275
276
277
278
279
280


281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
....
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287

1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
....
1686
1687
1688
1689
1690
1691
1692
1693
1694

1695
1696
1697
1698
1699
1700
1701
....
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762





1763
1764
1765
1766
1767
1768
1769
1770










1771
1772
1773
1774
1775
1776
1777

#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)

#if defined(SQLITE_ENABLE_FTS3) && !defined(SQLITE_CORE)
# define SQLITE_CORE 1
#endif



#include <assert.h>
#include <stdlib.h>
#include <stddef.h>
#include <stdio.h>
#include <string.h>

#include "fts3.h"
#include "fts3_hash.h"
#include "fts3_tokenizer.h"
#ifndef SQLITE_CORE 
# include "sqlite3ext.h"
  SQLITE_EXTENSION_INIT1
#endif

#include "fts3Int.h"



/* TODO(shess) MAN, this thing needs some refactoring.  At minimum, it
** would be nice to order the file better, perhaps something along the
** lines of:
**
**  - utility functions
................................................................................
    return res;
  }
}

/*
** Values that may be used as the first parameter to fts3DoclistMerge().
*/
#define MERGE_AND        1        /* D + D -> D */
#define MERGE_NOT        2        /* D + D -> D */
#define MERGE_OR         3        /* D + D -> D */

#define MERGE_POS_OR     4        /* P + P -> P */
#define MERGE_PHRASE     5        /* P + P -> D */
#define MERGE_POS_PHRASE 6        /* P + P -> P */
#define MERGE_NEAR       7        /* P + P -> D */
#define MERGE_POS_NEAR   8        /* P + P -> P */

static int fts3DoclistMerge(
  int mergetype,                  /* One of the MERGE_XXX constants */
  int nParam1,
  int nParam2,
  char *aBuffer,                  /* Pre-allocated output buffer */
  int *pnBuffer,                  /* OUT: Bytes written to aBuffer */
................................................................................
*/
static int evalFts3Expr(
  Fts3Table *p,                   /* Virtual table handle */
  Fts3Expr *pExpr,                /* Parsed fts3 expression */
  char **paOut,                   /* OUT: Pointer to malloc'd result buffer */
  int *pnOut                      /* OUT: Size of buffer at *paOut */
){
  int rc = SQLITE_OK;


  *paOut = 0;
  *pnOut = 0;

  if( pExpr ){
    if( pExpr->eType==FTSQUERY_PHRASE ){
      int isReqPos = (pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR);
      rc = fts3PhraseSelect(p, pExpr->pPhrase, isReqPos, paOut, pnOut);
................................................................................
            }
            pRight = pExpr->pRight;
            assert( pRight->eType==FTSQUERY_PHRASE );
            assert( pLeft->eType==FTSQUERY_PHRASE );

            nParam1 = pExpr->nNear+1;
            nParam2 = nParam1+pLeft->pPhrase->nToken+pRight->pPhrase->nToken-2;
            aBuffer = sqlite3_malloc(nLeft + nRight);
            rc = fts3DoclistMerge(mergetype, nParam1, nParam2, aBuffer,
                pnOut, aLeft, nLeft, aRight, nRight
            );
            if( rc!=SQLITE_OK ){
              sqlite3_free(aBuffer);
            }else{
              *paOut = aBuffer;
            }
            sqlite3_free(aLeft);
            break;
          }

          case FTSQUERY_NOT: {
            fts3DoclistMerge(MERGE_NOT, 0, 0, aLeft, pnOut,
                aLeft, nLeft, aRight, nRight
            );
            *paOut = aLeft;
            break;
          }

          case FTSQUERY_AND: {
            fts3DoclistMerge(MERGE_AND, 0, 0, aLeft, pnOut,
                aLeft, nLeft, aRight, nRight
            );
            *paOut = aLeft;
            break;
          }

          case FTSQUERY_OR: {





            char *aBuffer = sqlite3_malloc(nRight+nLeft);
            rc = fts3DoclistMerge(MERGE_OR, 0, 0, aBuffer, pnOut,
                aLeft, nLeft, aRight, nRight
            );
            *paOut = aBuffer;
            sqlite3_free(aLeft);
            break;
          }










        }
      }
      sqlite3_free(aRight);
    }
  }

  return rc;







>
>







<
<





<
<







 







<

|
>
|
|
|
|
|







 







|

>







 







|












<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<

>
>
>
>
>
|







>
>
>
>
>
>
>
>
>
>







274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289


290
291
292
293
294


295
296
297
298
299
300
301
....
1276
1277
1278
1279
1280
1281
1282

1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
....
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
....
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
















1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775

#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)

#if defined(SQLITE_ENABLE_FTS3) && !defined(SQLITE_CORE)
# define SQLITE_CORE 1
#endif

#include "fts3Int.h"

#include <assert.h>
#include <stdlib.h>
#include <stddef.h>
#include <stdio.h>
#include <string.h>

#include "fts3.h"


#ifndef SQLITE_CORE 
# include "sqlite3ext.h"
  SQLITE_EXTENSION_INIT1
#endif





/* TODO(shess) MAN, this thing needs some refactoring.  At minimum, it
** would be nice to order the file better, perhaps something along the
** lines of:
**
**  - utility functions
................................................................................
    return res;
  }
}

/*
** Values that may be used as the first parameter to fts3DoclistMerge().
*/

#define MERGE_NOT        2        /* D + D -> D */
#define MERGE_AND        3        /* D + D -> D */
#define MERGE_OR         4        /* D + D -> D */
#define MERGE_POS_OR     5        /* P + P -> P */
#define MERGE_PHRASE     6        /* P + P -> D */
#define MERGE_POS_PHRASE 7        /* P + P -> P */
#define MERGE_NEAR       8        /* P + P -> D */
#define MERGE_POS_NEAR   9        /* P + P -> P */

static int fts3DoclistMerge(
  int mergetype,                  /* One of the MERGE_XXX constants */
  int nParam1,
  int nParam2,
  char *aBuffer,                  /* Pre-allocated output buffer */
  int *pnBuffer,                  /* OUT: Bytes written to aBuffer */
................................................................................
*/
static int evalFts3Expr(
  Fts3Table *p,                   /* Virtual table handle */
  Fts3Expr *pExpr,                /* Parsed fts3 expression */
  char **paOut,                   /* OUT: Pointer to malloc'd result buffer */
  int *pnOut                      /* OUT: Size of buffer at *paOut */
){
  int rc = SQLITE_OK;             /* Return code */

  /* Zero the output parameters. */
  *paOut = 0;
  *pnOut = 0;

  if( pExpr ){
    if( pExpr->eType==FTSQUERY_PHRASE ){
      int isReqPos = (pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR);
      rc = fts3PhraseSelect(p, pExpr->pPhrase, isReqPos, paOut, pnOut);
................................................................................
            }
            pRight = pExpr->pRight;
            assert( pRight->eType==FTSQUERY_PHRASE );
            assert( pLeft->eType==FTSQUERY_PHRASE );

            nParam1 = pExpr->nNear+1;
            nParam2 = nParam1+pLeft->pPhrase->nToken+pRight->pPhrase->nToken-2;
            aBuffer = sqlite3_malloc(nLeft+nRight+1);
            rc = fts3DoclistMerge(mergetype, nParam1, nParam2, aBuffer,
                pnOut, aLeft, nLeft, aRight, nRight
            );
            if( rc!=SQLITE_OK ){
              sqlite3_free(aBuffer);
            }else{
              *paOut = aBuffer;
            }
            sqlite3_free(aLeft);
            break;
          }

















          case FTSQUERY_OR: {
            /* Allocate a buffer for the output. The maximum size is the
            ** sum of the sizes of the two input buffers. The +1 term is
            ** so that a buffer of zero bytes is never allocated - this can
            ** cause fts3DoclistMerge() to incorrectly return SQLITE_NOMEM.
            */
            char *aBuffer = sqlite3_malloc(nRight+nLeft+1);
            rc = fts3DoclistMerge(MERGE_OR, 0, 0, aBuffer, pnOut,
                aLeft, nLeft, aRight, nRight
            );
            *paOut = aBuffer;
            sqlite3_free(aLeft);
            break;
          }

          case FTSQUERY_AND:
          case FTSQUERY_NOT: {
            assert( FTSQUERY_NOT==MERGE_NOT && FTSQUERY_AND==MERGE_AND );
            fts3DoclistMerge(pExpr->eType, 0, 0, aLeft, pnOut,
                aLeft, nLeft, aRight, nRight
            );
            *paOut = aLeft;
            break;
          }
        }
      }
      sqlite3_free(aRight);
    }
  }

  return rc;

Changes to ext/fts3/fts3Int.h.

10
11
12
13
14
15
16




17
18
19
20
21
22
23
**
******************************************************************************
**
*/

#ifndef _FTSINT_H
#define _FTSINT_H





#include <sqlite3.h>
#include "fts3_tokenizer.h"
#include "fts3_hash.h"

/*
** This constant controls how often segments are merged. Once there are







>
>
>
>







10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
**
******************************************************************************
**
*/

#ifndef _FTSINT_H
#define _FTSINT_H

#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) 
# define NDEBUG 1
#endif

#include <sqlite3.h>
#include "fts3_tokenizer.h"
#include "fts3_hash.h"

/*
** This constant controls how often segments are merged. Once there are

Added test/fts3_common.tcl.















































































































































































































































































































































































































































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
# 2009 November 04
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#
# This file contains common code used the fts3 tests. At one point
# equivalent functionality was implemented in C code. But it is easier
# to use Tcl.
#

#-------------------------------------------------------------------------
# USAGE: fts3_integrity_check TBL
#
# This proc is used to verify that the full-text index is consistent with
# the contents of the fts3 table. In other words, it checks that the
# data in the %_contents table matches that in the %_segdir and %_segments 
# tables.
#
# This is not an efficient procedure. It uses a lot of memory and a lot
# of CPU. But it is better than not checking at all.
#
# The procedure is:
#
#   1) Read the entire full-text index from the %_segdir and %_segments
#      tables into memory. For each entry in the index, the following is
#      done:
#
#          set C($iDocid,$iCol,$iPosition) $zTerm
#
#   2) Iterate through each column of each row of the %_content table. 
#      Tokenize all documents, and check that for each token there is
#      a corresponding entry in the $C array. After checking a token,
#      [unset] the $C array entry.
#
#   3) Check that array $C is now empty.
#      
#
proc fts3_integrity_check {tbl} {

  fts3_read2 $tbl 1 A

  foreach zTerm [array names A] {
    foreach doclist $A($zTerm) {
      set docid 0
      while {[string length $doclist]>0} {
        set iCol 0
        set iPos 0
        set lPos [list]
        set lCol [list]

        # First varint of a doclist-entry is the docid. Delta-compressed
        # with respect to the docid of the previous entry.
        #
        incr docid [gobble_varint doclist]
        if {[info exists D($zTerm,$docid)]} {
          while {[set iDelta [gobble_varint doclist]] != 0} {}
          continue
        }
        set D($zTerm,$docid) 1

        # Gobble varints until the 0x00 that terminates the doclist-entry
        # is found.
        while {[set iDelta [gobble_varint doclist]] > 0} {
          if {$iDelta == 1} {
            set iCol [gobble_varint doclist]
            set iPos 0
          } else {
            incr iPos $iDelta
            incr iPos -2
            set C($docid,$iCol,$iPos) $zTerm
          }
        }
      }
    }
  }


  db eval "SELECT * FROM ${tbl}_content" E {
    set iCol 0
    set iDoc $E(docid)
    foreach col [lrange $E(*) 1 end] {
      set c $E($col)
      set sql {SELECT fts3_tokenizer_test('simple', $c)}

      foreach {pos term dummy} [db one $sql] {
        if {$C($iDoc,$iCol,$pos) != "$term"} {
          set    es "Error at docid=$iDoc col=$iCol pos=$pos. "
          append es "Index has \"$C($iDoc,$iCol,$pos)\", document has \"$term\""
          lappend errors $es
        }
        unset C($iDoc,$iCol,$pos)
      }
      incr iCol
    }
  }

  foreach c [array names C] {
    lappend errors "Bad index entry: $c -> $C($c)"
  }

  if {[info exists errors]} { return [join $errors "\n"] }
  return "ok"
}

# USAGE: fts3_terms TBL WHERE
#
# Argument TBL must be the name of an FTS3 table. Argument WHERE is an
# SQL expression that will be used as the WHERE clause when scanning
# the %_segdir table. As in the following query:
#
#   "SELECT * FROM ${TBL}_segdir WHERE ${WHERE}"
#
# This function returns a list of all terms present in the segments
# selected by the statement above.
#
proc fts3_terms {tbl where} {
  fts3_read $tbl $where a
  return [lsort [array names a]]
}


# USAGE: fts3_doclist TBL TERM WHERE
#
# Argument TBL must be the name of an FTS3 table. TERM is a term that may
# or may not be present in the table. Argument WHERE is used to select a 
# subset of the b-tree segments in the associated full-text index as 
# described above for [fts3_terms].
#
# This function returns the results of merging the doclists associated
# with TERM in the selected segments. Each doclist is an element of the
# returned list. Each doclist is formatted as follows:
#
#   [$docid ?$col[$off1 $off2...]?...]
#
# The formatting is odd for a Tcl command in order to be compatible with
# the original C-language implementation. If argument WHERE is "1", then 
# any empty doclists are omitted from the returned list.
#
proc fts3_doclist {tbl term where} {
  fts3_read $tbl $where a


  foreach doclist $a($term) {
    set docid 0

    while {[string length $doclist]>0} {
      set iCol 0
      set iPos 0
      set lPos [list]
      set lCol [list]
      incr docid [gobble_varint doclist]
  
      while {[set iDelta [gobble_varint doclist]] > 0} {
        if {$iDelta == 1} {
          lappend lCol [list $iCol $lPos]
          set iPos 0
          set lPos [list]
          set iCol [gobble_varint doclist]
        } else {
          incr iPos $iDelta
          incr iPos -2
          lappend lPos $iPos
        }
      }
  
      if {[llength $lPos]>0} {
        lappend lCol [list $iCol $lPos]
      }
  
      if {$where != "1" || [llength $lCol]>0} {
        set ret($docid) $lCol
      } else {
        unset -nocomplain ret($docid)
      }
    }
  }

  set lDoc [list]
  foreach docid [lsort -integer [array names ret]] {
    set lCol [list]
    set cols ""
    foreach col $ret($docid) {
      foreach {iCol lPos} $col {}
      append cols " $iCol\[[join $lPos { }]\]"
    }
    lappend lDoc "\[${docid}${cols}\]"
  }

  join $lDoc " "
}

###########################################################################

proc gobble_varint {varname} {
  upvar $varname blob
  set n [read_varint $blob ret]
  set blob [string range $blob $n end]
  return $ret
}
proc gobble_string {varname nLength} {
  upvar $varname blob
  set ret [string range $blob 0 [expr $nLength-1]]
  set blob [string range $blob $nLength end]
  return $ret
}

# The argument is a blob of data representing an FTS3 segment leaf. 
# Return a list consisting of alternating terms (strings) and doclists
# (blobs of data).
#
proc fts3_readleaf {blob} {
  set zPrev ""
  set terms [list]

  while {[string length $blob] > 0} {
    set nPrefix [gobble_varint blob]
    set nSuffix [gobble_varint blob]

    set zTerm [string range $zPrev 0 [expr $nPrefix-1]]
    append zTerm [gobble_string blob $nSuffix]
    set doclist [gobble_string blob [gobble_varint blob]]

    lappend terms $zTerm $doclist
    set zPrev $zTerm
  }

  return $terms
}

proc fts3_read2 {tbl where varname} {
  upvar $varname a
  array unset a
  db eval " SELECT start_block, leaves_end_block, root 
            FROM ${tbl}_segdir WHERE $where
            ORDER BY level ASC, idx DESC
  " {
    if {$start_block == 0} {
      foreach {t d} [fts3_readleaf $root] { lappend a($t) $d }
    } else {
      db eval " SELECT block 
                FROM ${tbl}_segments 
                WHERE blockid>=$start_block AND blockid<$leaves_end_block
                ORDER BY blockid
      " {
        foreach {t d} [fts3_readleaf $block] { lappend a($t) $d }

      }
    }
  }
}

proc fts3_read {tbl where varname} {
  upvar $varname a
  array unset a
  db eval " SELECT start_block, leaves_end_block, root 
            FROM ${tbl}_segdir WHERE $where
            ORDER BY level DESC, idx ASC
  " {
    if {$start_block == 0} {
      foreach {t d} [fts3_readleaf $root] { lappend a($t) $d }
    } else {
      db eval " SELECT block 
                FROM ${tbl}_segments 
                WHERE blockid>=$start_block AND blockid<$leaves_end_block
                ORDER BY blockid
      " {
        foreach {t d} [fts3_readleaf $block] { lappend a($t) $d }

      }
    }
  }
}

Changes to test/fts3malloc.test.

13
14
15
16
17
18
19

20



21
22
23
24
25
26
27
28
29
30
31
32
33
34
35

36



37
38
39
40
41
42



43
44
45

46
47
48


49




50
51
52
53
54
55
56
57
58
59
60

61
62
63
64
65
66
67
..
72
73
74
75
76
77
78































































79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
...
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
...
259
260
261
262
263
264
265

266
267
268
269
270
271
272
273



274
275
276
277
278
279
280
281















282
283
284
# within the FTS3 module code are handled correctly. 
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
ifcapable !fts3 { finish_test ; return }
source $testdir/malloc_common.tcl





do_malloc_test fts3_malloc-1.1 -sqlbody {
  CREATE VIRTUAL TABLE ft USING fts3(a, b, c);
}

do_malloc_test fts3_malloc-1.2 -sqlprep {
  CREATE VIRTUAL TABLE ft USING fts3(a, b, c);
} -sqlbody {
  DROP TABLE ft;
}

do_malloc_test fts3_malloc-1.3 -sqlprep {
  CREATE VIRTUAL TABLE ft USING fts3(content);
} -sqlbody {
  INSERT INTO ft VALUES('one two three four');
}





do_malloc_test fts3_malloc-1.4 -tclprep {
  db eval {CREATE VIRTUAL TABLE ft USING fts3(a, b)}
  for {set i 0} {$i<16} {incr i} {
    db eval { INSERT INTO ft VALUES('one two', 'three four') }
  }
} -sqlbody {



  INSERT INTO ft VALUES('one two', 'three four');
}


proc do_write_test {sql} {
  uplevel [list db eval $sql]
}







proc do_read_test {name sql result} {

  if {![info exists ::DO_MALLOC_TEST]} {
    set ::DO_MALLOC_TEST 1
  }

  set answers [list [list 0 $result]]
  if {$::DO_MALLOC_TEST } {
    set answers [list {1 {out of memory}} [list 0 $result]]
    set modes [list 100000 transient 1 persistent]
  } else {

    set modes [list 0 nofail]
  }
  set str [join $answers " OR "]

  foreach {nRepeat zName} $modes {
    for {set iFail 1} 1 {incr iFail} {
      if {$::DO_MALLOC_TEST} {sqlite3_memdebug_fail $iFail -repeat $nRepeat}
................................................................................
      }
      do_test $name.$zName.$iFail [list set {} $res] $str
      set nFail [sqlite3_memdebug_fail -1 -benigncnt nBenign]
      if {$nFail==0} break
    }
  }
}
































































proc normal_list {l} {
  set ret [list]
  foreach elem $l {lappend ret $elem}
  set ret
}

db close
file delete -force test.db test.db-journal
sqlite3 db test.db
sqlite3_db_config_lookaside db 0 0 0
set sqlite_fts3_enable_parentheses 1


do_test fts3_malloc-2.0 {
  execsql { CREATE VIRTUAL TABLE ft USING fts3(a, b) }
  for {set ii 1} {$ii < 32} {incr ii} {
    set a [list]
    set b [list]
    if {$ii & 0x01} {lappend a one   ; lappend b neung}
................................................................................
  20 {SELECT a FROM ft WHERE ft MATCH 'sahm NOT song'} {
    {three}                   {one three}           {three four}   
    {one three four}          {three five}          {one three five}
    {three four five}         {one three four five}
  }

  21 {SELECT a FROM ft WHERE b MATCH 'neung NEAR song NEAR sahm'} {
    {one two three}          {one two three four}  
    {one two three five}     {one two three four five}
  }

} {
  set result [normal_list $result]
  do_read_test fts3_malloc-2.$tn $sql $result
}

do_test fts3_malloc-3.0 {
  execsql BEGIN
  for {set ii 32} {$ii < 1024} {incr ii} {
    set a [list]
    set b [list]
................................................................................
    if {$ii & 0x0200} {lappend a ten   ; lappend b sip   }
    execsql { INSERT INTO ft VALUES($a, $b) }
  }
  execsql COMMIT
} {}
foreach {tn sql result} {
  1 "SELECT count(*) FROM ft" {1023}

  2 "SELECT a FROM ft WHERE a MATCH 'one two three four five six seven eight'" {
     {one two three four five six seven eight}
     {one two three four five six seven eight nine}
     {one two three four five six seven eight ten}
     {one two three four five six seven eight nine ten}
  }

  3 {SELECT count(*), sum(docid) FROM ft WHERE a MATCH 'o*'} {512 262144}



  4 {SELECT count(*), sum(docid) FROM ft WHERE a MATCH '"two three four"'} {
    128 66368
  }
} {
#set ::DO_MALLOC_TEST 0
  set result [normal_list $result]
  do_read_test fts3_malloc-3.$tn $sql $result
}
















finish_test








>

>
>
>



<





|
<
<
<
<
|
>

>
>
>
|
|
|
<
<
<
>
>
>
|
<
|
>
|
|
<
>
>
|
>
>
>
>
|

|
<
|
<
<




>







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>











<
<







 







|
|




|







 







>







|
>
>
>




<

|

>
>
>
>
>
>
>
>
>
>
>
>
>
>
>



13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

28
29
30
31
32
33




34
35
36
37
38
39
40
41
42



43
44
45
46

47
48
49
50

51
52
53
54
55
56
57
58
59
60

61


62
63
64
65
66
67
68
69
70
71
72
73
..
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158


159
160
161
162
163
164
165
...
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
...
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348

349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
# within the FTS3 module code are handled correctly. 
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
ifcapable !fts3 { finish_test ; return }
source $testdir/malloc_common.tcl
source $testdir/fts3_common.tcl

set sqlite_fts3_enable_parentheses 1

if 0 {
do_malloc_test fts3_malloc-1.1 -sqlbody {
  CREATE VIRTUAL TABLE ft USING fts3(a, b, c);
}

do_malloc_test fts3_malloc-1.2 -sqlprep {
  CREATE VIRTUAL TABLE ft USING fts3(a, b, c);
} -sqlbody {
  DROP TABLE ft;
}
}





set DO_MALLOC_TEST 0

#-------------------------------------------------------------------------
# This proc is used to test a single SELECT statement. Parameter $name is
# passed a name for the test case (i.e. "fts3_malloc-1.4.1") and parameter
# $sql is passed the text of the SELECT statement. Parameter $result is
# set to the expected output if the SELECT statement is successfully
# executed using [db eval].



#
# Example:
#
#   do_select_test testcase-1.1 "SELECT 1+1, 1+2" {1 2}

#
# If global variable DO_MALLOC_TEST is set to a non-zero value, or if
# it is not defined at all, then OOM testing is performed on the SELECT
# statement. Each OOM test case is said to pass if either (a) executing

# the SELECT statement succeeds and the results match those specified
# by parameter $result, or (b) TCL throws an "out of memory" error.
#
# If DO_MALLOC_TEST is defined and set to zero, then the SELECT statement
# is executed just once. In this case the test case passes if the results
# match the expected results passed via parameter $result.
#
proc do_select_test {name sql result} {

  if {![info exists ::DO_MALLOC_TEST]} { set ::DO_MALLOC_TEST 1 }




  if {$::DO_MALLOC_TEST } {
    set answers [list {1 {out of memory}} [list 0 $result]]
    set modes [list 100000 transient 1 persistent]
  } else {
    set answers [list [list 0 $result]]
    set modes [list 0 nofail]
  }
  set str [join $answers " OR "]

  foreach {nRepeat zName} $modes {
    for {set iFail 1} 1 {incr iFail} {
      if {$::DO_MALLOC_TEST} {sqlite3_memdebug_fail $iFail -repeat $nRepeat}
................................................................................
      }
      do_test $name.$zName.$iFail [list set {} $res] $str
      set nFail [sqlite3_memdebug_fail -1 -benigncnt nBenign]
      if {$nFail==0} break
    }
  }
}

#-------------------------------------------------------------------------
# Test a single write to the database. In this case a  "write" is a 
# DELETE, UPDATE or INSERT statement.
#
# If OOM testing is performed, there are several acceptable outcomes:
#
#   1) The write succeeds. No error is returned.
#
#   2) An "out of memory" exception is thrown and:
#
#     a) The statement has no effect, OR
#     b) The current transaction is rolled back, OR
#     c) The statement succeeds. This can only happen if the connection
#        is in auto-commit mode (after the statement is executed, so this
#        includes COMMIT statements).
#
# If the write operation eventually succeeds, zero is returned. If a
# transaction is rolled back, non-zero is returned.
#
# Parameter $name is the name to use for the test case (or test cases).
# The second parameter, $tbl, should be the name of the database table
# being modified. Parameter $sql contains the SQL statement to test.
#
proc do_write_test {name tbl sql} {
  # Figure out an statement to get a checksum for table $tbl.
  db eval "SELECT * FROM $tbl" V break
  set cksumsql "SELECT md5sum([join [concat rowid $V(*)] ,]) FROM $tbl"

  # Calculate the initial table checksum.
  set cksum1 [db one $cksumsql]

  if {![info exists ::DO_MALLOC_TEST]} { set ::DO_MALLOC_TEST 1 }

  if {$::DO_MALLOC_TEST } {
    set answers [list {1 {out of memory}} {0 {}}]
    set modes [list 100000 transient 1 persistent]
  } else {
    set answers [list {0 {}}]
    set modes [list 0 nofail]
  }
  set str [join $answers " OR "]

  foreach {nRepeat zName} $modes {
    for {set iFail 1} 1 {incr iFail} {
      if {$::DO_MALLOC_TEST} {sqlite3_memdebug_fail $iFail -repeat $nRepeat}

      set res [catchsql $sql]
      set nFail [sqlite3_memdebug_fail -1 -benigncnt nBenign]
      if {$nFail==0} {
        do_test $name.$zName.$iFail [list set {} $res] {0 {}}
        return
      } else {
        if {[lsearch $answers $res]>=0} {
          set res $str
        }
        do_test $name.$zName.$iFail [list set {} $res] $str
        set cksum2 [db one $cksumsql]
        if {$cksum1 != $cksum2} return
      }
    }
  }
}

proc normal_list {l} {
  set ret [list]
  foreach elem $l {lappend ret $elem}
  set ret
}

db close
file delete -force test.db test.db-journal
sqlite3 db test.db
sqlite3_db_config_lookaside db 0 0 0



do_test fts3_malloc-2.0 {
  execsql { CREATE VIRTUAL TABLE ft USING fts3(a, b) }
  for {set ii 1} {$ii < 32} {incr ii} {
    set a [list]
    set b [list]
    if {$ii & 0x01} {lappend a one   ; lappend b neung}
................................................................................
  20 {SELECT a FROM ft WHERE ft MATCH 'sahm NOT song'} {
    {three}                   {one three}           {three four}   
    {one three four}          {three five}          {one three five}
    {three four five}         {one three four five}
  }

  21 {SELECT a FROM ft WHERE b MATCH 'neung NEAR song NEAR sahm'} {
    {one two three}           {one two three four}  
    {one two three five}      {one two three four five}
  }

} {
  set result [normal_list $result]
  do_select_test fts3_malloc-2.$tn $sql $result
}

do_test fts3_malloc-3.0 {
  execsql BEGIN
  for {set ii 32} {$ii < 1024} {incr ii} {
    set a [list]
    set b [list]
................................................................................
    if {$ii & 0x0200} {lappend a ten   ; lappend b sip   }
    execsql { INSERT INTO ft VALUES($a, $b) }
  }
  execsql COMMIT
} {}
foreach {tn sql result} {
  1 "SELECT count(*) FROM ft" {1023}

  2 "SELECT a FROM ft WHERE a MATCH 'one two three four five six seven eight'" {
     {one two three four five six seven eight}
     {one two three four five six seven eight nine}
     {one two three four five six seven eight ten}
     {one two three four five six seven eight nine ten}
  }

  3 {SELECT count(*), sum(docid) FROM ft WHERE a MATCH 'o*'} {
    512 262144
  }

  4 {SELECT count(*), sum(docid) FROM ft WHERE a MATCH '"two three four"'} {
    128 66368
  }
} {

  set result [normal_list $result]
  do_select_test fts3_malloc-3.$tn $sql $result
}

do_test fts3_malloc-4.0 {
  execsql { DELETE FROM ft WHERE docid>=32 }
} {}
foreach {tn sql} {
  1 "DELETE FROM ft WHERE ft MATCH 'one'"
  2 "DELETE FROM ft WHERE ft MATCH 'three'"
  3 "DELETE FROM ft WHERE ft MATCH 'five'"
} {
  do_write_test fts3_malloc-4.1.$tn ft_content $sql
}
do_test fts3_malloc-4.2 {
  execsql { SELECT a FROM ft }
} {two four {two four}}


finish_test