/ Check-in [c27d46b3]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Further OOM testing for fts3 code. Add Tcl code implementing an integrity-check for fts3.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts3-refactor
Files: files | file ages | folders
SHA1:c27d46b33e8596b45c562c2742b05030e8899092
User & Date: dan 2009-11-14 18:41:01
Original User & Date: dan 2009-11-14 11:41:01
Context
2009-11-15
06:50
Fixes to fts3 integrity check code. check-in: d3cae986 user: dan tags: fts3-refactor
2009-11-14
18:41
Further OOM testing for fts3 code. Add Tcl code implementing an integrity-check for fts3. check-in: c27d46b3 user: dan tags: fts3-refactor
2009-11-13
17:36
Start reworking fts3 code to match the rest of SQLite (code conventions, malloc-failure handling etc.). check-in: 30a92f11 user: dan tags: fts3-refactor
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts3/fts3.c.

   274    274   
   275    275   #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
   276    276   
   277    277   #if defined(SQLITE_ENABLE_FTS3) && !defined(SQLITE_CORE)
   278    278   # define SQLITE_CORE 1
   279    279   #endif
   280    280   
          281  +#include "fts3Int.h"
          282  +
   281    283   #include <assert.h>
   282    284   #include <stdlib.h>
   283    285   #include <stddef.h>
   284    286   #include <stdio.h>
   285    287   #include <string.h>
   286    288   
   287    289   #include "fts3.h"
   288         -#include "fts3_hash.h"
   289         -#include "fts3_tokenizer.h"
   290    290   #ifndef SQLITE_CORE 
   291    291   # include "sqlite3ext.h"
   292    292     SQLITE_EXTENSION_INIT1
   293    293   #endif
   294    294   
   295         -#include "fts3Int.h"
   296         -
   297    295   
   298    296   
   299    297   /* TODO(shess) MAN, this thing needs some refactoring.  At minimum, it
   300    298   ** would be nice to order the file better, perhaps something along the
   301    299   ** lines of:
   302    300   **
   303    301   **  - utility functions
................................................................................
  1278   1276       return res;
  1279   1277     }
  1280   1278   }
  1281   1279   
  1282   1280   /*
  1283   1281   ** Values that may be used as the first parameter to fts3DoclistMerge().
  1284   1282   */
  1285         -#define MERGE_AND        1        /* D + D -> D */
  1286   1283   #define MERGE_NOT        2        /* D + D -> D */
  1287         -#define MERGE_OR         3        /* D + D -> D */
  1288         -#define MERGE_POS_OR     4        /* P + P -> P */
  1289         -#define MERGE_PHRASE     5        /* P + P -> D */
  1290         -#define MERGE_POS_PHRASE 6        /* P + P -> P */
  1291         -#define MERGE_NEAR       7        /* P + P -> D */
  1292         -#define MERGE_POS_NEAR   8        /* P + P -> P */
         1284  +#define MERGE_AND        3        /* D + D -> D */
         1285  +#define MERGE_OR         4        /* D + D -> D */
         1286  +#define MERGE_POS_OR     5        /* P + P -> P */
         1287  +#define MERGE_PHRASE     6        /* P + P -> D */
         1288  +#define MERGE_POS_PHRASE 7        /* P + P -> P */
         1289  +#define MERGE_NEAR       8        /* P + P -> D */
         1290  +#define MERGE_POS_NEAR   9        /* P + P -> P */
  1293   1291   
  1294   1292   static int fts3DoclistMerge(
  1295   1293     int mergetype,                  /* One of the MERGE_XXX constants */
  1296   1294     int nParam1,
  1297   1295     int nParam2,
  1298   1296     char *aBuffer,                  /* Pre-allocated output buffer */
  1299   1297     int *pnBuffer,                  /* OUT: Bytes written to aBuffer */
................................................................................
  1686   1684   */
  1687   1685   static int evalFts3Expr(
  1688   1686     Fts3Table *p,                   /* Virtual table handle */
  1689   1687     Fts3Expr *pExpr,                /* Parsed fts3 expression */
  1690   1688     char **paOut,                   /* OUT: Pointer to malloc'd result buffer */
  1691   1689     int *pnOut                      /* OUT: Size of buffer at *paOut */
  1692   1690   ){
  1693         -  int rc = SQLITE_OK;
         1691  +  int rc = SQLITE_OK;             /* Return code */
  1694   1692   
         1693  +  /* Zero the output parameters. */
  1695   1694     *paOut = 0;
  1696   1695     *pnOut = 0;
  1697   1696   
  1698   1697     if( pExpr ){
  1699   1698       if( pExpr->eType==FTSQUERY_PHRASE ){
  1700   1699         int isReqPos = (pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR);
  1701   1700         rc = fts3PhraseSelect(p, pExpr->pPhrase, isReqPos, paOut, pnOut);
................................................................................
  1726   1725               }
  1727   1726               pRight = pExpr->pRight;
  1728   1727               assert( pRight->eType==FTSQUERY_PHRASE );
  1729   1728               assert( pLeft->eType==FTSQUERY_PHRASE );
  1730   1729   
  1731   1730               nParam1 = pExpr->nNear+1;
  1732   1731               nParam2 = nParam1+pLeft->pPhrase->nToken+pRight->pPhrase->nToken-2;
  1733         -            aBuffer = sqlite3_malloc(nLeft + nRight);
         1732  +            aBuffer = sqlite3_malloc(nLeft+nRight+1);
  1734   1733               rc = fts3DoclistMerge(mergetype, nParam1, nParam2, aBuffer,
  1735   1734                   pnOut, aLeft, nLeft, aRight, nRight
  1736   1735               );
  1737   1736               if( rc!=SQLITE_OK ){
  1738   1737                 sqlite3_free(aBuffer);
  1739   1738               }else{
  1740   1739                 *paOut = aBuffer;
  1741   1740               }
  1742   1741               sqlite3_free(aLeft);
  1743   1742               break;
  1744   1743             }
  1745   1744   
  1746         -          case FTSQUERY_NOT: {
  1747         -            fts3DoclistMerge(MERGE_NOT, 0, 0, aLeft, pnOut,
  1748         -                aLeft, nLeft, aRight, nRight
  1749         -            );
  1750         -            *paOut = aLeft;
  1751         -            break;
  1752         -          }
  1753         -
  1754         -          case FTSQUERY_AND: {
  1755         -            fts3DoclistMerge(MERGE_AND, 0, 0, aLeft, pnOut,
  1756         -                aLeft, nLeft, aRight, nRight
  1757         -            );
  1758         -            *paOut = aLeft;
  1759         -            break;
  1760         -          }
  1761         -
  1762   1745             case FTSQUERY_OR: {
  1763         -            char *aBuffer = sqlite3_malloc(nRight+nLeft);
         1746  +            /* Allocate a buffer for the output. The maximum size is the
         1747  +            ** sum of the sizes of the two input buffers. The +1 term is
         1748  +            ** so that a buffer of zero bytes is never allocated - this can
         1749  +            ** cause fts3DoclistMerge() to incorrectly return SQLITE_NOMEM.
         1750  +            */
         1751  +            char *aBuffer = sqlite3_malloc(nRight+nLeft+1);
  1764   1752               rc = fts3DoclistMerge(MERGE_OR, 0, 0, aBuffer, pnOut,
  1765   1753                   aLeft, nLeft, aRight, nRight
  1766   1754               );
  1767   1755               *paOut = aBuffer;
  1768   1756               sqlite3_free(aLeft);
  1769   1757               break;
  1770   1758             }
         1759  +
         1760  +          case FTSQUERY_AND:
         1761  +          case FTSQUERY_NOT: {
         1762  +            assert( FTSQUERY_NOT==MERGE_NOT && FTSQUERY_AND==MERGE_AND );
         1763  +            fts3DoclistMerge(pExpr->eType, 0, 0, aLeft, pnOut,
         1764  +                aLeft, nLeft, aRight, nRight
         1765  +            );
         1766  +            *paOut = aLeft;
         1767  +            break;
         1768  +          }
  1771   1769           }
  1772   1770         }
  1773   1771         sqlite3_free(aRight);
  1774   1772       }
  1775   1773     }
  1776   1774   
  1777   1775     return rc;

Changes to ext/fts3/fts3Int.h.

    10     10   **
    11     11   ******************************************************************************
    12     12   **
    13     13   */
    14     14   
    15     15   #ifndef _FTSINT_H
    16     16   #define _FTSINT_H
           17  +
           18  +#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) 
           19  +# define NDEBUG 1
           20  +#endif
    17     21   
    18     22   #include <sqlite3.h>
    19     23   #include "fts3_tokenizer.h"
    20     24   #include "fts3_hash.h"
    21     25   
    22     26   /*
    23     27   ** This constant controls how often segments are merged. Once there are

Added test/fts3_common.tcl.

            1  +# 2009 November 04
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +#
           12  +# This file contains common code used the fts3 tests. At one point
           13  +# equivalent functionality was implemented in C code. But it is easier
           14  +# to use Tcl.
           15  +#
           16  +
           17  +#-------------------------------------------------------------------------
           18  +# USAGE: fts3_integrity_check TBL
           19  +#
           20  +# This proc is used to verify that the full-text index is consistent with
           21  +# the contents of the fts3 table. In other words, it checks that the
           22  +# data in the %_contents table matches that in the %_segdir and %_segments 
           23  +# tables.
           24  +#
           25  +# This is not an efficient procedure. It uses a lot of memory and a lot
           26  +# of CPU. But it is better than not checking at all.
           27  +#
           28  +# The procedure is:
           29  +#
           30  +#   1) Read the entire full-text index from the %_segdir and %_segments
           31  +#      tables into memory. For each entry in the index, the following is
           32  +#      done:
           33  +#
           34  +#          set C($iDocid,$iCol,$iPosition) $zTerm
           35  +#
           36  +#   2) Iterate through each column of each row of the %_content table. 
           37  +#      Tokenize all documents, and check that for each token there is
           38  +#      a corresponding entry in the $C array. After checking a token,
           39  +#      [unset] the $C array entry.
           40  +#
           41  +#   3) Check that array $C is now empty.
           42  +#      
           43  +#
           44  +proc fts3_integrity_check {tbl} {
           45  +
           46  +  fts3_read2 $tbl 1 A
           47  +
           48  +  foreach zTerm [array names A] {
           49  +    foreach doclist $A($zTerm) {
           50  +      set docid 0
           51  +      while {[string length $doclist]>0} {
           52  +        set iCol 0
           53  +        set iPos 0
           54  +        set lPos [list]
           55  +        set lCol [list]
           56  +
           57  +        # First varint of a doclist-entry is the docid. Delta-compressed
           58  +        # with respect to the docid of the previous entry.
           59  +        #
           60  +        incr docid [gobble_varint doclist]
           61  +        if {[info exists D($zTerm,$docid)]} {
           62  +          while {[set iDelta [gobble_varint doclist]] != 0} {}
           63  +          continue
           64  +        }
           65  +        set D($zTerm,$docid) 1
           66  +
           67  +        # Gobble varints until the 0x00 that terminates the doclist-entry
           68  +        # is found.
           69  +        while {[set iDelta [gobble_varint doclist]] > 0} {
           70  +          if {$iDelta == 1} {
           71  +            set iCol [gobble_varint doclist]
           72  +            set iPos 0
           73  +          } else {
           74  +            incr iPos $iDelta
           75  +            incr iPos -2
           76  +            set C($docid,$iCol,$iPos) $zTerm
           77  +          }
           78  +        }
           79  +      }
           80  +    }
           81  +  }
           82  +
           83  +
           84  +  db eval "SELECT * FROM ${tbl}_content" E {
           85  +    set iCol 0
           86  +    set iDoc $E(docid)
           87  +    foreach col [lrange $E(*) 1 end] {
           88  +      set c $E($col)
           89  +      set sql {SELECT fts3_tokenizer_test('simple', $c)}
           90  +
           91  +      foreach {pos term dummy} [db one $sql] {
           92  +        if {$C($iDoc,$iCol,$pos) != "$term"} {
           93  +          set    es "Error at docid=$iDoc col=$iCol pos=$pos. "
           94  +          append es "Index has \"$C($iDoc,$iCol,$pos)\", document has \"$term\""
           95  +          lappend errors $es
           96  +        }
           97  +        unset C($iDoc,$iCol,$pos)
           98  +      }
           99  +      incr iCol
          100  +    }
          101  +  }
          102  +
          103  +  foreach c [array names C] {
          104  +    lappend errors "Bad index entry: $c -> $C($c)"
          105  +  }
          106  +
          107  +  if {[info exists errors]} { return [join $errors "\n"] }
          108  +  return "ok"
          109  +}
          110  +
          111  +# USAGE: fts3_terms TBL WHERE
          112  +#
          113  +# Argument TBL must be the name of an FTS3 table. Argument WHERE is an
          114  +# SQL expression that will be used as the WHERE clause when scanning
          115  +# the %_segdir table. As in the following query:
          116  +#
          117  +#   "SELECT * FROM ${TBL}_segdir WHERE ${WHERE}"
          118  +#
          119  +# This function returns a list of all terms present in the segments
          120  +# selected by the statement above.
          121  +#
          122  +proc fts3_terms {tbl where} {
          123  +  fts3_read $tbl $where a
          124  +  return [lsort [array names a]]
          125  +}
          126  +
          127  +
          128  +# USAGE: fts3_doclist TBL TERM WHERE
          129  +#
          130  +# Argument TBL must be the name of an FTS3 table. TERM is a term that may
          131  +# or may not be present in the table. Argument WHERE is used to select a 
          132  +# subset of the b-tree segments in the associated full-text index as 
          133  +# described above for [fts3_terms].
          134  +#
          135  +# This function returns the results of merging the doclists associated
          136  +# with TERM in the selected segments. Each doclist is an element of the
          137  +# returned list. Each doclist is formatted as follows:
          138  +#
          139  +#   [$docid ?$col[$off1 $off2...]?...]
          140  +#
          141  +# The formatting is odd for a Tcl command in order to be compatible with
          142  +# the original C-language implementation. If argument WHERE is "1", then 
          143  +# any empty doclists are omitted from the returned list.
          144  +#
          145  +proc fts3_doclist {tbl term where} {
          146  +  fts3_read $tbl $where a
          147  +
          148  +
          149  +  foreach doclist $a($term) {
          150  +    set docid 0
          151  +
          152  +    while {[string length $doclist]>0} {
          153  +      set iCol 0
          154  +      set iPos 0
          155  +      set lPos [list]
          156  +      set lCol [list]
          157  +      incr docid [gobble_varint doclist]
          158  +  
          159  +      while {[set iDelta [gobble_varint doclist]] > 0} {
          160  +        if {$iDelta == 1} {
          161  +          lappend lCol [list $iCol $lPos]
          162  +          set iPos 0
          163  +          set lPos [list]
          164  +          set iCol [gobble_varint doclist]
          165  +        } else {
          166  +          incr iPos $iDelta
          167  +          incr iPos -2
          168  +          lappend lPos $iPos
          169  +        }
          170  +      }
          171  +  
          172  +      if {[llength $lPos]>0} {
          173  +        lappend lCol [list $iCol $lPos]
          174  +      }
          175  +  
          176  +      if {$where != "1" || [llength $lCol]>0} {
          177  +        set ret($docid) $lCol
          178  +      } else {
          179  +        unset -nocomplain ret($docid)
          180  +      }
          181  +    }
          182  +  }
          183  +
          184  +  set lDoc [list]
          185  +  foreach docid [lsort -integer [array names ret]] {
          186  +    set lCol [list]
          187  +    set cols ""
          188  +    foreach col $ret($docid) {
          189  +      foreach {iCol lPos} $col {}
          190  +      append cols " $iCol\[[join $lPos { }]\]"
          191  +    }
          192  +    lappend lDoc "\[${docid}${cols}\]"
          193  +  }
          194  +
          195  +  join $lDoc " "
          196  +}
          197  +
          198  +###########################################################################
          199  +
          200  +proc gobble_varint {varname} {
          201  +  upvar $varname blob
          202  +  set n [read_varint $blob ret]
          203  +  set blob [string range $blob $n end]
          204  +  return $ret
          205  +}
          206  +proc gobble_string {varname nLength} {
          207  +  upvar $varname blob
          208  +  set ret [string range $blob 0 [expr $nLength-1]]
          209  +  set blob [string range $blob $nLength end]
          210  +  return $ret
          211  +}
          212  +
          213  +# The argument is a blob of data representing an FTS3 segment leaf. 
          214  +# Return a list consisting of alternating terms (strings) and doclists
          215  +# (blobs of data).
          216  +#
          217  +proc fts3_readleaf {blob} {
          218  +  set zPrev ""
          219  +  set terms [list]
          220  +
          221  +  while {[string length $blob] > 0} {
          222  +    set nPrefix [gobble_varint blob]
          223  +    set nSuffix [gobble_varint blob]
          224  +
          225  +    set zTerm [string range $zPrev 0 [expr $nPrefix-1]]
          226  +    append zTerm [gobble_string blob $nSuffix]
          227  +    set doclist [gobble_string blob [gobble_varint blob]]
          228  +
          229  +    lappend terms $zTerm $doclist
          230  +    set zPrev $zTerm
          231  +  }
          232  +
          233  +  return $terms
          234  +}
          235  +
          236  +proc fts3_read2 {tbl where varname} {
          237  +  upvar $varname a
          238  +  array unset a
          239  +  db eval " SELECT start_block, leaves_end_block, root 
          240  +            FROM ${tbl}_segdir WHERE $where
          241  +            ORDER BY level ASC, idx DESC
          242  +  " {
          243  +    if {$start_block == 0} {
          244  +      foreach {t d} [fts3_readleaf $root] { lappend a($t) $d }
          245  +    } else {
          246  +      db eval " SELECT block 
          247  +                FROM ${tbl}_segments 
          248  +                WHERE blockid>=$start_block AND blockid<$leaves_end_block
          249  +                ORDER BY blockid
          250  +      " {
          251  +        foreach {t d} [fts3_readleaf $block] { lappend a($t) $d }
          252  +
          253  +      }
          254  +    }
          255  +  }
          256  +}
          257  +
          258  +proc fts3_read {tbl where varname} {
          259  +  upvar $varname a
          260  +  array unset a
          261  +  db eval " SELECT start_block, leaves_end_block, root 
          262  +            FROM ${tbl}_segdir WHERE $where
          263  +            ORDER BY level DESC, idx ASC
          264  +  " {
          265  +    if {$start_block == 0} {
          266  +      foreach {t d} [fts3_readleaf $root] { lappend a($t) $d }
          267  +    } else {
          268  +      db eval " SELECT block 
          269  +                FROM ${tbl}_segments 
          270  +                WHERE blockid>=$start_block AND blockid<$leaves_end_block
          271  +                ORDER BY blockid
          272  +      " {
          273  +        foreach {t d} [fts3_readleaf $block] { lappend a($t) $d }
          274  +
          275  +      }
          276  +    }
          277  +  }
          278  +}
          279  +

Changes to test/fts3malloc.test.

    13     13   # within the FTS3 module code are handled correctly. 
    14     14   #
    15     15   
    16     16   set testdir [file dirname $argv0]
    17     17   source $testdir/tester.tcl
    18     18   ifcapable !fts3 { finish_test ; return }
    19     19   source $testdir/malloc_common.tcl
           20  +source $testdir/fts3_common.tcl
    20     21   
           22  +set sqlite_fts3_enable_parentheses 1
           23  +
           24  +if 0 {
    21     25   do_malloc_test fts3_malloc-1.1 -sqlbody {
    22     26     CREATE VIRTUAL TABLE ft USING fts3(a, b, c);
    23     27   }
    24         -
    25     28   do_malloc_test fts3_malloc-1.2 -sqlprep {
    26     29     CREATE VIRTUAL TABLE ft USING fts3(a, b, c);
    27     30   } -sqlbody {
    28     31     DROP TABLE ft;
    29     32   }
    30         -
    31         -do_malloc_test fts3_malloc-1.3 -sqlprep {
    32         -  CREATE VIRTUAL TABLE ft USING fts3(content);
    33         -} -sqlbody {
    34         -  INSERT INTO ft VALUES('one two three four');
    35     33   }
    36     34   
    37         -do_malloc_test fts3_malloc-1.4 -tclprep {
    38         -  db eval {CREATE VIRTUAL TABLE ft USING fts3(a, b)}
    39         -  for {set i 0} {$i<16} {incr i} {
    40         -    db eval { INSERT INTO ft VALUES('one two', 'three four') }
    41         -  }
    42         -} -sqlbody {
    43         -  INSERT INTO ft VALUES('one two', 'three four');
    44         -}
    45         -
    46         -proc do_write_test {sql} {
    47         -  uplevel [list db eval $sql]
    48         -}
    49         -
    50         -proc do_read_test {name sql result} {
    51         -
    52         -  if {![info exists ::DO_MALLOC_TEST]} {
    53         -    set ::DO_MALLOC_TEST 1
    54         -  }
    55         -
    56         -  set answers [list [list 0 $result]]
           35  +set DO_MALLOC_TEST 0
           36  +
           37  +#-------------------------------------------------------------------------
           38  +# This proc is used to test a single SELECT statement. Parameter $name is
           39  +# passed a name for the test case (i.e. "fts3_malloc-1.4.1") and parameter
           40  +# $sql is passed the text of the SELECT statement. Parameter $result is
           41  +# set to the expected output if the SELECT statement is successfully
           42  +# executed using [db eval].
           43  +#
           44  +# Example:
           45  +#
           46  +#   do_select_test testcase-1.1 "SELECT 1+1, 1+2" {1 2}
           47  +#
           48  +# If global variable DO_MALLOC_TEST is set to a non-zero value, or if
           49  +# it is not defined at all, then OOM testing is performed on the SELECT
           50  +# statement. Each OOM test case is said to pass if either (a) executing
           51  +# the SELECT statement succeeds and the results match those specified
           52  +# by parameter $result, or (b) TCL throws an "out of memory" error.
           53  +#
           54  +# If DO_MALLOC_TEST is defined and set to zero, then the SELECT statement
           55  +# is executed just once. In this case the test case passes if the results
           56  +# match the expected results passed via parameter $result.
           57  +#
           58  +proc do_select_test {name sql result} {
           59  +
           60  +  if {![info exists ::DO_MALLOC_TEST]} { set ::DO_MALLOC_TEST 1 }
           61  +
    57     62     if {$::DO_MALLOC_TEST } {
    58     63       set answers [list {1 {out of memory}} [list 0 $result]]
    59     64       set modes [list 100000 transient 1 persistent]
    60     65     } else {
           66  +    set answers [list [list 0 $result]]
    61     67       set modes [list 0 nofail]
    62     68     }
    63     69     set str [join $answers " OR "]
    64     70   
    65     71     foreach {nRepeat zName} $modes {
    66     72       for {set iFail 1} 1 {incr iFail} {
    67     73         if {$::DO_MALLOC_TEST} {sqlite3_memdebug_fail $iFail -repeat $nRepeat}
................................................................................
    72     78         }
    73     79         do_test $name.$zName.$iFail [list set {} $res] $str
    74     80         set nFail [sqlite3_memdebug_fail -1 -benigncnt nBenign]
    75     81         if {$nFail==0} break
    76     82       }
    77     83     }
    78     84   }
           85  +
           86  +#-------------------------------------------------------------------------
           87  +# Test a single write to the database. In this case a  "write" is a 
           88  +# DELETE, UPDATE or INSERT statement.
           89  +#
           90  +# If OOM testing is performed, there are several acceptable outcomes:
           91  +#
           92  +#   1) The write succeeds. No error is returned.
           93  +#
           94  +#   2) An "out of memory" exception is thrown and:
           95  +#
           96  +#     a) The statement has no effect, OR
           97  +#     b) The current transaction is rolled back, OR
           98  +#     c) The statement succeeds. This can only happen if the connection
           99  +#        is in auto-commit mode (after the statement is executed, so this
          100  +#        includes COMMIT statements).
          101  +#
          102  +# If the write operation eventually succeeds, zero is returned. If a
          103  +# transaction is rolled back, non-zero is returned.
          104  +#
          105  +# Parameter $name is the name to use for the test case (or test cases).
          106  +# The second parameter, $tbl, should be the name of the database table
          107  +# being modified. Parameter $sql contains the SQL statement to test.
          108  +#
          109  +proc do_write_test {name tbl sql} {
          110  +  # Figure out an statement to get a checksum for table $tbl.
          111  +  db eval "SELECT * FROM $tbl" V break
          112  +  set cksumsql "SELECT md5sum([join [concat rowid $V(*)] ,]) FROM $tbl"
          113  +
          114  +  # Calculate the initial table checksum.
          115  +  set cksum1 [db one $cksumsql]
          116  +
          117  +  if {![info exists ::DO_MALLOC_TEST]} { set ::DO_MALLOC_TEST 1 }
          118  +
          119  +  if {$::DO_MALLOC_TEST } {
          120  +    set answers [list {1 {out of memory}} {0 {}}]
          121  +    set modes [list 100000 transient 1 persistent]
          122  +  } else {
          123  +    set answers [list {0 {}}]
          124  +    set modes [list 0 nofail]
          125  +  }
          126  +  set str [join $answers " OR "]
          127  +
          128  +  foreach {nRepeat zName} $modes {
          129  +    for {set iFail 1} 1 {incr iFail} {
          130  +      if {$::DO_MALLOC_TEST} {sqlite3_memdebug_fail $iFail -repeat $nRepeat}
          131  +
          132  +      set res [catchsql $sql]
          133  +      set nFail [sqlite3_memdebug_fail -1 -benigncnt nBenign]
          134  +      if {$nFail==0} {
          135  +        do_test $name.$zName.$iFail [list set {} $res] {0 {}}
          136  +        return
          137  +      } else {
          138  +        if {[lsearch $answers $res]>=0} {
          139  +          set res $str
          140  +        }
          141  +        do_test $name.$zName.$iFail [list set {} $res] $str
          142  +        set cksum2 [db one $cksumsql]
          143  +        if {$cksum1 != $cksum2} return
          144  +      }
          145  +    }
          146  +  }
          147  +}
    79    148   
    80    149   proc normal_list {l} {
    81    150     set ret [list]
    82    151     foreach elem $l {lappend ret $elem}
    83    152     set ret
    84    153   }
    85    154   
    86    155   db close
    87    156   file delete -force test.db test.db-journal
    88    157   sqlite3 db test.db
    89    158   sqlite3_db_config_lookaside db 0 0 0
    90         -set sqlite_fts3_enable_parentheses 1
    91         -
    92    159   
    93    160   do_test fts3_malloc-2.0 {
    94    161     execsql { CREATE VIRTUAL TABLE ft USING fts3(a, b) }
    95    162     for {set ii 1} {$ii < 32} {incr ii} {
    96    163       set a [list]
    97    164       set b [list]
    98    165       if {$ii & 0x01} {lappend a one   ; lappend b neung}
................................................................................
   229    296     20 {SELECT a FROM ft WHERE ft MATCH 'sahm NOT song'} {
   230    297       {three}                   {one three}           {three four}   
   231    298       {one three four}          {three five}          {one three five}
   232    299       {three four five}         {one three four five}
   233    300     }
   234    301   
   235    302     21 {SELECT a FROM ft WHERE b MATCH 'neung NEAR song NEAR sahm'} {
   236         -    {one two three}          {one two three four}  
   237         -    {one two three five}     {one two three four five}
          303  +    {one two three}           {one two three four}  
          304  +    {one two three five}      {one two three four five}
   238    305     }
   239    306   
   240    307   } {
   241    308     set result [normal_list $result]
   242         -  do_read_test fts3_malloc-2.$tn $sql $result
          309  +  do_select_test fts3_malloc-2.$tn $sql $result
   243    310   }
   244    311   
   245    312   do_test fts3_malloc-3.0 {
   246    313     execsql BEGIN
   247    314     for {set ii 32} {$ii < 1024} {incr ii} {
   248    315       set a [list]
   249    316       set b [list]
................................................................................
   259    326       if {$ii & 0x0200} {lappend a ten   ; lappend b sip   }
   260    327       execsql { INSERT INTO ft VALUES($a, $b) }
   261    328     }
   262    329     execsql COMMIT
   263    330   } {}
   264    331   foreach {tn sql result} {
   265    332     1 "SELECT count(*) FROM ft" {1023}
          333  +
   266    334     2 "SELECT a FROM ft WHERE a MATCH 'one two three four five six seven eight'" {
   267    335        {one two three four five six seven eight}
   268    336        {one two three four five six seven eight nine}
   269    337        {one two three four five six seven eight ten}
   270    338        {one two three four five six seven eight nine ten}
   271    339     }
   272    340   
   273         -  3 {SELECT count(*), sum(docid) FROM ft WHERE a MATCH 'o*'} {512 262144}
          341  +  3 {SELECT count(*), sum(docid) FROM ft WHERE a MATCH 'o*'} {
          342  +    512 262144
          343  +  }
          344  +
   274    345     4 {SELECT count(*), sum(docid) FROM ft WHERE a MATCH '"two three four"'} {
   275    346       128 66368
   276    347     }
   277    348   } {
   278         -#set ::DO_MALLOC_TEST 0
   279    349     set result [normal_list $result]
   280         -  do_read_test fts3_malloc-3.$tn $sql $result
          350  +  do_select_test fts3_malloc-3.$tn $sql $result
          351  +}
          352  +
          353  +do_test fts3_malloc-4.0 {
          354  +  execsql { DELETE FROM ft WHERE docid>=32 }
          355  +} {}
          356  +foreach {tn sql} {
          357  +  1 "DELETE FROM ft WHERE ft MATCH 'one'"
          358  +  2 "DELETE FROM ft WHERE ft MATCH 'three'"
          359  +  3 "DELETE FROM ft WHERE ft MATCH 'five'"
          360  +} {
          361  +  do_write_test fts3_malloc-4.1.$tn ft_content $sql
   281    362   }
          363  +do_test fts3_malloc-4.2 {
          364  +  execsql { SELECT a FROM ft }
          365  +} {two four {two four}}
          366  +
   282    367   
   283    368   finish_test
   284    369