SQLite

Check-in [3b17924754]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix some fts3 related issues with the autoconf and amalgamation build systems.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both
Files: files | file ages | folders
SHA1: 3b17924754343c0163464dabf01a9c46ffccef28
User & Date: dan 2009-11-19 15:25:25.000
Context
2009-11-19
15:26
Merge fts3-refactor into the main trunk. (check-in: 551dbe30ea user: dan tags: trunk)
15:25
Fix some fts3 related issues with the autoconf and amalgamation build systems. (Closed-Leaf check-in: 3b17924754 user: dan)
14:57
Merge with [4bd4330709]. (check-in: 7a46d1ebe3 user: dan tags: fts3-refactor)
Changes
Unified Diff Ignore Whitespace Patch
Changes to Makefile.in.
307
308
309
310
311
312
313
314
315
316
317
318
319

320
321
322

323
324
325
326
327
328
329
  $(TOP)/ext/fts2/fts2_porter.c \
  $(TOP)/ext/fts2/fts2_tokenizer.h \
  $(TOP)/ext/fts2/fts2_tokenizer.c \
  $(TOP)/ext/fts2/fts2_tokenizer1.c
SRC += \
  $(TOP)/ext/fts3/fts3.c \
  $(TOP)/ext/fts3/fts3.h \
  $(TOP)/ext/fts3/fts3_expr.c \
  $(TOP)/ext/fts3/fts3_expr.h \
  $(TOP)/ext/fts3/fts3_hash.c \
  $(TOP)/ext/fts3/fts3_hash.h \
  $(TOP)/ext/fts3/fts3_icu.c \
  $(TOP)/ext/fts3/fts3_porter.c \

  $(TOP)/ext/fts3/fts3_tokenizer.h \
  $(TOP)/ext/fts3/fts3_tokenizer.c \
  $(TOP)/ext/fts3/fts3_tokenizer1.c

SRC += \
  $(TOP)/ext/icu/sqliteicu.h \
  $(TOP)/ext/icu/icu.c
SRC += \
  $(TOP)/ext/rtree/rtree.h \
  $(TOP)/ext/rtree/rtree.c








|
|




>


|
>







307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
  $(TOP)/ext/fts2/fts2_porter.c \
  $(TOP)/ext/fts2/fts2_tokenizer.h \
  $(TOP)/ext/fts2/fts2_tokenizer.c \
  $(TOP)/ext/fts2/fts2_tokenizer1.c
SRC += \
  $(TOP)/ext/fts3/fts3.c \
  $(TOP)/ext/fts3/fts3.h \
  $(TOP)/ext/fts3/fts3Int.h \
  $(TOP)/ext/fts3/fts3_expr.c \
  $(TOP)/ext/fts3/fts3_hash.c \
  $(TOP)/ext/fts3/fts3_hash.h \
  $(TOP)/ext/fts3/fts3_icu.c \
  $(TOP)/ext/fts3/fts3_porter.c \
  $(TOP)/ext/fts3/fts3_snippet.c \
  $(TOP)/ext/fts3/fts3_tokenizer.h \
  $(TOP)/ext/fts3/fts3_tokenizer.c \
  $(TOP)/ext/fts3/fts3_tokenizer1.c \
  $(TOP)/ext/fts3/fts3_write.c
SRC += \
  $(TOP)/ext/icu/sqliteicu.h \
  $(TOP)/ext/icu/icu.c
SRC += \
  $(TOP)/ext/rtree/rtree.h \
  $(TOP)/ext/rtree/rtree.c

424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
  $(TOP)/ext/fts1/fts1_tokenizer.h
HDR += \
  $(TOP)/ext/fts2/fts2.h \
  $(TOP)/ext/fts2/fts2_hash.h \
  $(TOP)/ext/fts2/fts2_tokenizer.h
HDR += \
  $(TOP)/ext/fts3/fts3.h \
  $(TOP)/ext/fts3/fts3_expr.h \
  $(TOP)/ext/fts3/fts3_hash.h \
  $(TOP)/ext/fts3/fts3_tokenizer.h
HDR += \
  $(TOP)/ext/rtree/rtree.h
HDR += \
  $(TOP)/ext/icu/sqliteicu.h








|







426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
  $(TOP)/ext/fts1/fts1_tokenizer.h
HDR += \
  $(TOP)/ext/fts2/fts2.h \
  $(TOP)/ext/fts2/fts2_hash.h \
  $(TOP)/ext/fts2/fts2_tokenizer.h
HDR += \
  $(TOP)/ext/fts3/fts3.h \
  $(TOP)/ext/fts3/fts3Int.h \
  $(TOP)/ext/fts3/fts3_hash.h \
  $(TOP)/ext/fts3/fts3_tokenizer.h
HDR += \
  $(TOP)/ext/rtree/rtree.h
HDR += \
  $(TOP)/ext/icu/sqliteicu.h

configure became executable.
Changes to ext/fts3/fts3_snippet.c.
1
2
3
4
5
6
7
8
9
10
11
12
13


14
15
16
17
18
19
20
/*
** 2009 Oct 23
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
******************************************************************************
*/



#include "fts3Int.h"
#include <string.h>
#include <assert.h>
#include <ctype.h>

typedef struct Snippet Snippet;














>
>







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
/*
** 2009 Oct 23
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
******************************************************************************
*/

#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)

#include "fts3Int.h"
#include <string.h>
#include <assert.h>
#include <ctype.h>

typedef struct Snippet Snippet;

744
745
746
747
748
749
750

  Snippet *p;                     /* Snippet structure */
  int rc = snippetAllOffsets(pCsr, &p);
  snippetText(pCsr, p, zStart, zEnd, zEllipsis);
  sqlite3_result_text(pCtx, p->zSnippet, p->nSnippet, SQLITE_TRANSIENT);
  fts3SnippetFree(p);
}









>
746
747
748
749
750
751
752
753
  Snippet *p;                     /* Snippet structure */
  int rc = snippetAllOffsets(pCsr, &p);
  snippetText(pCsr, p, zStart, zEnd, zEllipsis);
  sqlite3_result_text(pCtx, p->zSnippet, p->nSnippet, SQLITE_TRANSIENT);
  fts3SnippetFree(p);
}

#endif
Changes to ext/fts3/fts3_tokenizer.c.
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
#endif

#include "fts3Int.h"
#include <assert.h>
#include <ctype.h>
#include <string.h>

static int safe_isspace(char c){
  return (c&0x80)==0 ? isspace(c) : 0;
}

/*
** Implementation of the SQL scalar function for accessing the underlying 
** hash table. This function may be called as follows:
**
**   SELECT <function-name>(<key-name>);
**   SELECT <function-name>(<key-name>, <pointer>);
**







<
<
<
<







31
32
33
34
35
36
37




38
39
40
41
42
43
44
#endif

#include "fts3Int.h"
#include <assert.h>
#include <ctype.h>
#include <string.h>





/*
** Implementation of the SQL scalar function for accessing the underlying 
** hash table. This function may be called as follows:
**
**   SELECT <function-name>(<key-name>);
**   SELECT <function-name>(<key-name>, <pointer>);
**
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
  char *zCopy;
  char *zEnd;                     /* Pointer to nul-term of zCopy */
  sqlite3_tokenizer_module *m;

  if( !z ){
    zCopy = sqlite3_mprintf("simple");
  }else{
    while( safe_isspace(*z) ) z++;
    if( sqlite3_strnicmp(z, "tokenize", 8) || fts3IsIdChar(z[8])){
      return SQLITE_OK;
    }
    zCopy = sqlite3_mprintf("%s", &z[8]);
    *pzTokenizer = zArg;
  }
  if( !zCopy ){







|







162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
  char *zCopy;
  char *zEnd;                     /* Pointer to nul-term of zCopy */
  sqlite3_tokenizer_module *m;

  if( !z ){
    zCopy = sqlite3_mprintf("simple");
  }else{
    while( (*z&0x80) && isspace(*z) ) z++;
    if( sqlite3_strnicmp(z, "tokenize", 8) || fts3IsIdChar(z[8])){
      return SQLITE_OK;
    }
    zCopy = sqlite3_mprintf("%s", &z[8]);
    *pzTokenizer = zArg;
  }
  if( !zCopy ){
Changes to ext/fts3/fts3_write.c.
13
14
15
16
17
18
19


20
21
22
23
24
25
26
** This file is part of the SQLite FTS3 extension module. Specifically,
** this file contains code to insert, update and delete rows from FTS3
** tables. It also contains code to merge FTS3 b-tree segments. Some
** of the sub-routines used to merge segments are also used by the query 
** code in fts3.c.
*/



#include "fts3Int.h"
#include <string.h>
#include <assert.h>
#include <stdlib.h>

#define INTERIOR_MAX 2048         /* Soft limit for segment node size */
#define LEAF_MAX 2048             /* Soft limit for segment leaf size */







>
>







13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
** This file is part of the SQLite FTS3 extension module. Specifically,
** this file contains code to insert, update and delete rows from FTS3
** tables. It also contains code to merge FTS3 b-tree segments. Some
** of the sub-routines used to merge segments are also used by the query 
** code in fts3.c.
*/

#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)

#include "fts3Int.h"
#include <string.h>
#include <assert.h>
#include <stdlib.h>

#define INTERIOR_MAX 2048         /* Soft limit for segment node size */
#define LEAF_MAX 2048             /* Soft limit for segment leaf size */
1943
1944
1945
1946
1947
1948
1949

  int rc = sqlite3Fts3PendingTermsFlush(p);
  if( rc==SQLITE_OK ){
    rc = fts3SegmentMerge(p, -1);
  }
  return rc;
}









>
1945
1946
1947
1948
1949
1950
1951
1952
  int rc = sqlite3Fts3PendingTermsFlush(p);
  if( rc==SQLITE_OK ){
    rc = fts3SegmentMerge(p, -1);
  }
  return rc;
}

#endif
Changes to main.mk.
183
184
185
186
187
188
189

190
191
192
193
194
195
196
  $(TOP)/ext/fts3/fts3.c \
  $(TOP)/ext/fts3/fts3.h \
  $(TOP)/ext/fts3/fts3_expr.c \
  $(TOP)/ext/fts3/fts3_hash.c \
  $(TOP)/ext/fts3/fts3_hash.h \
  $(TOP)/ext/fts3/fts3_icu.c \
  $(TOP)/ext/fts3/fts3_porter.c \

  $(TOP)/ext/fts3/fts3_tokenizer.h \
  $(TOP)/ext/fts3/fts3_tokenizer.c \
  $(TOP)/ext/fts3/fts3_tokenizer1.c \
  $(TOP)/ext/fts3/fts3_write.c
SRC += \
  $(TOP)/ext/icu/sqliteicu.h \
  $(TOP)/ext/icu/icu.c







>







183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
  $(TOP)/ext/fts3/fts3.c \
  $(TOP)/ext/fts3/fts3.h \
  $(TOP)/ext/fts3/fts3_expr.c \
  $(TOP)/ext/fts3/fts3_hash.c \
  $(TOP)/ext/fts3/fts3_hash.h \
  $(TOP)/ext/fts3/fts3_icu.c \
  $(TOP)/ext/fts3/fts3_porter.c \
  $(TOP)/ext/fts3/fts3_snippet.c \
  $(TOP)/ext/fts3/fts3_tokenizer.h \
  $(TOP)/ext/fts3/fts3_tokenizer.c \
  $(TOP)/ext/fts3/fts3_tokenizer1.c \
  $(TOP)/ext/fts3/fts3_write.c
SRC += \
  $(TOP)/ext/icu/sqliteicu.h \
  $(TOP)/ext/icu/icu.c
Changes to test/fts3malloc.test.
25
26
27
28
29
30
31
32



33
34
35
36
37
38
39
sqlite3_db_config_lookaside db 0 0 0

set sqlite_fts3_enable_parentheses 1
set DO_MALLOC_TEST 1

# Test organization:
#
# fts3_malloc-1.*: Test CREATE and DROP table statements.



#
#

#-------------------------------------------------------------------------
# This proc is used to test a single SELECT statement. Parameter $name is
# passed a name for the test case (i.e. "fts3_malloc-1.4.1") and parameter
# $sql is passed the text of the SELECT statement. Parameter $result is







|
>
>
>







25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
sqlite3_db_config_lookaside db 0 0 0

set sqlite_fts3_enable_parentheses 1
set DO_MALLOC_TEST 1

# Test organization:
#
# fts3_malloc-1.*: Test OOM during CREATE and DROP table statements.
# fts3_malloc-2.*: Test OOM during SELECT operations.
# fts3_malloc-3.*: Test OOM during SELECT operations with a larger database.
# fts3_malloc-4.*: Test OOM during database write operations.
#
#

#-------------------------------------------------------------------------
# This proc is used to test a single SELECT statement. Parameter $name is
# passed a name for the test case (i.e. "fts3_malloc-1.4.1") and parameter
# $sql is passed the text of the SELECT statement. Parameter $result is
50
51
52
53
54
55
56


57





58
59
60
61
62
63
64
65
# the SELECT statement succeeds and the results match those specified
# by parameter $result, or (b) TCL throws an "out of memory" error.
#
# If DO_MALLOC_TEST is defined and set to zero, then the SELECT statement
# is executed just once. In this case the test case passes if the results
# match the expected results passed via parameter $result.
#








proc do_passive_test {name sql catchres} {
  if {![info exists ::DO_MALLOC_TEST]} { set ::DO_MALLOC_TEST 1 }

  if {$::DO_MALLOC_TEST} {
    set answers [list {1 {out of memory}} $catchres]
    set modes [list 100000 transient 1 persistent]
  } else {
    set answers [list $catchres]







>
>
|
>
>
>
>
>
|







53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
# the SELECT statement succeeds and the results match those specified
# by parameter $result, or (b) TCL throws an "out of memory" error.
#
# If DO_MALLOC_TEST is defined and set to zero, then the SELECT statement
# is executed just once. In this case the test case passes if the results
# match the expected results passed via parameter $result.
#
proc do_select_test {name sql result} {
  doPassiveTest $name $sql [list 0 $result]
}

proc do_error_test {name sql error} {
  doPassiveTest $name $sql [list 1 $error]
}

proc doPassiveTest {name sql catchres} {
  if {![info exists ::DO_MALLOC_TEST]} { set ::DO_MALLOC_TEST 1 }

  if {$::DO_MALLOC_TEST} {
    set answers [list {1 {out of memory}} $catchres]
    set modes [list 100000 transient 1 persistent]
  } else {
    set answers [list $catchres]
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
      do_test $name.$zName.$iFail [list set {} $res] $str
      set nFail [sqlite3_memdebug_fail -1 -benigncnt nBenign]
      if {$nFail==0} break
    }
  }
}

proc do_select_test {name sql result} {
  do_passive_test $name $sql [list 0 $result]
}

proc do_error_test {name sql error} {
  do_passive_test $name $sql [list 1 $error]
}

#-------------------------------------------------------------------------
# Test a single write to the database. In this case a  "write" is a 
# DELETE, UPDATE or INSERT statement.
#
# If OOM testing is performed, there are several acceptable outcomes:
#







<
<
<
<
<
<
<







88
89
90
91
92
93
94







95
96
97
98
99
100
101
      do_test $name.$zName.$iFail [list set {} $res] $str
      set nFail [sqlite3_memdebug_fail -1 -benigncnt nBenign]
      if {$nFail==0} break
    }
  }
}









#-------------------------------------------------------------------------
# Test a single write to the database. In this case a  "write" is a 
# DELETE, UPDATE or INSERT statement.
#
# If OOM testing is performed, there are several acceptable outcomes:
#
Changes to tool/mksqlite3c.tcl.
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
# files are seen in a #include statement in the C code, include the complete
# text of the file in-line.  The file only needs to be included once.
#
foreach hdr {
   btree.h
   btreeInt.h
   fts3.h
   fts3_expr.h
   fts3_hash.h
   fts3_tokenizer.h
   hash.h
   hwtime.h
   keywordhash.h
   mutex.h
   opcodes.h







<







83
84
85
86
87
88
89

90
91
92
93
94
95
96
# files are seen in a #include statement in the C code, include the complete
# text of the file in-line.  The file only needs to be included once.
#
foreach hdr {
   btree.h
   btreeInt.h
   fts3.h

   fts3_hash.h
   fts3_tokenizer.h
   hash.h
   hwtime.h
   keywordhash.h
   mutex.h
   opcodes.h
291
292
293
294
295
296
297


298
299
300
301
302
303
304
305
306

   fts3.c
   fts3_expr.c
   fts3_hash.c
   fts3_porter.c
   fts3_tokenizer.c
   fts3_tokenizer1.c



   rtree.c
   icu.c
   fts3_icu.c
} {
  copy_file tsrc/$file
}

close $out







>
>









290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307

   fts3.c
   fts3_expr.c
   fts3_hash.c
   fts3_porter.c
   fts3_tokenizer.c
   fts3_tokenizer1.c
   fts3_write.c
   fts3_snippet.c

   rtree.c
   icu.c
   fts3_icu.c
} {
  copy_file tsrc/$file
}

close $out