/ Check-in [df1a4b48]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Make DL_POSITION the default mode in FTS1. Remove the need to compile with SQLITE_CORE when SQLITE_ENABLE_FTS1 is used. (CVS 3462)
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: df1a4b4834fdc88056371bcc767c5dfde2eaab72
User & Date: drh 2006-10-03 19:37:37
Context
2006-10-04
11:55
Tests cases automatically remove many of their temporary files. (CVS 3463) check-in: eef0ec0d user: drh tags: trunk
2006-10-03
19:37
Make DL_POSITION the default mode in FTS1. Remove the need to compile with SQLITE_CORE when SQLITE_ENABLE_FTS1 is used. (CVS 3462) check-in: df1a4b48 user: drh tags: trunk
19:12
Modify the makefile(s) to know about the FTS1 module - however FTS1 is turned off by default. Bump the version number to 3.3.8. (CVS 3461) check-in: 288ff637 user: drh tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts1/fts1.c.

9
10
11
12
13
14
15




16
17
18
19
20
21
22
...
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
....
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
**     * The FTS1 module is being built as an extension
**       (in which case SQLITE_CORE is not defined), or
**
**     * The FTS1 module is being built into the core of
**       SQLite (in which case SQLITE_ENABLE_FTS1 is defined).
*/
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS1)





#include <assert.h>
#if !defined(__APPLE__)
#include <malloc.h>
#else
#include <stdlib.h>
#endif
................................................................................
typedef enum DocListType {
  DL_DOCIDS,              /* docids only */
  DL_POSITIONS,           /* docids + positions */
  DL_POSITIONS_OFFSETS    /* docids + positions + offsets */
} DocListType;

/*
** By default, positions and offsets are stored in the doclists.
** To change this so that only positions are stored, compile
** with
**
**          -DDL_DEFAULT=DL_POSITIONS
**
*/
#ifndef DL_DEFAULT
# define DL_DEFAULT DL_POSITIONS_OFFSETS
#endif

typedef struct DocList {
  char *pData;
  int nData;
  DocListType iType;
  int iLastColumn;    /* the last column written */
................................................................................
  v->azColumn = spec->azColumn;
  spec->azColumn = 0;

  if( spec->azTokenizer==0 ){
    return SQLITE_NOMEM;
  }
  /* TODO(shess) For now, add new tokenizers as else if clauses. */
  if( spec->azTokenizer[0]==0 || !strcmp(spec->azTokenizer[0], "simple") ){
    sqlite3Fts1SimpleTokenizerModule(&m);
  }else if( strcmp(spec->azTokenizer[0], "porter")==0 ){
    sqlite3Fts1PorterTokenizerModule(&m);
  }else{
    *pzErr = sqlite3_mprintf("unknown tokenizer: %s", spec->azTokenizer[0]);
    rc = SQLITE_ERROR;
    goto err;
  }
  for(n=0; spec->azTokenizer[n]; n++){}







>
>
>
>







 







|
|
<

|



|







 







|

|







9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
...
180
181
182
183
184
185
186
187
188

189
190
191
192
193
194
195
196
197
198
199
200
201
....
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
**     * The FTS1 module is being built as an extension
**       (in which case SQLITE_CORE is not defined), or
**
**     * The FTS1 module is being built into the core of
**       SQLite (in which case SQLITE_ENABLE_FTS1 is defined).
*/
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS1)

#if defined(SQLITE_ENABLE_FTS1) && !defined(SQLITE_CORE)
# define SQLITE_CORE 1
#endif

#include <assert.h>
#if !defined(__APPLE__)
#include <malloc.h>
#else
#include <stdlib.h>
#endif
................................................................................
typedef enum DocListType {
  DL_DOCIDS,              /* docids only */
  DL_POSITIONS,           /* docids + positions */
  DL_POSITIONS_OFFSETS    /* docids + positions + offsets */
} DocListType;

/*
** By default, only positions and not offsets are stored in the doclists.
** To change this so that offsets are stored too, compile with

**
**          -DDL_DEFAULT=DL_POSITIONS_OFFSETS
**
*/
#ifndef DL_DEFAULT
# define DL_DEFAULT DL_POSITIONS
#endif

typedef struct DocList {
  char *pData;
  int nData;
  DocListType iType;
  int iLastColumn;    /* the last column written */
................................................................................
  v->azColumn = spec->azColumn;
  spec->azColumn = 0;

  if( spec->azTokenizer==0 ){
    return SQLITE_NOMEM;
  }
  /* TODO(shess) For now, add new tokenizers as else if clauses. */
  if( spec->azTokenizer[0]==0 || startsWith(spec->azTokenizer[0], "simple") ){
    sqlite3Fts1SimpleTokenizerModule(&m);
  }else if( startsWith(spec->azTokenizer[0], "porter") ){
    sqlite3Fts1PorterTokenizerModule(&m);
  }else{
    *pzErr = sqlite3_mprintf("unknown tokenizer: %s", spec->azTokenizer[0]);
    rc = SQLITE_ERROR;
    goto err;
  }
  for(n=0; spec->azTokenizer[n]; n++){}

Changes to test/fts1porter.test.

8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
.....
23569
23570
23571
23572
23573
23574
23575
23576
23577
23578
23579
23580
23581
23582
23583
#    May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library.  The
# focus of this script is testing the FTS1 module, and in particular
# the Porter stemmer.
#
# $Id: fts1porter.test,v 1.4 2006/10/01 20:41:03 drh Exp $
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# If SQLITE_ENABLE_FTS1 is defined, omit this file.
ifcapable !fts1 {
................................................................................
}                         

# Create a full-text index to use for testing the stemmer.
#
db close
sqlite3 db :memory:
db eval {
  CREATE VIRTUAL TABLE t1 USING fts1(word, tokenize porter);
}

foreach {pfrom pto} $porter_test_data {
  do_test fts1porter-$pfrom {
    execsql {
      DELETE FROM t1_term;
      DELETE FROM t1_content;







|







 







|







8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
.....
23569
23570
23571
23572
23573
23574
23575
23576
23577
23578
23579
23580
23581
23582
23583
#    May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library.  The
# focus of this script is testing the FTS1 module, and in particular
# the Porter stemmer.
#
# $Id: fts1porter.test,v 1.5 2006/10/03 19:37:37 drh Exp $
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# If SQLITE_ENABLE_FTS1 is defined, omit this file.
ifcapable !fts1 {
................................................................................
}                         

# Create a full-text index to use for testing the stemmer.
#
db close
sqlite3 db :memory:
db eval {
  CREATE VIRTUAL TABLE t1 USING fts1(word, tokenize Porter);
}

foreach {pfrom pto} $porter_test_data {
  do_test fts1porter-$pfrom {
    execsql {
      DELETE FROM t1_term;
      DELETE FROM t1_content;