SQLite

Check-in [df1a4b4834]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Make DL_POSITION the default mode in FTS1. Remove the need to compile with SQLITE_CORE when SQLITE_ENABLE_FTS1 is used. (CVS 3462)
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: df1a4b4834fdc88056371bcc767c5dfde2eaab72
User & Date: drh 2006-10-03 19:37:37.000
Context
2006-10-04
11:55
Tests cases automatically remove many of their temporary files. (CVS 3463) (check-in: eef0ec0d7a user: drh tags: trunk)
2006-10-03
19:37
Make DL_POSITION the default mode in FTS1. Remove the need to compile with SQLITE_CORE when SQLITE_ENABLE_FTS1 is used. (CVS 3462) (check-in: df1a4b4834 user: drh tags: trunk)
19:12
Modify the makefile(s) to know about the FTS1 module - however FTS1 is turned off by default. Bump the version number to 3.3.8. (CVS 3461) (check-in: 288ff63783 user: drh tags: trunk)
Changes
Unified Diff Ignore Whitespace Patch
Changes to ext/fts1/fts1.c.
︙︙
9
10
11
12
13
14
15




16
17
18
19
20
21
22

**     * The FTS1 module is being built as an extension
**       (in which case SQLITE_CORE is not defined), or
**
**     * The FTS1 module is being built into the core of
**       SQLite (in which case SQLITE_ENABLE_FTS1 is defined).
*/
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS1)





#include <assert.h>
#if !defined(__APPLE__)
#include <malloc.h>
#else
#include <stdlib.h>
#endif








>
>
>
>








9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

**     * The FTS1 module is being built as an extension
**       (in which case SQLITE_CORE is not defined), or
**
**     * The FTS1 module is being built into the core of
**       SQLite (in which case SQLITE_ENABLE_FTS1 is defined).
*/
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS1)

#if defined(SQLITE_ENABLE_FTS1) && !defined(SQLITE_CORE)
# define SQLITE_CORE 1
#endif

#include <assert.h>
#if !defined(__APPLE__)
#include <malloc.h>
#else
#include <stdlib.h>
#endif
︙︙
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198

typedef enum DocListType {
  DL_DOCIDS,              /* docids only */
  DL_POSITIONS,           /* docids + positions */
  DL_POSITIONS_OFFSETS    /* docids + positions + offsets */
} DocListType;

/*
** By default, positions and offsets are stored in the doclists.
** To change this so that only positions are stored, compile
** with
**
**          -DDL_DEFAULT=DL_POSITIONS
**
*/
#ifndef DL_DEFAULT
# define DL_DEFAULT DL_POSITIONS_OFFSETS
#endif

typedef struct DocList {
  char *pData;
  int nData;
  DocListType iType;
  int iLastColumn;    /* the last column written */








|
|
<

|



|








180
181
182
183
184
185
186
187
188

189
190
191
192
193
194
195
196
197
198
199
200
201

typedef enum DocListType {
  DL_DOCIDS,              /* docids only */
  DL_POSITIONS,           /* docids + positions */
  DL_POSITIONS_OFFSETS    /* docids + positions + offsets */
} DocListType;

/*
** By default, only positions and not offsets are stored in the doclists.
** To change this so that offsets are stored too, compile with

**
**          -DDL_DEFAULT=DL_POSITIONS_OFFSETS
**
*/
#ifndef DL_DEFAULT
# define DL_DEFAULT DL_POSITIONS
#endif

typedef struct DocList {
  char *pData;
  int nData;
  DocListType iType;
  int iLastColumn;    /* the last column written */
︙︙
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924

  v->azColumn = spec->azColumn;
  spec->azColumn = 0;

  if( spec->azTokenizer==0 ){
    return SQLITE_NOMEM;
  }
  /* TODO(shess) For now, add new tokenizers as else if clauses. */
  if( spec->azTokenizer[0]==0 || !strcmp(spec->azTokenizer[0], "simple") ){
    sqlite3Fts1SimpleTokenizerModule(&m);
  }else if( strcmp(spec->azTokenizer[0], "porter")==0 ){
    sqlite3Fts1PorterTokenizerModule(&m);
  }else{
    *pzErr = sqlite3_mprintf("unknown tokenizer: %s", spec->azTokenizer[0]);
    rc = SQLITE_ERROR;
    goto err;
  }
  for(n=0; spec->azTokenizer[n]; n++){}








|

|








1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927

  v->azColumn = spec->azColumn;
  spec->azColumn = 0;

  if( spec->azTokenizer==0 ){
    return SQLITE_NOMEM;
  }
  /* TODO(shess) For now, add new tokenizers as else if clauses. */
  if( spec->azTokenizer[0]==0 || startsWith(spec->azTokenizer[0], "simple") ){
    sqlite3Fts1SimpleTokenizerModule(&m);
  }else if( startsWith(spec->azTokenizer[0], "porter") ){
    sqlite3Fts1PorterTokenizerModule(&m);
  }else{
    *pzErr = sqlite3_mprintf("unknown tokenizer: %s", spec->azTokenizer[0]);
    rc = SQLITE_ERROR;
    goto err;
  }
  for(n=0; spec->azTokenizer[n]; n++){}
︙︙
Changes to test/fts1porter.test.
︙︙
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22

#    May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library.  The
# focus of this script is testing the FTS1 module, and in particular
# the Porter stemmer.
#
# $Id: fts1porter.test,v 1.4 2006/10/01 20:41:03 drh Exp $
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# If SQLITE_ENABLE_FTS1 is defined, omit this file.
ifcapable !fts1 {








|








8
9
10
11
12
13
14
15
16
17
18
19
20
21
22

#    May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library.  The
# focus of this script is testing the FTS1 module, and in particular
# the Porter stemmer.
#
# $Id: fts1porter.test,v 1.5 2006/10/03 19:37:37 drh Exp $
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# If SQLITE_ENABLE_FTS1 is defined, omit this file.
ifcapable !fts1 {
︙︙
23569
23570
23571
23572
23573
23574
23575
23576
23577
23578
23579
23580
23581
23582
23583

}                         

# Create a full-text index to use for testing the stemmer.
#
db close
sqlite3 db :memory:
db eval {
  CREATE VIRTUAL TABLE t1 USING fts1(word, tokenize porter);
}

foreach {pfrom pto} $porter_test_data {
  do_test fts1porter-$pfrom {
    execsql {
      DELETE FROM t1_term;
      DELETE FROM t1_content;








|








23569
23570
23571
23572
23573
23574
23575
23576
23577
23578
23579
23580
23581
23582
23583

}                         

# Create a full-text index to use for testing the stemmer.
#
db close
sqlite3 db :memory:
db eval {
  CREATE VIRTUAL TABLE t1 USING fts1(word, tokenize Porter);
}

foreach {pfrom pto} $porter_test_data {
  do_test fts1porter-$pfrom {
    execsql {
      DELETE FROM t1_term;
      DELETE FROM t1_content;
︙︙