SQLite: Check-in [df1a4b48]

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview

Comment:	Make DL_POSITION the default mode in FTS1. Remove the need to compile with SQLITE_CORE when SQLITE_ENABLE_FTS1 is used. (CVS 3462)
Downloads:	Tarball \| ZIP archive \| SQL archive
Timelines:	family \| ancestors \| descendants \| both \| trunk
Files:	files \| file ages \| folders
SHA1:	df1a4b4834fdc88056371bcc767c5dfde2eaab72
User & Date:	drh 2006-10-03 19:37:37

Context

2006-10-04
11:55		Tests cases automatically remove many of their temporary files. (CVS 3463) (check-in: eef0ec0d user: drh tags: trunk)
2006-10-03
19:37		Make DL_POSITION the default mode in FTS1. Remove the need to compile with SQLITE_CORE when SQLITE_ENABLE_FTS1 is used. (CVS 3462) (check-in: df1a4b48 user: drh tags: trunk)
19:12		Modify the makefile(s) to know about the FTS1 module - however FTS1 is turned off by default. Bump the version number to 3.3.8. (CVS 3461) (check-in: 288ff637 user: drh tags: trunk)

Changes

Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts1/fts1.c.

**     * The FTS1 module is being built as an extension
**       (in which case SQLITE_CORE is not defined), or
**
**     * The FTS1 module is being built into the core of
**       SQLite (in which case SQLITE_ENABLE_FTS1 is defined).
*/
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS1)





#include <assert.h>
#if !defined(__APPLE__)
#include <malloc.h>
#else
#include <stdlib.h>
#endif
................................................................................
typedef enum DocListType {
  DL_DOCIDS,              /* docids only */
  DL_POSITIONS,           /* docids + positions */
  DL_POSITIONS_OFFSETS    /* docids + positions + offsets */
} DocListType;

/*
** By default, positions and offsets are stored in the doclists.
** To change this so that only positions are stored, compile
** with
**
**          -DDL_DEFAULT=DL_POSITIONS
**
*/
#ifndef DL_DEFAULT
# define DL_DEFAULT DL_POSITIONS_OFFSETS
#endif

typedef struct DocList {
  char *pData;
  int nData;
  DocListType iType;
  int iLastColumn;    /* the last column written */
................................................................................
  v->azColumn = spec->azColumn;
  spec->azColumn = 0;

  if( spec->azTokenizer==0 ){
    return SQLITE_NOMEM;
  }
  /* TODO(shess) For now, add new tokenizers as else if clauses. */
  if( spec->azTokenizer[0]==0 || !strcmp(spec->azTokenizer[0], "simple") ){
    sqlite3Fts1SimpleTokenizerModule(&m);
  }else if( strcmp(spec->azTokenizer[0], "porter")==0 ){
    sqlite3Fts1PorterTokenizerModule(&m);
  }else{
    *pzErr = sqlite3_mprintf("unknown tokenizer: %s", spec->azTokenizer[0]);
    rc = SQLITE_ERROR;
    goto err;
  }
  for(n=0; spec->azTokenizer[n]; n++){}

**     * The FTS1 module is being built as an extension
**       (in which case SQLITE_CORE is not defined), or
**
**     * The FTS1 module is being built into the core of
**       SQLite (in which case SQLITE_ENABLE_FTS1 is defined).
*/
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS1)

#if defined(SQLITE_ENABLE_FTS1) && !defined(SQLITE_CORE)
# define SQLITE_CORE 1
#endif

#include <assert.h>
#if !defined(__APPLE__)
#include <malloc.h>
#else
#include <stdlib.h>
#endif
................................................................................
typedef enum DocListType {
  DL_DOCIDS,              /* docids only */
  DL_POSITIONS,           /* docids + positions */
  DL_POSITIONS_OFFSETS    /* docids + positions + offsets */
} DocListType;

/*
** By default, only positions and not offsets are stored in the doclists.
** To change this so that offsets are stored too, compile with

**
**          -DDL_DEFAULT=DL_POSITIONS_OFFSETS
**
*/
#ifndef DL_DEFAULT
# define DL_DEFAULT DL_POSITIONS
#endif

typedef struct DocList {
  char *pData;
  int nData;
  DocListType iType;
  int iLastColumn;    /* the last column written */
................................................................................
  v->azColumn = spec->azColumn;
  spec->azColumn = 0;

  if( spec->azTokenizer==0 ){
    return SQLITE_NOMEM;
  }
  /* TODO(shess) For now, add new tokenizers as else if clauses. */
  if( spec->azTokenizer[0]==0 || startsWith(spec->azTokenizer[0], "simple") ){
    sqlite3Fts1SimpleTokenizerModule(&m);
  }else if( startsWith(spec->azTokenizer[0], "porter") ){
    sqlite3Fts1PorterTokenizerModule(&m);
  }else{
    *pzErr = sqlite3_mprintf("unknown tokenizer: %s", spec->azTokenizer[0]);
    rc = SQLITE_ERROR;
    goto err;
  }
  for(n=0; spec->azTokenizer[n]; n++){}

Changes to test/fts1porter.test.

#    May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library.  The
# focus of this script is testing the FTS1 module, and in particular
# the Porter stemmer.
#
# $Id: fts1porter.test,v 1.4 2006/10/01 20:41:03 drh Exp $
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# If SQLITE_ENABLE_FTS1 is defined, omit this file.
ifcapable !fts1 {
................................................................................
}                         

# Create a full-text index to use for testing the stemmer.
#
db close
sqlite3 db :memory:
db eval {
  CREATE VIRTUAL TABLE t1 USING fts1(word, tokenize porter);
}

foreach {pfrom pto} $porter_test_data {
  do_test fts1porter-$pfrom {
    execsql {
      DELETE FROM t1_term;
      DELETE FROM t1_content;

#    May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library.  The
# focus of this script is testing the FTS1 module, and in particular
# the Porter stemmer.
#
# $Id: fts1porter.test,v 1.5 2006/10/03 19:37:37 drh Exp $
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# If SQLITE_ENABLE_FTS1 is defined, omit this file.
ifcapable !fts1 {
................................................................................
}                         

# Create a full-text index to use for testing the stemmer.
#
db close
sqlite3 db :memory:
db eval {
  CREATE VIRTUAL TABLE t1 USING fts1(word, tokenize Porter);
}

foreach {pfrom pto} $porter_test_data {
  do_test fts1porter-$pfrom {
    execsql {
      DELETE FROM t1_term;
      DELETE FROM t1_content;