SQLite

 * offset to handle some variance.  So the estimate would be
 * (iPosition*w->iStartOffset/w->iPosition-64), which is delta-encoded
 * as normal.  Offsets more than 64 chars from the estimate are
 * encoded as the delta to the previous start offset + 128.  An
 * additional tiny increment can be gained by using the end offset of
 * the previous token to make the estimate a tiny bit more precise.
*/

/* It is not safe to call isspace(), tolower(), or isalnum() on
** hi-bit-set characters.  This is the same solution used in the
** tokenizer.
*/
/* TODO(shess) The snippet-generation code should be using the
** tokenizer-generated tokens rather than doing its own local
** tokenization.
*/
/* TODO(shess) Is __isascii() a portable version of (c&0x80)==0? */
static int safe_isspace(char c){
  return (c&0x80)==0 ? isspace(c) : 0;
}
static int safe_tolower(char c){
  return (c&0x80)==0 ? tolower(c) : c;
}
static int safe_isalnum(char c){
  return (c&0x80)==0 ? isalnum(c) : 0;
}

typedef enum DocListType {
  DL_DOCIDS,              /* docids only */
  DL_POSITIONS,           /* docids + positions */
  DL_POSITIONS_OFFSETS    /* docids + positions + offsets */
} DocListType;

  int i, c;
  switch( *z ){
    case 0: {
      *tokenType = TOKEN_EOF;
      return 0;
    }
    case ' ': case '\t': case '\n': case '\f': case '\r': {
      for(i=1; safe_isspace(z[i]); i++){}
      *tokenType = TOKEN_SPACE;
      return i;
    }
    case '\'':
    case '"': {
      int delim = z[0];
      for(i=1; (c=z[i])!=0; i++){

**     input:      delimiters ( '[' , ']' , '...' )
**     output:     [ ] ...
*/
static void tokenListToIdList(char **azIn){
  int i, j;
  if( azIn ){
    for(i=0, j=-1; azIn[i]; i++){
      if( safe_isalnum(azIn[i][0]) || azIn[i][1] ){
        dequoteString(azIn[i]);
        if( j>=0 ){
          azIn[j] = azIn[i];
        }
        j++;
      }
    }

** 
** Ignore leading space in *s.
**
** To put it another way, return true if the first token of
** s[] is t[].
*/
static int startsWith(const char *s, const char *t){
  while( safe_isspace(*s) ){ s++; }
  while( *t ){
    if( safe_tolower(*s++)!=safe_tolower(*t++) ) return 0;
  }
  return *s!='_' && !safe_isalnum(*s);
}

/*
** An instance of this structure defines the "spec" of a
** full text index.  This structure is populated by parseSpec
** and use by fulltextConnect and fulltextCreate.
*/

    clearTableSpec(pSpec);
    return SQLITE_NOMEM;
  }
  for(i=0; i<pSpec->nColumn; i++){
    char *p;
    pSpec->azContentColumn[i] = sqlite3_mprintf("c%d%s", i, azArg[i]);
    for (p = pSpec->azContentColumn[i]; *p ; ++p) {
      if( !safe_isalnum(*p) ) *p = '_';
    }
  }

  /*
  ** Parse the tokenizer specification string.
  */
  pSpec->azTokenizer = tokenizeString(zTokenizer, &n);

      return aMatch[i].iStart;
    }
    if( i>0 && aMatch[i-1].iStart+aMatch[i-1].nByte>=iBreak ){
      return aMatch[i-1].iStart;
    }
  }
  for(i=1; i<=10; i++){
    if( safe_isspace(zDoc[iBreak-i]) ){
      return iBreak - i + 1;
    }
    if( safe_isspace(zDoc[iBreak+i]) ){
      return iBreak + i + 1;
    }
  }
  return iBreak;
}

/*
** If the StringBuffer does not end in white space, add a single
** space character to the end.
*/
static void appendWhiteSpace(StringBuffer *p){
  if( p->len==0 ) return;
  if( safe_isspace(p->s[p->len-1]) ) return;
  append(p, " ");
}

/*
** Remove white space from teh end of the StringBuffer
*/
static void trimWhiteSpace(StringBuffer *p){
  while( p->len>0 && safe_isspace(p->s[p->len-1]) ){
    p->len--;
  }
}



/*

*/

#if 0
# define TRACE(A)  printf A; fflush(stdout)
#else
# define TRACE(A)
#endif

/* It is not safe to call isspace(), tolower(), or isalnum() on
** hi-bit-set characters.  This is the same solution used in the
** tokenizer.
*/
/* TODO(shess) The snippet-generation code should be using the
** tokenizer-generated tokens rather than doing its own local
** tokenization.
*/
/* TODO(shess) Is __isascii() a portable version of (c&0x80)==0? */
static int safe_isspace(char c){
  return (c&0x80)==0 ? isspace(c) : 0;
}
static int safe_tolower(char c){
  return (c&0x80)==0 ? tolower(c) : c;
}
static int safe_isalnum(char c){
  return (c&0x80)==0 ? isalnum(c) : 0;
}

typedef enum DocListType {
  DL_DOCIDS,              /* docids only */
  DL_POSITIONS,           /* docids + positions */
  DL_POSITIONS_OFFSETS    /* docids + positions + offsets */
} DocListType;

    if( i>0 ) append(sb, ", ");
    append(sb, azString[i]);
  }
}

static int endsInWhiteSpace(StringBuffer *p){
  return stringBufferLength(p)>0 &&
    safe_isspace(stringBufferData(p)[stringBufferLength(p)-1]);
}

/* If the StringBuffer ends in something other than white space, add a
** single space character to the end.
*/
static void appendWhiteSpace(StringBuffer *p){
  if( stringBufferLength(p)==0 ) return;

  int i, c;
  switch( *z ){
    case 0: {
      *tokenType = TOKEN_EOF;
      return 0;
    }
    case ' ': case '\t': case '\n': case '\f': case '\r': {
      for(i=1; safe_isspace(z[i]); i++){}
      *tokenType = TOKEN_SPACE;
      return i;
    }
    case '\'':
    case '"': {
      int delim = z[0];
      for(i=1; (c=z[i])!=0; i++){

**     input:      delimiters ( '[' , ']' , '...' )
**     output:     [ ] ...
*/
static void tokenListToIdList(char **azIn){
  int i, j;
  if( azIn ){
    for(i=0, j=-1; azIn[i]; i++){
      if( safe_isalnum(azIn[i][0]) || azIn[i][1] ){
        dequoteString(azIn[i]);
        if( j>=0 ){
          azIn[j] = azIn[i];
        }
        j++;
      }
    }

** 
** Ignore leading space in *s.
**
** To put it another way, return true if the first token of
** s[] is t[].
*/
static int startsWith(const char *s, const char *t){
  while( safe_isspace(*s) ){ s++; }
  while( *t ){
    if( safe_tolower(*s++)!=safe_tolower(*t++) ) return 0;
  }
  return *s!='_' && !safe_isalnum(*s);
}

/*
** An instance of this structure defines the "spec" of a
** full text index.  This structure is populated by parseSpec
** and use by fulltextConnect and fulltextCreate.
*/

    clearTableSpec(pSpec);
    return SQLITE_NOMEM;
  }
  for(i=0; i<pSpec->nColumn; i++){
    char *p;
    pSpec->azContentColumn[i] = sqlite3_mprintf("c%d%s", i, azArg[i]);
    for (p = pSpec->azContentColumn[i]; *p ; ++p) {
      if( !safe_isalnum(*p) ) *p = '_';
    }
  }

  /*
  ** Parse the tokenizer specification string.
  */
  pSpec->azTokenizer = tokenizeString(zTokenizer, &n);

      return aMatch[i].iStart;
    }
    if( i>0 && aMatch[i-1].iStart+aMatch[i-1].nByte>=iBreak ){
      return aMatch[i-1].iStart;
    }
  }
  for(i=1; i<=10; i++){
    if( safe_isspace(zDoc[iBreak-i]) ){
      return iBreak - i + 1;
    }
    if( safe_isspace(zDoc[iBreak+i]) ){
      return iBreak + i + 1;
    }
  }
  return iBreak;
}

# 2007 March 28
#
# The author disclaims copyright to this source code.
#
#*************************************************************************
# This file implements regression tests for SQLite library.  The focus
# of this script is testing isspace/isalnum/tolower problems with the
# FTS1 module.  Unfortunately, this code isn't a really principled set
# of tests, because it's impossible to know where new uses of these
# functions might appear.
#
# $Id: fts1k.test,v 1.1 2007/03/29 16:30:41 shess Exp $
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# If SQLITE_ENABLE_FTS1 is defined, omit this file.
ifcapable !fts1 {
  finish_test
  return
}

# Tests that startsWith() (calls isspace, tolower, isalnum) can handle
# hi-bit chars.  parseSpec() also calls isalnum here.
do_test fts1k-1.1 {
  execsql "CREATE VIRTUAL TABLE t1 USING fts1(content, \x80)"
} {}

# Additionally tests isspace() call in getToken(), and isalnum() call
# in tokenListToIdList().
do_test fts1k-1.2 {
  catch {
    execsql "CREATE VIRTUAL TABLE t2 USING fts1(content, tokenize \x80)"
  }
  sqlite3_errmsg $DB
} "unknown tokenizer: \x80"

# Additionally test final isalnum() in startsWith().
do_test fts1k-1.3 {
  execsql "CREATE VIRTUAL TABLE t3 USING fts1(content, tokenize\x80)"
} {}

# The snippet-generation code has calls to isspace() which are sort of
# hard to get to.  It finds convenient breakpoints by starting ~40
# chars before and after the matched term, and scanning ~10 chars
# around that position for isspace() characters.  The long word with
# embedded hi-bit chars causes one of these isspace() calls to be
# exercised.  The version with a couple extra spaces should cause the
# other isspace() call to be exercised.  [Both cases have been tested
# in the debugger, but I'm hoping to continue to catch it if simple
# constant changes change things slightly.
#
# The trailing and leading hi-bit chars help with code which tests for
# isspace() to coalesce multiple spaces.

set word "\x80xxxxx\x80xxxxx\x80xxxxx\x80xxxxx\x80xxxxx\x80xxxxx\x80"
set phrase1 "$word $word $word target $word $word $word"
set phrase2 "$word $word $word    target    $word $word $word"

db eval {CREATE VIRTUAL TABLE t4 USING fts1(content)}
db eval "INSERT INTO t4 (content) VALUES ('$phrase1')"
db eval "INSERT INTO t4 (content) VALUES ('$phrase2')"

do_test fts1k-1.4 {
  execsql {SELECT rowid, length(snippet(t4)) FROM t4 WHERE t4 MATCH 'target'}
} {1 111 2 117}

finish_test

# 2007 March 28
#
# The author disclaims copyright to this source code.
#
#*************************************************************************
# This file implements regression tests for SQLite library.  The focus
# of this script is testing isspace/isalnum/tolower problems with the
# FTS2 module.  Unfortunately, this code isn't a really principled set
# of tests, because it's impossible to know where new uses of these
# functions might appear.
#
# $Id: fts2l.test,v 1.1 2007/03/29 16:30:41 shess Exp $
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# If SQLITE_ENABLE_FTS2 is defined, omit this file.
ifcapable !fts2 {
  finish_test
  return
}

# Tests that startsWith() (calls isspace, tolower, isalnum) can handle
# hi-bit chars.  parseSpec() also calls isalnum here.
do_test fts2l-1.1 {
  execsql "CREATE VIRTUAL TABLE t1 USING fts2(content, \x80)"
} {}

# Additionally tests isspace() call in getToken(), and isalnum() call
# in tokenListToIdList().
do_test fts2l-1.2 {
  catch {
    execsql "CREATE VIRTUAL TABLE t2 USING fts2(content, tokenize \x80)"
  }
  sqlite3_errmsg $DB
} "unknown tokenizer: \x80"

# Additionally test final isalnum() in startsWith().
do_test fts2l-1.3 {
  execsql "CREATE VIRTUAL TABLE t3 USING fts2(content, tokenize\x80)"
} {}

# The snippet-generation code has calls to isspace() which are sort of
# hard to get to.  It finds convenient breakpoints by starting ~40
# chars before and after the matched term, and scanning ~10 chars
# around that position for isspace() characters.  The long word with
# embedded hi-bit chars causes one of these isspace() calls to be
# exercised.  The version with a couple extra spaces should cause the
# other isspace() call to be exercised.  [Both cases have been tested
# in the debugger, but I'm hoping to continue to catch it if simple
# constant changes change things slightly.
#
# The trailing and leading hi-bit chars help with code which tests for
# isspace() to coalesce multiple spaces.

set word "\x80xxxxx\x80xxxxx\x80xxxxx\x80xxxxx\x80xxxxx\x80xxxxx\x80"
set phrase1 "$word $word $word target $word $word $word"
set phrase2 "$word $word $word    target    $word $word $word"

db eval {CREATE VIRTUAL TABLE t4 USING fts2(content)}
db eval "INSERT INTO t4 (content) VALUES ('$phrase1')"
db eval "INSERT INTO t4 (content) VALUES ('$phrase2')"

do_test fts2l-1.4 {
  execsql {SELECT rowid, length(snippet(t4)) FROM t4 WHERE t4 MATCH 'target'}
} {1 111 2 117}

finish_test