/ Check-in [b8b465ed]
Login
Overview
Comment:Remove all vestiges of ctype.h from FTS3.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1:b8b465ed2c56b23b50334680a2a940885b1ac530
User & Date: drh 2010-08-06 19:00:12
Original Comment: Remove all vestiges of ctype.h from FTS3.
Context
2010-08-09
14:26
Fix two comment typos that were reported on the mailing list. check-in: 88bf9016 user: drh tags: trunk
2010-08-06
19:00
Remove all vestiges of ctype.h from FTS3. This addresses ticket [991789d9f3136a] among other bug reports. check-in: b8b465ed user: drh tags: trunk
13:50
Do not run some tests in notify3.test with the inmemory_journal permutation. They do not pass as the tests assume that the database schema is not loaded until the first invocation of [db eval]. This is not true with the inmemory_journal permutation. check-in: 698fba82 user: dan tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts3/fts3_expr.c.

74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
...
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114

/*
** Default span for NEAR operators.
*/
#define SQLITE_FTS3_DEFAULT_NEAR_PARAM 10

#include "fts3Int.h"
#include <ctype.h>
#include <string.h>
#include <assert.h>

typedef struct ParseContext ParseContext;
struct ParseContext {
  sqlite3_tokenizer *pTokenizer;      /* Tokenizer module */
  const char **azCol;                 /* Array of column names for fts3 table */
................................................................................
** an integer that falls outside of the range of the unsigned char type
** is undefined (and sometimes, "undefined" means segfault). This wrapper
** is defined to accept an argument of type char, and always returns 0 for
** any values that fall outside of the range of the unsigned char type (i.e.
** negative values).
*/
static int fts3isspace(char c){
  return (c&0x80)==0 ? isspace(c) : 0;
}

/*
** Extract the next token from buffer z (length n) using the tokenizer
** and other information (column names etc.) in pParse. Create an Fts3Expr
** structure of type FTSQUERY_PHRASE containing a phrase consisting of this
** single token and set *ppExpr to point to it. If the end of the buffer is







<







 







|







74
75
76
77
78
79
80

81
82
83
84
85
86
87
..
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113

/*
** Default span for NEAR operators.
*/
#define SQLITE_FTS3_DEFAULT_NEAR_PARAM 10

#include "fts3Int.h"

#include <string.h>
#include <assert.h>

typedef struct ParseContext ParseContext;
struct ParseContext {
  sqlite3_tokenizer *pTokenizer;      /* Tokenizer module */
  const char **azCol;                 /* Array of column names for fts3 table */
................................................................................
** an integer that falls outside of the range of the unsigned char type
** is undefined (and sometimes, "undefined" means segfault). This wrapper
** is defined to accept an argument of type char, and always returns 0 for
** any values that fall outside of the range of the unsigned char type (i.e.
** negative values).
*/
static int fts3isspace(char c){
  return c==' ' || c=='\t' || c=='\n' || c=='\r' || c=='\v' || c=='\f';
}

/*
** Extract the next token from buffer z (length n) using the tokenizer
** and other information (column names etc.) in pParse. Create an Fts3Expr
** structure of type FTSQUERY_PHRASE containing a phrase consisting of this
** single token and set *ppExpr to point to it. If the end of the buffer is

Changes to ext/fts3/fts3_porter.c.

26
27
28
29
30
31
32
33
34
35
36
37
38
39
40

#include "fts3Int.h"

#include <assert.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <ctype.h>

#include "fts3_tokenizer.h"

/*
** Class derived from sqlite3_tokenizer
*/
typedef struct porter_tokenizer {







<







26
27
28
29
30
31
32

33
34
35
36
37
38
39

#include "fts3Int.h"

#include <assert.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>


#include "fts3_tokenizer.h"

/*
** Class derived from sqlite3_tokenizer
*/
typedef struct porter_tokenizer {

Changes to ext/fts3/fts3_snippet.c.

12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
*/

#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)

#include "fts3Int.h"
#include <string.h>
#include <assert.h>
#include <ctype.h>


/*
** Used as an fts3ExprIterate() context when loading phrase doclists to
** Fts3Expr.aDoclist[]/nDoclist.
*/
typedef struct LoadDoclistCtx LoadDoclistCtx;







<







12
13
14
15
16
17
18

19
20
21
22
23
24
25
*/

#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)

#include "fts3Int.h"
#include <string.h>
#include <assert.h>



/*
** Used as an fts3ExprIterate() context when loading phrase doclists to
** Fts3Expr.aDoclist[]/nDoclist.
*/
typedef struct LoadDoclistCtx LoadDoclistCtx;

Changes to ext/fts3/fts3_tokenizer.c.

28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
#include "sqlite3ext.h"
#ifndef SQLITE_CORE
  SQLITE_EXTENSION_INIT1
#endif

#include "fts3Int.h"
#include <assert.h>
#include <ctype.h>
#include <string.h>

/*
** Implementation of the SQL scalar function for accessing the underlying 
** hash table. This function may be called as follows:
**
**   SELECT <function-name>(<key-name>);







<







28
29
30
31
32
33
34

35
36
37
38
39
40
41
#include "sqlite3ext.h"
#ifndef SQLITE_CORE
  SQLITE_EXTENSION_INIT1
#endif

#include "fts3Int.h"
#include <assert.h>

#include <string.h>

/*
** Implementation of the SQL scalar function for accessing the underlying 
** hash table. This function may be called as follows:
**
**   SELECT <function-name>(<key-name>);

Changes to ext/fts3/fts3_tokenizer1.c.

26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
..
49
50
51
52
53
54
55



56
57
58
59
60
61
62
..
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
...
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203

#include "fts3Int.h"

#include <assert.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <ctype.h>

#include "fts3_tokenizer.h"

typedef struct simple_tokenizer {
  sqlite3_tokenizer base;
  char delim[128];             /* flag ASCII delimiters */
} simple_tokenizer;
................................................................................
  int nTokenAllocated;         /* space allocated to zToken buffer */
} simple_tokenizer_cursor;


static int simpleDelim(simple_tokenizer *t, unsigned char c){
  return c<0x80 && t->delim[c];
}




/*
** Create a new tokenizer instance.
*/
static int simpleCreate(
  int argc, const char * const *argv,
  sqlite3_tokenizer **ppTokenizer
................................................................................
      }
      t->delim[ch] = 1;
    }
  } else {
    /* Mark non-alphanumeric ASCII characters as delimiters */
    int i;
    for(i=1; i<0x80; i++){
      t->delim[i] = !isalnum(i) ? -1 : 0;
    }
  }

  *ppTokenizer = &t->base;
  return SQLITE_OK;
}

................................................................................
        c->pToken = pNew;
      }
      for(i=0; i<n; i++){
        /* TODO(shess) This needs expansion to handle UTF-8
        ** case-insensitivity.
        */
        unsigned char ch = p[iStartOffset+i];
        c->pToken[i] = (char)(ch<0x80 ? tolower(ch) : ch);
      }
      *ppToken = c->pToken;
      *pnBytes = n;
      *piStartOffset = iStartOffset;
      *piEndOffset = c->iOffset;
      *piPosition = c->iToken++;








<







 







>
>
>







 







|







 







|







26
27
28
29
30
31
32

33
34
35
36
37
38
39
..
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
..
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
...
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205

#include "fts3Int.h"

#include <assert.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>


#include "fts3_tokenizer.h"

typedef struct simple_tokenizer {
  sqlite3_tokenizer base;
  char delim[128];             /* flag ASCII delimiters */
} simple_tokenizer;
................................................................................
  int nTokenAllocated;         /* space allocated to zToken buffer */
} simple_tokenizer_cursor;


static int simpleDelim(simple_tokenizer *t, unsigned char c){
  return c<0x80 && t->delim[c];
}
static int fts3_isalnum(int x){
  return (x>='0' && x<='9') || (x>='A' && x<='Z') || (x>='a' && x<='z');
}

/*
** Create a new tokenizer instance.
*/
static int simpleCreate(
  int argc, const char * const *argv,
  sqlite3_tokenizer **ppTokenizer
................................................................................
      }
      t->delim[ch] = 1;
    }
  } else {
    /* Mark non-alphanumeric ASCII characters as delimiters */
    int i;
    for(i=1; i<0x80; i++){
      t->delim[i] = !fts3_isalnum(i) ? -1 : 0;
    }
  }

  *ppTokenizer = &t->base;
  return SQLITE_OK;
}

................................................................................
        c->pToken = pNew;
      }
      for(i=0; i<n; i++){
        /* TODO(shess) This needs expansion to handle UTF-8
        ** case-insensitivity.
        */
        unsigned char ch = p[iStartOffset+i];
        c->pToken[i] = (char)((ch>='A' && ch<='Z') ? ch-'A'+'a' : ch);
      }
      *ppToken = c->pToken;
      *pnBytes = n;
      *piStartOffset = iStartOffset;
      *piEndOffset = c->iOffset;
      *piPosition = c->iToken++;