SQLite

Check-in [ee52589c80]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Remove additional occurrences of <ctype.h> from FTS2.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: ee52589c80a6d37dc8c7a86a0e3b531ec4dd8d45
User & Date: drh 2010-09-17 17:10:31.000
Context
2010-09-17
19:04
Add tests for some syntax diagrams in lang_select.html. (check-in: 2254e93bd5 user: dan tags: trunk)
17:10
Remove additional occurrences of <ctype.h> from FTS2. (check-in: ee52589c80 user: drh tags: trunk)
16:01
Add new test file fts3shared.test to test the previous change. (check-in: a207f74408 user: dan tags: trunk)
Changes
Unified Diff Ignore Whitespace Patch
Changes to ext/fts2/fts2_porter.c.
︙︙
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39

#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2)


#include <assert.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <ctype.h>

#include "fts2_tokenizer.h"

/*
** Class derived from sqlite3_tokenizer
*/
typedef struct porter_tokenizer {








<








25
26
27
28
29
30
31

32
33
34
35
36
37
38

#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2)


#include <assert.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>


#include "fts2_tokenizer.h"

/*
** Class derived from sqlite3_tokenizer
*/
typedef struct porter_tokenizer {
︙︙
Changes to ext/fts2/fts2_tokenizer1.c.
︙︙
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39

#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2)


#include <assert.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <ctype.h>

#include "fts2_tokenizer.h"

typedef struct simple_tokenizer {
  sqlite3_tokenizer base;
  char delim[128];             /* flag ASCII delimiters */
} simple_tokenizer;








<








25
26
27
28
29
30
31

32
33
34
35
36
37
38

#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2)


#include <assert.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>


#include "fts2_tokenizer.h"

typedef struct simple_tokenizer {
  sqlite3_tokenizer base;
  char delim[128];             /* flag ASCII delimiters */
} simple_tokenizer;
︙︙
85
86
87
88
89
90
91
92

93
94
95
96
97
98
99

      }
      t->delim[ch] = 1;
    }
  } else {
    /* Mark non-alphanumeric ASCII characters as delimiters */
    int i;
    for(i=1; i<0x80; i++){
      t->delim[i] = !isalnum(i);

    }
  }

  *ppTokenizer = &t->base;
  return SQLITE_OK;
}









|
>








84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99

      }
      t->delim[ch] = 1;
    }
  } else {
    /* Mark non-alphanumeric ASCII characters as delimiters */
    int i;
    for(i=1; i<0x80; i++){
      t->delim[i] = !((i>='0' && i<='9') || (i>='A' && i<='Z') ||
                      (i>='a' && i<='z'));
    }
  }

  *ppTokenizer = &t->base;
  return SQLITE_OK;
}
︙︙
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201

        if( c->pToken==NULL ) return SQLITE_NOMEM;
      }
      for(i=0; i<n; i++){
        /* TODO(shess) This needs expansion to handle UTF-8
        ** case-insensitivity.
        */
        unsigned char ch = p[iStartOffset+i];
        c->pToken[i] = ch<0x80 ? tolower(ch) : ch;
      }
      *ppToken = c->pToken;
      *pnBytes = n;
      *piStartOffset = iStartOffset;
      *piEndOffset = c->iOffset;
      *piPosition = c->iToken++;









|








187
188
189
190
191
192
193
194
195
196
197
198
199
200
201

        if( c->pToken==NULL ) return SQLITE_NOMEM;
      }
      for(i=0; i<n; i++){
        /* TODO(shess) This needs expansion to handle UTF-8
        ** case-insensitivity.
        */
        unsigned char ch = p[iStartOffset+i];
        c->pToken[i] = (ch>='A' && ch<='Z') ? (ch - 'A' + 'a') : ch;
      }
      *ppToken = c->pToken;
      *pnBytes = n;
      *piStartOffset = iStartOffset;
      *piEndOffset = c->iOffset;
      *piPosition = c->iToken++;
︙︙