/ Check-in [0dd09fc0]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Avoid passing NULL to the xOpen method of an FTS3/4 tokenizer.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1:0dd09fc034c127718366d3a3183e367d2f9fd82d
User & Date: dan 2011-04-19 06:43:22
Context
2011-04-20
13:35
Update a comment in e_createtable.test. check-in: d8b149f5 user: dan tags: trunk
2011-04-19
06:43
Avoid passing NULL to the xOpen method of an FTS3/4 tokenizer. check-in: 0dd09fc0 user: dan tags: trunk
2011-04-17
17:09
Change the prototype for the open() system call to agree with Posix. Though a faulty function prototype in a pointer cast is a seemingly innocuous error, the correct prototype is necessary for pthreads to work correctly on NetBSD. check-in: 3e135748 user: drh tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts3/fts3_write.c.

537
538
539
540
541
542
543








544
545
546
547
548
549
550
...
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
  sqlite3_tokenizer *pTokenizer = p->pTokenizer;
  sqlite3_tokenizer_module const *pModule = pTokenizer->pModule;
  sqlite3_tokenizer_cursor *pCsr;
  int (*xNext)(sqlite3_tokenizer_cursor *pCursor,
      const char**,int*,int*,int*,int*);

  assert( pTokenizer && pModule );









  rc = pModule->xOpen(pTokenizer, zText, -1, &pCsr);
  if( rc!=SQLITE_OK ){
    return rc;
  }
  pCsr->pTokenizer = pTokenizer;

................................................................................
** Argument apVal is the same as the similarly named argument passed to
** fts3InsertData(). Parameter iDocid is the docid of the new row.
*/
static int fts3InsertTerms(Fts3Table *p, sqlite3_value **apVal, u32 *aSz){
  int i;                          /* Iterator variable */
  for(i=2; i<p->nColumn+2; i++){
    const char *zText = (const char *)sqlite3_value_text(apVal[i]);
    if( zText ){
      int rc = fts3PendingTermsAdd(p, zText, i-2, &aSz[i-2]);
      if( rc!=SQLITE_OK ){
        return rc;
      }
    }
    aSz[p->nColumn] += sqlite3_value_bytes(apVal[i]);
  }
  return SQLITE_OK;
}

/*







>
>
>
>
>
>
>
>







 







<
|
|
|
<







537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
...
636
637
638
639
640
641
642

643
644
645

646
647
648
649
650
651
652
  sqlite3_tokenizer *pTokenizer = p->pTokenizer;
  sqlite3_tokenizer_module const *pModule = pTokenizer->pModule;
  sqlite3_tokenizer_cursor *pCsr;
  int (*xNext)(sqlite3_tokenizer_cursor *pCursor,
      const char**,int*,int*,int*,int*);

  assert( pTokenizer && pModule );

  /* If the user has inserted a NULL value, this function may be called with
  ** zText==0. In this case, add zero token entries to the hash table and 
  ** return early. */
  if( zText==0 ){
    *pnWord = 0;
    return SQLITE_OK;
  }

  rc = pModule->xOpen(pTokenizer, zText, -1, &pCsr);
  if( rc!=SQLITE_OK ){
    return rc;
  }
  pCsr->pTokenizer = pTokenizer;

................................................................................
** Argument apVal is the same as the similarly named argument passed to
** fts3InsertData(). Parameter iDocid is the docid of the new row.
*/
static int fts3InsertTerms(Fts3Table *p, sqlite3_value **apVal, u32 *aSz){
  int i;                          /* Iterator variable */
  for(i=2; i<p->nColumn+2; i++){
    const char *zText = (const char *)sqlite3_value_text(apVal[i]);

    int rc = fts3PendingTermsAdd(p, zText, i-2, &aSz[i-2]);
    if( rc!=SQLITE_OK ){
      return rc;

    }
    aSz[p->nColumn] += sqlite3_value_bytes(apVal[i]);
  }
  return SQLITE_OK;
}

/*

Changes to test/fts3atoken.test.

20
21
22
23
24
25
26


27
28
29
30
31
32
33
...
162
163
164
165
166
167
168







169
170
171
172
173

174



# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts3 {
  finish_test
  return
}



proc escape_string {str} {
  set out ""
  foreach char [split $str ""] {
    scan $char %c i
    if {$i<=127} {
      append out $char
    } else {
................................................................................
  append output "2 then then "
  append output "3 [string tolower $longtoken] $longtoken"

  do_icu_test fts3token-4.6 MiddleOfTheOcean  $input $output
  do_icu_test fts3token-4.7 th_TH  $input $output
  do_icu_test fts3token-4.8 en_US  $input $output
}








do_test fts3token-internal {
  execsql { SELECT fts3_tokenizer_internal_test() }
} {ok}


finish_test









>
>







 







>
>
>
>
>
>
>





>

>
>
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
...
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186

# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts3 {
  finish_test
  return
}

set ::testprefix fts3token

proc escape_string {str} {
  set out ""
  foreach char [split $str ""] {
    scan $char %c i
    if {$i<=127} {
      append out $char
    } else {
................................................................................
  append output "2 then then "
  append output "3 [string tolower $longtoken] $longtoken"

  do_icu_test fts3token-4.6 MiddleOfTheOcean  $input $output
  do_icu_test fts3token-4.7 th_TH  $input $output
  do_icu_test fts3token-4.8 en_US  $input $output
}

do_execsql_test 5.1 {
  CREATE VIRTUAL TABLE x1 USING fts3(name,TOKENIZE icu en_US);
  insert into x1 (name) values (NULL);
  insert into x1 (name) values (NULL);
  delete from x1;
}

do_test fts3token-internal {
  execsql { SELECT fts3_tokenizer_internal_test() }
} {ok}


finish_test