/ Check-in [892b7411]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix a problem with identifying white-space characters outside of the ascii range in the ICU tokenizer.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1:892b74116a3b23268895b96433d18ef00c1433d8
User & Date: dan 2012-06-18 20:52:32
Context
2012-06-19
00:45
Improved rounding accuracy on text-to-float conversions. check-in: 699b792c user: drh tags: trunk
2012-06-18
20:52
Fix a problem with identifying white-space characters outside of the ascii range in the ICU tokenizer. check-in: 892b7411 user: dan tags: trunk
2012-06-16
15:26
In Lemon, when comparing the output to the *.h file to see if it has changed, make sure that the proposed new output and the preexisting output are the same size before deciding that they are the same. check-in: 0c2fb18d user: drh tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts3/fts3_icu.c.

195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
    iEnd = ubrk_next(pCsr->pIter);
    if( iEnd==UBRK_DONE ){
      return SQLITE_DONE;
    }

    while( iStart<iEnd ){
      int iWhite = iStart;
      U8_NEXT(pCsr->aChar, iWhite, pCsr->nChar, c);
      if( u_isspace(c) ){
        iStart = iWhite;
      }else{
        break;
      }
    }
    assert(iStart<=iEnd);







|







195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
    iEnd = ubrk_next(pCsr->pIter);
    if( iEnd==UBRK_DONE ){
      return SQLITE_DONE;
    }

    while( iStart<iEnd ){
      int iWhite = iStart;
      U16_NEXT(pCsr->aChar, iWhite, pCsr->nChar, c);
      if( u_isspace(c) ){
        iStart = iWhite;
      }else{
        break;
      }
    }
    assert(iStart<=iEnd);

Changes to test/fts3atoken.test.

170
171
172
173
174
175
176










177
178
179
180
181
182
183
184
185
186
187

  do_execsql_test 5.1 {
    CREATE VIRTUAL TABLE x1 USING fts3(name,TOKENIZE icu en_US);
    insert into x1 (name) values (NULL);
    insert into x1 (name) values (NULL);
    delete from x1;
  }










}


do_test fts3token-internal {
  execsql { SELECT fts3_tokenizer_internal_test() }
} {ok}


finish_test









>
>
>
>
>
>
>
>
>
>











170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197

  do_execsql_test 5.1 {
    CREATE VIRTUAL TABLE x1 USING fts3(name,TOKENIZE icu en_US);
    insert into x1 (name) values (NULL);
    insert into x1 (name) values (NULL);
    delete from x1;
  }

  proc cp_to_str {codepoint_list} {
    set fmt [string repeat %c [llength $codepoint_list]]
    eval [list format $fmt] $codepoint_list
  }

  do_test 5.2 {
    set str [cp_to_str {19968 26085 32822 32645 27874 23433 20986}]
    execsql { INSERT INTO x1 VALUES($str) }
  } {}
}


do_test fts3token-internal {
  execsql { SELECT fts3_tokenizer_internal_test() }
} {ok}


finish_test