Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
| Comment: | Fix a problem with identifying white-space characters outside of the ascii range in the ICU tokenizer. |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | trunk |
| Files: | files | file ages | folders |
| SHA1: |
892b74116a3b23268895b96433d18ef0 |
| User & Date: | dan 2012-06-18 20:52:32 |
Context
|
2012-06-19
| ||
| 00:45 | Improved rounding accuracy on text-to-float conversions. (check-in: 699b792c user: drh tags: trunk) | |
|
2012-06-18
| ||
| 20:52 | Fix a problem with identifying white-space characters outside of the ascii range in the ICU tokenizer. (check-in: 892b7411 user: dan tags: trunk) | |
|
2012-06-16
| ||
| 15:26 | In Lemon, when comparing the output to the *.h file to see if it has changed, make sure that the proposed new output and the preexisting output are the same size before deciding that they are the same. (check-in: 0c2fb18d user: drh tags: trunk) | |
Changes
Changes to ext/fts3/fts3_icu.c.
195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 |
iEnd = ubrk_next(pCsr->pIter);
if( iEnd==UBRK_DONE ){
return SQLITE_DONE;
}
while( iStart<iEnd ){
int iWhite = iStart;
U8_NEXT(pCsr->aChar, iWhite, pCsr->nChar, c);
if( u_isspace(c) ){
iStart = iWhite;
}else{
break;
}
}
assert(iStart<=iEnd);
|
| |
195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 |
iEnd = ubrk_next(pCsr->pIter);
if( iEnd==UBRK_DONE ){
return SQLITE_DONE;
}
while( iStart<iEnd ){
int iWhite = iStart;
U16_NEXT(pCsr->aChar, iWhite, pCsr->nChar, c);
if( u_isspace(c) ){
iStart = iWhite;
}else{
break;
}
}
assert(iStart<=iEnd);
|
Changes to test/fts3atoken.test.
170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 |
do_execsql_test 5.1 {
CREATE VIRTUAL TABLE x1 USING fts3(name,TOKENIZE icu en_US);
insert into x1 (name) values (NULL);
insert into x1 (name) values (NULL);
delete from x1;
}
}
do_test fts3token-internal {
execsql { SELECT fts3_tokenizer_internal_test() }
} {ok}
finish_test
|
> > > > > > > > > > |
170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 |
do_execsql_test 5.1 {
CREATE VIRTUAL TABLE x1 USING fts3(name,TOKENIZE icu en_US);
insert into x1 (name) values (NULL);
insert into x1 (name) values (NULL);
delete from x1;
}
proc cp_to_str {codepoint_list} {
set fmt [string repeat %c [llength $codepoint_list]]
eval [list format $fmt] $codepoint_list
}
do_test 5.2 {
set str [cp_to_str {19968 26085 32822 32645 27874 23433 20986}]
execsql { INSERT INTO x1 VALUES($str) }
} {}
}
do_test fts3token-internal {
execsql { SELECT fts3_tokenizer_internal_test() }
} {ok}
finish_test
|