# 2012 May 25 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #************************************************************************* # # The tests in this file focus on testing the "unicode" FTS tokenizer. # set testdir [file dirname $argv0] source $testdir/tester.tcl ifcapable !fts3_unicode { finish_test ; return } set ::testprefix fts4unicode proc do_unicode_token_test {tn input res} { set input [string map {' ''} $input] uplevel [list do_execsql_test $tn " SELECT fts3_tokenizer_test('unicode61', 'remove_diacritics=0', '$input'); " [list [list {*}$res]]] } proc do_unicode_token_test2 {tn input res} { set input [string map {' ''} $input] uplevel [list do_execsql_test $tn " SELECT fts3_tokenizer_test('unicode61', '$input'); " [list [list {*}$res]]] } proc do_unicode_token_test3 {tn args} { set res [lindex $args end] set sql "SELECT fts3_tokenizer_test('unicode61'" foreach a [lrange $args 0 end-1] { append sql ", '" append sql [string map {' ''} $a] append sql "'" } append sql ")" uplevel [list do_execsql_test $tn $sql [list [list {*}$res]]] } do_unicode_token_test 1.0 {a B c D} {0 a a 1 b B 2 c c 3 d D} do_unicode_token_test 1.1 {Ä Ö Ü} {0 ä Ä 1 ö Ö 2 ü Ü} do_unicode_token_test 1.2 {xÄx xÖx xÜx} {0 xäx xÄx 1 xöx xÖx 2 xüx xÜx} # 0x00DF is a small "sharp s". 0x1E9E is a capital sharp s. do_unicode_token_test 1.3 "\uDF" "0 \uDF \uDF" do_unicode_token_test 1.4 "\u1E9E" "0 ß \u1E9E" do_unicode_token_test 1.5 "\u1E9E" "0 \uDF \u1E9E" do_unicode_token_test 1.6 "The quick brown fox" { 0 the The 1 quick quick 2 brown brown 3 fox fox } do_unicode_token_test 1.7 "The\u00bfquick\u224ebrown\u2263fox" { 0 the The 1 quick quick 2 brown brown 3 fox fox } do_unicode_token_test2 1.8 {a B c D} {0 a a 1 b B 2 c c 3 d D} do_unicode_token_test2 1.9 {Ä Ö Ü} {0 a Ä 1 o Ö 2 u Ü} do_unicode_token_test2 1.10 {xÄx xÖx xÜx} {0 xax xÄx 1 xox xÖx 2 xux xÜx} # Check that diacritics are removed if remove_diacritics=1 is specified. # And that they do not break tokens. do_unicode_token_test2 1.11 "xx\u0301xx" "0 xxxx xx\u301xx" # Title-case mappings work do_unicode_token_test 1.12 "\u01c5" "0 \u01c6 \u01c5" #------------------------------------------------------------------------- # set docs [list { Enhance the INSERT syntax to allow multiple rows to be inserted via the VALUES clause. } { Enhance the CREATE VIRTUAL TABLE command to support the IF NOT EXISTS clause. } { Added the sqlite3_stricmp() interface as a counterpart to sqlite3_strnicmp(). } { Added the sqlite3_db_readonly() interface. } { Added the SQLITE_FCNTL_PRAGMA file control, giving VFS implementations the ability to add new PRAGMA statements or to override built-in PRAGMAs. } { Queries of the form: "SELECT max(x), y FROM table" returns the value of y on the same row that contains the maximum x value. } { Added support for the FTS4 languageid option. } { Documented support for the FTS4 content option. This feature has actually been in the code since version 3.7.9 but is only now considered to be officially supported. } { Pending statements no longer block ROLLBACK. Instead, the pending statement will return SQLITE_ABORT upon next access after the ROLLBACK. } { Improvements to the handling of CSV inputs in the command-line shell } { Fix a bug introduced in version 3.7.10 that might cause a LEFT JOIN to be incorrectly converted into an INNER JOIN if the WHERE clause indexable terms connected by OR. }] set map(a) [list "\u00C4" "\u00E4"] ; # LATIN LETTER A WITH DIAERESIS set map(e) [list "\u00CB" "\u00EB"] ; # LATIN LETTER E WITH DIAERESIS set map(i) [list "\u00CF" "\u00EF"] ; # LATIN LETTER I WITH DIAERESIS set map(o) [list "\u00D6" "\u00F6"] ; # LATIN LETTER O WITH DIAERESIS set map(u) [list "\u00DC" "\u00FC"] ; # LATIN LETTER U WITH DIAERESIS set map(y) [list "\u0178" "\u00FF"] ; # LATIN LETTER Y WITH DIAERESIS set map(h) [list "\u1E26" "\u1E27"] ; # LATIN LETTER H WITH DIAERESIS set map(w) [list "\u1E84" "\u1E85"] ; # LATIN LETTER W WITH DIAERESIS set map(x) [list "\u1E8C" "\u1E8D"] ; # LATIN LETTER X WITH DIAERESIS foreach k [array names map] { lappend mappings [string toupper $k] [lindex $map($k) 0] lappend mappings $k [lindex $map($k) 1] } proc mapdoc {doc} { set doc [regsub -all {[[:space:]]+} $doc " "] string map $::mappings [string trim $doc] } do_test 2.0 { execsql { CREATE VIRTUAL TABLE t2 USING fts4(tokenize=unicode61, x); } foreach doc $docs { set d [mapdoc $doc] execsql { INSERT INTO t2 VALUES($d) } } } {} do_test 2.1 { set q [mapdoc "row"] execsql { SELECT * FROM t2 WHERE t2 MATCH $q } } [list [mapdoc { Queries of the form: "SELECT max(x), y FROM table" returns the value of y on the same row that contains the maximum x value. }]] foreach {tn query snippet} { 2 "row" { ...returns the value of y on the same [row] that contains the maximum x value. } 3 "ROW" { ...returns the value of y on the same [row] that contains the maximum x value. } 4 "rollback" { ...[ROLLBACK]. Instead, the pending statement will return SQLITE_ABORT upon next access after the [ROLLBACK]. } 5 "rOllback" { ...[ROLLBACK]. Instead, the pending statement will return SQLITE_ABORT upon next access after the [ROLLBACK]. } 6 "lang*" { Added support for the FTS4 [languageid] option. } } { do_test 2.$tn { set q [mapdoc $query] execsql { SELECT snippet(t2, '[', ']', '...') FROM t2 WHERE t2 MATCH $q } } [list [mapdoc $snippet]] } #------------------------------------------------------------------------- # Make sure the unicode61 tokenizer does not crash if it is passed a # NULL pointer. reset_db do_execsql_test 3.1 { CREATE VIRTUAL TABLE t1 USING fts4(tokenize=unicode61, x, y); INSERT INTO t1 VALUES(NULL, 'a b c'); } do_execsql_test 3.2 { SELECT snippet(t1, '[', ']') FROM t1 WHERE t1 MATCH 'b' } {{a [b] c}} do_execsql_test 3.3 { BEGIN; DELETE FROM t1; INSERT INTO t1 VALUES('b b b b b b b b b b b', 'b b b b b b b b b b b b b'); INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 VALUES('a b c', NULL); INSERT INTO t1 VALUES('a x c', NULL); COMMIT; } do_execsql_test 3.4 { SELECT * FROM t1 WHERE t1 MATCH 'a b'; } {{a b c} {}} #------------------------------------------------------------------------- # reset_db do_test 4.1 { set a "abc\uFFFEdef" set b "abc\uD800def" set c "\uFFFEdef" set d "\uD800def" execsql { CREATE VIRTUAL TABLE t1 USING fts4(tokenize=unicode61, x); INSERT INTO t1 VALUES($a); INSERT INTO t1 VALUES($b); INSERT INTO t1 VALUES($c); INSERT INTO t1 VALUES($d); } } {} do_test 4.2 { set a [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0x62}] set b [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0xBF 0x62}] set c [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0xBF 0xBF 0x62}] set d [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0xBF 0xBF 0xBF 0x62}] execsql { INSERT INTO t1 VALUES($a); INSERT INTO t1 VALUES($b); INSERT INTO t1 VALUES($c); INSERT INTO t1 VALUES($d); } } {} do_test 4.3 { set a [binary format c* {0xF7 0xBF 0xBF 0xBF}] set b [binary format c* {0xF7 0xBF 0xBF 0xBF 0xBF}] set c [binary format c* {0xF7 0xBF 0xBF 0xBF 0xBF 0xBF}] set d [binary format c* {0xF7 0xBF 0xBF 0xBF 0xBF 0xBF 0xBF}] execsql { INSERT INTO t1 VALUES($a); INSERT INTO t1 VALUES($b); INSERT INTO t1 VALUES($c); INSERT INTO t1 VALUES($d); } } {} #------------------------------------------------------------------------- do_unicode_token_test3 5.1 {tokenchars=} { sqlite3_reset sqlite3_column_int } { 0 sqlite3 sqlite3 1 reset reset 2 sqlite3 sqlite3 3 column column 4 int int } do_unicode_token_test3 5.2 {tokenchars=_} { sqlite3_reset sqlite3_column_int } { 0 sqlite3_reset sqlite3_reset 1 sqlite3_column_int sqlite3_column_int } do_unicode_token_test3 5.3 {separators=xyz} { Laotianxhorseyrunszfast } { 0 laotian Laotian 1 horse horse 2 runs runs 3 fast fast } do_unicode_token_test3 5.4 {tokenchars=xyz} { Laotianxhorseyrunszfast } { 0 laotianxhorseyrunszfast Laotianxhorseyrunszfast } do_unicode_token_test3 5.5 {tokenchars=_} {separators=zyx} { sqlite3_resetxsqlite3_column_intyhonda_phantom } { 0 sqlite3_reset sqlite3_reset 1 sqlite3_column_int sqlite3_column_int 2 honda_phantom honda_phantom } do_unicode_token_test3 5.6 "separators=\u05D1" "abc\u05D1def" { 0 abc abc 1 def def } do_unicode_token_test3 5.7 \ "tokenchars=\u2444\u2445" \ "separators=\u05D0\u05D1\u05D2" \ "\u2444fre\u2445sh\u05D0water\u05D2fish.\u2445timer" \ [list \ 0 \u2444fre\u2445sh \u2444fre\u2445sh \ 1 water water \ 2 fish fish \ 3 \u2445timer \u2445timer \ ] # Check that it is not possible to add a standalone diacritic codepoint # to either separators or tokenchars. do_unicode_token_test3 5.8 "separators=\u0301" \ "hello\u0301world \u0301helloworld" \ "0 helloworld hello\u0301world 1 helloworld helloworld" do_unicode_token_test3 5.9 "tokenchars=\u0301" \ "hello\u0301world \u0301helloworld" \ "0 helloworld hello\u0301world 1 helloworld helloworld" do_unicode_token_test3 5.10 "separators=\u0301" \ "remove_diacritics=0" \ "hello\u0301world \u0301helloworld" \ "0 hello\u0301world hello\u0301world 1 helloworld helloworld" do_unicode_token_test3 5.11 "tokenchars=\u0301" \ "remove_diacritics=0" \ "hello\u0301world \u0301helloworld" \ "0 hello\u0301world hello\u0301world 1 helloworld helloworld" #------------------------------------------------------------------------- proc do_tokenize {tokenizer txt} { set res [list] foreach {a b c} [db one {SELECT fts3_tokenizer_test($tokenizer, $txt)}] { lappend res $b } set res } # Argument $lCodepoint must be a list of codepoints (integers) that # correspond to whitespace characters. This command creates a string # $W from the codepoints, then tokenizes "${W}hello{$W}world${W}" # using tokenizer $tokenizer. The test passes if the tokenizer successfully # extracts the two 5 character tokens. # proc do_isspace_test {tn tokenizer lCp} { set whitespace [format [string repeat %c [llength $lCp]] {*}$lCp] set txt "${whitespace}hello${whitespace}world${whitespace}" uplevel [list do_test $tn [list do_tokenize $tokenizer $txt] {hello world}] } set tokenizers [list unicode61] ifcapable icu { lappend tokenizers icu } # Some tests to check that the tokenizers can both identify white-space # codepoints. All codepoints tested below are of type "Zs" in the # UnicodeData.txt file. foreach T $tokenizers { do_isspace_test 6.$T.1 $T 32 do_isspace_test 6.$T.2 $T 160 do_isspace_test 6.$T.3 $T 5760 do_isspace_test 6.$T.4 $T 6158 do_isspace_test 6.$T.5 $T 8192 do_isspace_test 6.$T.6 $T 8193 do_isspace_test 6.$T.7 $T 8194 do_isspace_test 6.$T.8 $T 8195 do_isspace_test 6.$T.9 $T 8196 do_isspace_test 6.$T.10 $T 8197 do_isspace_test 6.$T.11 $T 8198 do_isspace_test 6.$T.12 $T 8199 do_isspace_test 6.$T.13 $T 8200 do_isspace_test 6.$T.14 $T 8201 do_isspace_test 6.$T.15 $T 8202 do_isspace_test 6.$T.16 $T 8239 do_isspace_test 6.$T.17 $T 8287 do_isspace_test 6.$T.18 $T 12288 do_isspace_test 6.$T.19 $T {32 160 5760 6158} do_isspace_test 6.$T.19 $T {8192 8193 8194 8195} do_isspace_test 6.$T.19 $T {8196 8197 8198 8199} do_isspace_test 6.$T.19 $T {8200 8201 8202 8239} do_isspace_test 6.$T.19 $T {8287 12288} } #------------------------------------------------------------------------- # Test that the private use ranges are treated as alphanumeric. # breakpoint foreach {tn1 c} { 1 \ue000 2 \ue001 3 \uf000 4 \uf8fe 5 \uf8ff } { foreach {tn2 config res} { 1 "" "0 hello*world hello*world" 2 "separators=*" "0 hello hello 1 world world" } { set config [string map [list * $c] $config] set input [string map [list * $c] "hello*world"] set output [string map [list * $c] $res] do_unicode_token_test3 7.$tn1.$tn2 {*}$config $input $output } } finish_test