Index: ext/fts3/fts3.c ================================================================== --- ext/fts3/fts3.c +++ ext/fts3/fts3.c @@ -3552,11 +3552,13 @@ ** to by the argument to point to the "simple" tokenizer implementation. ** And so on. */ void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule); void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule); +#ifndef SQLITE_DISABLE_FTS3_UNICODE void sqlite3Fts3UnicodeTokenizer(sqlite3_tokenizer_module const**ppModule); +#endif #ifdef SQLITE_ENABLE_ICU void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule); #endif /* @@ -3568,16 +3570,22 @@ int sqlite3Fts3Init(sqlite3 *db){ int rc = SQLITE_OK; Fts3Hash *pHash = 0; const sqlite3_tokenizer_module *pSimple = 0; const sqlite3_tokenizer_module *pPorter = 0; +#ifndef SQLITE_DISABLE_FTS3_UNICODE const sqlite3_tokenizer_module *pUnicode = 0; +#endif #ifdef SQLITE_ENABLE_ICU const sqlite3_tokenizer_module *pIcu = 0; sqlite3Fts3IcuTokenizerModule(&pIcu); #endif + +#ifndef SQLITE_DISABLE_FTS3_UNICODE + sqlite3Fts3UnicodeTokenizer(&pUnicode); +#endif #ifdef SQLITE_TEST rc = sqlite3Fts3InitTerm(db); if( rc!=SQLITE_OK ) return rc; #endif @@ -3585,11 +3593,10 @@ rc = sqlite3Fts3InitAux(db); if( rc!=SQLITE_OK ) return rc; sqlite3Fts3SimpleTokenizerModule(&pSimple); sqlite3Fts3PorterTokenizerModule(&pPorter); - sqlite3Fts3UnicodeTokenizer(&pUnicode); /* Allocate and initialise the hash-table used to store tokenizers. */ pHash = sqlite3_malloc(sizeof(Fts3Hash)); if( !pHash ){ rc = SQLITE_NOMEM; @@ -3599,11 +3606,14 @@ /* Load the built-in tokenizers into the hash table */ if( rc==SQLITE_OK ){ if( sqlite3Fts3HashInsert(pHash, "simple", 7, (void *)pSimple) || sqlite3Fts3HashInsert(pHash, "porter", 7, (void *)pPorter) + +#ifndef SQLITE_DISABLE_FTS3_UNICODE || sqlite3Fts3HashInsert(pHash, "unicode61", 10, (void *)pUnicode) +#endif #ifdef SQLITE_ENABLE_ICU || (pIcu && sqlite3Fts3HashInsert(pHash, "icu", 4, (void *)pIcu)) #endif ){ rc = SQLITE_NOMEM; Index: ext/fts3/fts3_unicode.c ================================================================== --- ext/fts3/fts3_unicode.c +++ ext/fts3/fts3_unicode.c @@ -11,10 +11,12 @@ ****************************************************************************** ** ** Implementation of the "unicode" full-text-search tokenizer. */ +#ifndef SQLITE_DISABLE_FTS3_UNICODE + #include "fts3Int.h" #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) #include #include @@ -239,5 +241,6 @@ }; *ppModule = &module; } #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ +#endif /* ifndef SQLITE_DISABLE_FTS3_UNICODE */ Index: ext/fts3/fts3_unicode2.c ================================================================== --- ext/fts3/fts3_unicode2.c +++ ext/fts3/fts3_unicode2.c @@ -13,10 +13,12 @@ /* ** DO NOT EDIT THIS MACHINE GENERATED FILE. */ +#ifndef SQLITE_DISABLE_FTS3_UNICODE + #include /* ** Return true if the argument corresponds to a unicode codepoint ** classified as either a letter or a number. Otherwise false. @@ -287,5 +289,6 @@ ret = c + 40; } return ret; } +#endif /* ifndef SQLITE_DISABLE_FTS3_UNICODE */ Index: ext/fts3/unicode/mkunicode.tcl ================================================================== --- ext/fts3/unicode/mkunicode.tcl +++ ext/fts3/unicode/mkunicode.tcl @@ -505,10 +505,12 @@ /* ** DO NOT EDIT THIS MACHINE GENERATED FILE. */ }] puts "" + puts "#ifndef SQLITE_DISABLE_FTS3_UNICODE" + puts "" puts "#include " puts "" } proc print_test_main {} { @@ -564,6 +566,8 @@ if {$::generate_test_code} { print_test_isalnum sqlite3FtsUnicodeIsalnum $lRange print_tolower_test sqlite3FtsUnicodeTolower print_test_main } + +puts "#endif /* ifndef SQLITE_DISABLE_FTS3_UNICODE */" Index: src/test_config.c ================================================================== --- src/test_config.c +++ src/test_config.c @@ -304,10 +304,16 @@ #ifdef SQLITE_ENABLE_FTS3 Tcl_SetVar2(interp, "sqlite_options", "fts3", "1", TCL_GLOBAL_ONLY); #else Tcl_SetVar2(interp, "sqlite_options", "fts3", "0", TCL_GLOBAL_ONLY); #endif + +#if !defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_DISABLE_FTS3_UNICODE) + Tcl_SetVar2(interp, "sqlite_options", "fts3_unicode", "0", TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp, "sqlite_options", "fts3_unicode", "1", TCL_GLOBAL_ONLY); +#endif #ifdef SQLITE_OMIT_GET_TABLE Tcl_SetVar2(interp, "sqlite_options", "gettable", "0", TCL_GLOBAL_ONLY); #else Tcl_SetVar2(interp, "sqlite_options", "gettable", "1", TCL_GLOBAL_ONLY); Index: test/fts3fault2.test ================================================================== --- test/fts3fault2.test +++ test/fts3fault2.test @@ -129,28 +129,30 @@ execsql { INSERT INTO ft(ft) VALUES('rebuild') } } -test { faultsim_test_result {0 {}} } -do_test 5.0 { - faultsim_delete_and_reopen - execsql { - CREATE VIRTUAL TABLE ft USING fts4(a, tokenize=unicode61); - } - faultsim_save_and_close -} {} - -do_faultsim_test 5.1 -faults oom* -prep { - faultsim_restore_and_reopen - db eval {SELECT * FROM sqlite_master} -} -body { - execsql { INSERT INTO ft VALUES('the quick brown fox'); } - execsql { INSERT INTO ft VALUES( - 'theunusuallylongtokenthatjustdragsonandonandonandthendragsonsomemoreeof' - ); - } - execsql { SELECT docid FROM ft WHERE ft MATCH 'th*' } -} -test { - faultsim_test_result {0 {1 2}} +ifcapable fts3_unicode { + do_test 5.0 { + faultsim_delete_and_reopen + execsql { + CREATE VIRTUAL TABLE ft USING fts4(a, tokenize=unicode61); + } + faultsim_save_and_close + } {} + + do_faultsim_test 5.1 -faults oom* -prep { + faultsim_restore_and_reopen + db eval {SELECT * FROM sqlite_master} + } -body { + execsql { INSERT INTO ft VALUES('the quick brown fox'); } + execsql { INSERT INTO ft VALUES( + 'theunusuallylongtokenthatjustdragsonandonandonandthendragsonsomemoreeof' + ); + } + execsql { SELECT docid FROM ft WHERE ft MATCH 'th*' } + } -test { + faultsim_test_result {0 {1 2}} + } } finish_test Index: test/fts4unicode.test ================================================================== --- test/fts4unicode.test +++ test/fts4unicode.test @@ -12,11 +12,11 @@ # The tests in this file focus on testing the "unicode" FTS tokenizer. # set testdir [file dirname $argv0] source $testdir/tester.tcl -ifcapable !fts3 { finish_test ; return } +ifcapable !fts3_unicode { finish_test ; return } set ::testprefix fts4unicode proc do_unicode_token_test {tn input res} { set input [string map {' ''} $input] uplevel [list do_execsql_test $tn "