# 2007 June 21
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library. The focus
# of this script is testing the pluggable tokeniser feature of the
# FTS2 module.
#
# $Id: fts2token.test,v 1.3 2007/06/25 12:05:40 danielk1977 Exp $
#
set testdir [file dirname $argv0]
source $testdir/tester.tcl
# If SQLITE_ENABLE_FTS2 is defined, omit this file.
ifcapable !fts2 {
finish_test
return
}
proc escape_string {str} {
set out ""
foreach char [split $str ""] {
scan $char %c i
if {$i<=127} {
append out $char
} else {
append out [format {\x%.4x} $i]
}
}
set out
}
#--------------------------------------------------------------------------
# Test cases fts2token-1.* are the warm-body test for the SQL scalar
# function fts2_tokenizer(). The procedure is as follows:
#
# 1: Verify that there is no such fts2 tokenizer as 'blah'.
#
# 2: Query for the built-in tokenizer 'simple'. Insert a copy of the
# retrieved value as tokenizer 'blah'.
#
# 3: Test that the value returned for tokenizer 'blah' is now the
# same as that retrieved for 'simple'.
#
# 4: Test that it is now possible to create an fts2 table using
# tokenizer 'blah' (it was not possible in step 1).
#
# 5: Test that the table created to use tokenizer 'blah' is usable.
#
do_test fts2token-1.1 {
catchsql {
CREATE VIRTUAL TABLE t1 USING fts2(content, tokenize blah);
}
} {1 {unknown tokenizer: blah}}
do_test fts2token-1.2 {
execsql {
SELECT fts2_tokenizer('blah', fts2_tokenizer('simple')) IS NULL;
}
} {0}
do_test fts2token-1.3 {
execsql {
SELECT fts2_tokenizer('blah') == fts2_tokenizer('simple');
}
} {1}
do_test fts2token-1.4 {
catchsql {
CREATE VIRTUAL TABLE t1 USING fts2(content, tokenize blah);
}
} {0 {}}
do_test fts2token-1.5 {
execsql {
INSERT INTO t1(content) VALUES('There was movement at the station');
INSERT INTO t1(content) VALUES('For the word has passed around');
INSERT INTO t1(content) VALUES('That the colt from ol regret had got away');
SELECT content FROM t1 WHERE content MATCH 'movement'
}
} {{There was movement at the station}}
#--------------------------------------------------------------------------
# Test cases fts2token-2.* test error cases in the scalar function based
# API for getting and setting tokenizers.
#
do_test fts2token-2.1 {
catchsql {
SELECT fts2_tokenizer('nosuchtokenizer');
}
} {1 {unknown tokenizer: nosuchtokenizer}}
#--------------------------------------------------------------------------
# Test cases fts2token-3.* test the three built-in tokenizers with a
# simple input string via the built-in test function. This is as much
# to test the test function as the tokenizer implementations.
#
do_test fts2token-3.1 {
execsql {
SELECT fts2_tokenizer_test('simple', 'I don''t see how');
}
} {{0 i I 1 don don 2 t t 3 see see 4 how how}}
do_test fts2token-3.2 {
execsql {
SELECT fts2_tokenizer_test('porter', 'I don''t see how');
}
} {{0 i I 1 don don 2 t t 3 see see 4 how how}}
ifcapable icu {
do_test fts2token-3.3 {
execsql {
SELECT fts2_tokenizer_test('icu', 'I don''t see how');
}
} {{0 i I 1 don't don't 2 see see 3 how how}}
}
#--------------------------------------------------------------------------
# Test cases fts2token-4.* test the ICU tokenizer. In practice, this
# tokenizer only has two modes - "thai" and "everybody else". Some other
# Asian languages (Lao, Khmer etc.) require the same special treatment as
# Thai, but ICU doesn't support them yet.
#
ifcapable icu {
proc do_icu_test {name locale input output} {
set ::out [db eval { SELECT fts2_tokenizer_test('icu', $locale, $input) }]
do_test $name {
lindex $::out 0
} $output
}
do_icu_test fts2token-4.1 en_US {} {}
do_icu_test fts2token-4.2 en_US {Test cases fts2} [list \
0 test Test 1 cases cases 2 fts2 fts2
]
# The following test shows that ICU is smart enough to recognise
# Thai chararacters, even when the locale is set to English/United
# States.
#
set input "\u0e2d\u0e30\u0e44\u0e23\u0e19\u0e30\u0e04\u0e23\u0e31\u0e1a"
set output "0 \u0e2d\u0e30\u0e44\u0e23 \u0e2d\u0e30\u0e44\u0e23 "
append output "1 \u0e19\u0e30 \u0e19\u0e30 "
append output "2 \u0e04\u0e23\u0e31\u0e1a \u0e04\u0e23\u0e31\u0e1a"
do_icu_test fts2token-4.3 th_TH $input $output
do_icu_test fts2token-4.4 en_US $input $output
# ICU handles an unknown locale by falling back to the default.
# So this is not an error.
do_icu_test fts2token-4.5 MiddleOfTheOcean $input $output
set longtoken "AReallyReallyLongTokenOneThatWillSurelyRequire"
append longtoken "AReallocInTheIcuTokenizerCode"
set input "short tokens then "
append input $longtoken
set output "0 short short "
append output "1 tokens tokens "
append output "2 then then "
append output "3 [string tolower $longtoken] $longtoken"
do_icu_test fts2token-4.6 MiddleOfTheOcean $input $output
do_icu_test fts2token-4.7 th_TH $input $output
do_icu_test fts2token-4.8 en_US $input $output
}
do_test fts2token-internal {
execsql { SELECT fts2_tokenizer_internal_test() }
} {ok}
finish_test