/ Check-in [9ce6f40d]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add tests for the fts4 unicode61 tokenchars and separators options.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 9ce6f40dfb54b35cecba3cc9c1ec0d111f6e9f11
User & Date: dan 2013-09-13 12:10:09
Context
2013-09-13
16:36
Change the PRAGMA parser to use a binary search for the pragma name. Also: Minor performance enhancement to sqlite3DbFree() and to the token dequoter. check-in: 870c030b user: drh tags: trunk
12:10
Add tests for the fts4 unicode61 tokenchars and separators options. check-in: 9ce6f40d user: dan tags: trunk
2013-09-12
23:12
Fix typo in a macro name: "GlogUpperToLower" should be "GlobUpperToLower" check-in: 73634ca4 user: drh tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to test/fts4unicode.test.

   433    433   }
   434    434   do_execsql_test 8.2.2 {
   435    435     SELECT rowid FROM t4 WHERE t4 MATCH 'o';
   436    436   } {1 3}
   437    437   do_execsql_test 8.2.3 {
   438    438     SELECT rowid FROM t4 WHERE t4 MATCH 'a';
   439    439   } {2 4}
          440  +
          441  +#-------------------------------------------------------------------------
          442  +#
          443  +foreach {tn sql} {
          444  +  1 {
          445  +    CREATE VIRTUAL TABLE t5 USING fts4(tokenize=unicode61 [tokenchars= .]);
          446  +    CREATE VIRTUAL TABLE t6 USING fts4(
          447  +        tokenize=unicode61 [tokenchars=="] "tokenchars=[]");
          448  +    CREATE VIRTUAL TABLE t7 USING fts4(tokenize=unicode61 [separators=x\xC4]);
          449  +  }
          450  +  2 {
          451  +    CREATE VIRTUAL TABLE t5 USING fts4(tokenize=unicode61 "tokenchars= .");
          452  +    CREATE VIRTUAL TABLE t6 USING fts4(tokenize=unicode61 "tokenchars=[=""]");
          453  +    CREATE VIRTUAL TABLE t7 USING fts4(tokenize=unicode61 "separators=x\xC4");
          454  +  }
          455  +  3 {
          456  +    CREATE VIRTUAL TABLE t5 USING fts4(tokenize=unicode61 'tokenchars= .');
          457  +    CREATE VIRTUAL TABLE t6 USING fts4(tokenize=unicode61 'tokenchars=="[]');
          458  +    CREATE VIRTUAL TABLE t7 USING fts4(tokenize=unicode61 'separators=x\xC4');
          459  +  }
          460  +  4 {
          461  +    CREATE VIRTUAL TABLE t5 USING fts4(tokenize=unicode61 `tokenchars= .`);
          462  +    CREATE VIRTUAL TABLE t6 USING fts4(tokenize=unicode61 `tokenchars=[="]`);
          463  +    CREATE VIRTUAL TABLE t7 USING fts4(tokenize=unicode61 `separators=x\xC4`);
          464  +  }
          465  +} {
          466  +  do_execsql_test 9.$tn.0 { 
          467  +    DROP TABLE IF EXISTS t5;
          468  +    DROP TABLE IF EXISTS t5aux;
          469  +    DROP TABLE IF EXISTS t6;
          470  +    DROP TABLE IF EXISTS t6aux;
          471  +    DROP TABLE IF EXISTS t7;
          472  +    DROP TABLE IF EXISTS t7aux;
          473  +  }
          474  +  do_execsql_test 9.$tn.1 $sql
          475  +
          476  +  do_execsql_test 9.$tn.2 {
          477  +    CREATE VIRTUAL TABLE t5aux USING fts4aux(t5);
          478  +    INSERT INTO t5 VALUES('one two three/four.five.six');
          479  +    SELECT * FROM t5aux;
          480  +  } {
          481  +    four.five.six   * 1 1 four.five.six   0 1 1 
          482  +    {one two three} * 1 1 {one two three} 0 1 1
          483  +  }
          484  +
          485  +  do_execsql_test 9.$tn.3 {
          486  +    CREATE VIRTUAL TABLE t6aux USING fts4aux(t6);
          487  +    INSERT INTO t6 VALUES('alpha=beta"gamma/delta[epsilon]zeta');
          488  +    SELECT * FROM t6aux;
          489  +  } {
          490  +    {alpha=beta"gamma}   * 1 1 {alpha=beta"gamma} 0 1 1 
          491  +    {delta[epsilon]zeta} * 1 1 {delta[epsilon]zeta} 0 1 1
          492  +  }
          493  +
          494  +  do_execsql_test 9.$tn.4 {
          495  +    CREATE VIRTUAL TABLE t7aux USING fts4aux(t7);
          496  +    INSERT INTO t7 VALUES('alephxbeth\xC4gimel');
          497  +    SELECT * FROM t7aux;
          498  +  } {
          499  +    aleph * 1 1 aleph 0 1 1 
          500  +    beth  * 1 1 beth  0 1 1 
          501  +    gimel * 1 1 gimel 0 1 1
          502  +  }
          503  +}
          504  +
          505  +# Check that multiple options are handled correctly.
          506  +#
          507  +do_execsql_test 10.1 {
          508  +  DROP TABLE IF EXISTS t1;
          509  +  CREATE VIRTUAL TABLE t1 USING fts4(tokenize=unicode61
          510  +    "tokenchars=xyz" "tokenchars=.=" "separators=.=" "separators=xy"
          511  +    "separators=a" "separators=a" "tokenchars=a" "tokenchars=a"
          512  +  );
          513  +
          514  +  INSERT INTO t1 VALUES('oneatwoxthreeyfour');
          515  +  INSERT INTO t1 VALUES('a.single=word');
          516  +  CREATE VIRTUAL TABLE t1aux USING fts4aux(t1);
          517  +  SELECT * FROM t1aux;
          518  +} {
          519  +  .single=word * 1 1 .single=word 0 1 1 
          520  +  four         * 1 1 four         0 1 1 
          521  +  one          * 1 1 one          0 1 1 
          522  +  three        * 1 1 three        0 1 1 
          523  +  two          * 1 1 two          0 1 1
          524  +}
          525  +
          526  +# Test that case folding happens after tokenization, not before.
          527  +#
          528  +do_execsql_test 10.2 {
          529  +  DROP TABLE IF EXISTS t2;
          530  +  CREATE VIRTUAL TABLE t2 USING fts4(tokenize=unicode61 "separators=aB");
          531  +  INSERT INTO t2 VALUES('oneatwoBthree');
          532  +  INSERT INTO t2 VALUES('onebtwoAthree');
          533  +  CREATE VIRTUAL TABLE t2aux USING fts4aux(t2);
          534  +  SELECT * FROM t2aux;
          535  +} {
          536  +  one           * 1 1 one           0 1 1 
          537  +  onebtwoathree * 1 1 onebtwoathree 0 1 1 
          538  +  three         * 1 1 three         0 1 1 
          539  +  two           * 1 1 two           0 1 1
          540  +}
          541  +
   440    542   
   441    543   finish_test
          544  +
          545  +
          546  +
          547  +