/ Check-in [e71495a8]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:If SQLITE_DISABLE_FTS3_UNICODE is defined, do not build the "unicode61" tokenizer.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts4-unicode
Files: files | file ages | folders
SHA1: e71495a817b479bc23c5403d99255e3f098eb054
User & Date: dan 2012-05-26 18:28:14
Context
2012-05-26
18:42
Merge fts4-unicode branch with trunk. check-in: 25ba1f84 user: dan tags: trunk
18:28
If SQLITE_DISABLE_FTS3_UNICODE is defined, do not build the "unicode61" tokenizer. Closed-Leaf check-in: e71495a8 user: dan tags: fts4-unicode
17:57
Change the format of the tables used by sqlite3FtsUnicodeTolower() to make them a little smaller. check-in: b89d3834 user: dan tags: fts4-unicode
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts3/fts3.c.

  3550   3550   **
  3551   3551   ** Calling sqlite3Fts3SimpleTokenizerModule() sets the value pointed
  3552   3552   ** to by the argument to point to the "simple" tokenizer implementation.
  3553   3553   ** And so on.
  3554   3554   */
  3555   3555   void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
  3556   3556   void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule);
         3557  +#ifndef SQLITE_DISABLE_FTS3_UNICODE
  3557   3558   void sqlite3Fts3UnicodeTokenizer(sqlite3_tokenizer_module const**ppModule);
         3559  +#endif
  3558   3560   #ifdef SQLITE_ENABLE_ICU
  3559   3561   void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule);
  3560   3562   #endif
  3561   3563   
  3562   3564   /*
  3563   3565   ** Initialise the fts3 extension. If this extension is built as part
  3564   3566   ** of the sqlite library, then this function is called directly by
................................................................................
  3566   3568   ** function is called by the sqlite3_extension_init() entry point.
  3567   3569   */
  3568   3570   int sqlite3Fts3Init(sqlite3 *db){
  3569   3571     int rc = SQLITE_OK;
  3570   3572     Fts3Hash *pHash = 0;
  3571   3573     const sqlite3_tokenizer_module *pSimple = 0;
  3572   3574     const sqlite3_tokenizer_module *pPorter = 0;
         3575  +#ifndef SQLITE_DISABLE_FTS3_UNICODE
  3573   3576     const sqlite3_tokenizer_module *pUnicode = 0;
         3577  +#endif
  3574   3578   
  3575   3579   #ifdef SQLITE_ENABLE_ICU
  3576   3580     const sqlite3_tokenizer_module *pIcu = 0;
  3577   3581     sqlite3Fts3IcuTokenizerModule(&pIcu);
  3578   3582   #endif
         3583  +
         3584  +#ifndef SQLITE_DISABLE_FTS3_UNICODE
         3585  +  sqlite3Fts3UnicodeTokenizer(&pUnicode);
         3586  +#endif
  3579   3587   
  3580   3588   #ifdef SQLITE_TEST
  3581   3589     rc = sqlite3Fts3InitTerm(db);
  3582   3590     if( rc!=SQLITE_OK ) return rc;
  3583   3591   #endif
  3584   3592   
  3585   3593     rc = sqlite3Fts3InitAux(db);
  3586   3594     if( rc!=SQLITE_OK ) return rc;
  3587   3595   
  3588   3596     sqlite3Fts3SimpleTokenizerModule(&pSimple);
  3589   3597     sqlite3Fts3PorterTokenizerModule(&pPorter);
  3590         -  sqlite3Fts3UnicodeTokenizer(&pUnicode);
  3591   3598   
  3592   3599     /* Allocate and initialise the hash-table used to store tokenizers. */
  3593   3600     pHash = sqlite3_malloc(sizeof(Fts3Hash));
  3594   3601     if( !pHash ){
  3595   3602       rc = SQLITE_NOMEM;
  3596   3603     }else{
  3597   3604       sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1);
  3598   3605     }
  3599   3606   
  3600   3607     /* Load the built-in tokenizers into the hash table */
  3601   3608     if( rc==SQLITE_OK ){
  3602   3609       if( sqlite3Fts3HashInsert(pHash, "simple", 7, (void *)pSimple)
  3603   3610        || sqlite3Fts3HashInsert(pHash, "porter", 7, (void *)pPorter) 
         3611  +
         3612  +#ifndef SQLITE_DISABLE_FTS3_UNICODE
  3604   3613        || sqlite3Fts3HashInsert(pHash, "unicode61", 10, (void *)pUnicode) 
         3614  +#endif
  3605   3615   #ifdef SQLITE_ENABLE_ICU
  3606   3616        || (pIcu && sqlite3Fts3HashInsert(pHash, "icu", 4, (void *)pIcu))
  3607   3617   #endif
  3608   3618       ){
  3609   3619         rc = SQLITE_NOMEM;
  3610   3620       }
  3611   3621     }

Changes to ext/fts3/fts3_unicode.c.

     9      9   **    May you share freely, never taking more than you give.
    10     10   **
    11     11   ******************************************************************************
    12     12   **
    13     13   ** Implementation of the "unicode" full-text-search tokenizer.
    14     14   */
    15     15   
           16  +#ifndef SQLITE_DISABLE_FTS3_UNICODE
           17  +
    16     18   #include "fts3Int.h"
    17     19   #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
    18     20   
    19     21   #include <assert.h>
    20     22   #include <stdlib.h>
    21     23   #include <stdio.h>
    22     24   #include <string.h>
................................................................................
   237    239       unicodeNext,
   238    240       0,
   239    241     };
   240    242     *ppModule = &module;
   241    243   }
   242    244   
   243    245   #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
          246  +#endif /* ifndef SQLITE_DISABLE_FTS3_UNICODE */

Changes to ext/fts3/fts3_unicode2.c.

    11     11   ******************************************************************************
    12     12   */
    13     13   
    14     14   /*
    15     15   ** DO NOT EDIT THIS MACHINE GENERATED FILE.
    16     16   */
    17     17   
           18  +#ifndef SQLITE_DISABLE_FTS3_UNICODE
           19  +
    18     20   #include <assert.h>
    19     21   
    20     22   /*
    21     23   ** Return true if the argument corresponds to a unicode codepoint
    22     24   ** classified as either a letter or a number. Otherwise false.
    23     25   **
    24     26   ** The results are undefined if the value passed to this function
................................................................................
   285    287     
   286    288     else if( c>=66560 && c<66600 ){
   287    289       ret = c + 40;
   288    290     }
   289    291   
   290    292     return ret;
   291    293   }
          294  +#endif /* ifndef SQLITE_DISABLE_FTS3_UNICODE */

Changes to ext/fts3/unicode/mkunicode.tcl.

   503    503   */
   504    504   
   505    505   /*
   506    506   ** DO NOT EDIT THIS MACHINE GENERATED FILE.
   507    507   */
   508    508     }]
   509    509     puts ""
          510  +  puts "#ifndef SQLITE_DISABLE_FTS3_UNICODE"
          511  +  puts ""
   510    512     puts "#include <assert.h>"
   511    513     puts ""
   512    514   }
   513    515   
   514    516   proc print_test_main {} {
   515    517     puts ""
   516    518     puts "#include <stdio.h>"
................................................................................
   562    564   # was specified.
   563    565   #
   564    566   if {$::generate_test_code} {
   565    567     print_test_isalnum sqlite3FtsUnicodeIsalnum $lRange
   566    568     print_tolower_test sqlite3FtsUnicodeTolower 
   567    569     print_test_main 
   568    570   }
          571  +
          572  +puts "#endif /* ifndef SQLITE_DISABLE_FTS3_UNICODE */"
   569    573   

Changes to src/test_config.c.

   302    302   #endif
   303    303   
   304    304   #ifdef SQLITE_ENABLE_FTS3
   305    305     Tcl_SetVar2(interp, "sqlite_options", "fts3", "1", TCL_GLOBAL_ONLY);
   306    306   #else
   307    307     Tcl_SetVar2(interp, "sqlite_options", "fts3", "0", TCL_GLOBAL_ONLY);
   308    308   #endif
          309  +
          310  +#if !defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_DISABLE_FTS3_UNICODE)
          311  +  Tcl_SetVar2(interp, "sqlite_options", "fts3_unicode", "0", TCL_GLOBAL_ONLY);
          312  +#else
          313  +  Tcl_SetVar2(interp, "sqlite_options", "fts3_unicode", "1", TCL_GLOBAL_ONLY);
          314  +#endif
   309    315   
   310    316   #ifdef SQLITE_OMIT_GET_TABLE
   311    317     Tcl_SetVar2(interp, "sqlite_options", "gettable", "0", TCL_GLOBAL_ONLY);
   312    318   #else
   313    319     Tcl_SetVar2(interp, "sqlite_options", "gettable", "1", TCL_GLOBAL_ONLY);
   314    320   #endif
   315    321   

Changes to test/fts3fault2.test.

   127    127     db eval {SELECT * FROM sqlite_master}
   128    128   } -body {
   129    129     execsql { INSERT INTO ft(ft) VALUES('rebuild') }
   130    130   } -test {
   131    131     faultsim_test_result {0 {}}
   132    132   }
   133    133   
   134         -do_test 5.0 {
   135         -  faultsim_delete_and_reopen
   136         -  execsql {
   137         -    CREATE VIRTUAL TABLE ft USING fts4(a, tokenize=unicode61);
   138         -  }
   139         -  faultsim_save_and_close
   140         -} {}
   141         -
   142         -do_faultsim_test 5.1 -faults oom* -prep {
   143         -  faultsim_restore_and_reopen
   144         -  db eval {SELECT * FROM sqlite_master}
   145         -} -body {
   146         -  execsql { INSERT INTO ft VALUES('the quick brown fox'); }
   147         -  execsql { INSERT INTO ft VALUES(
   148         -      'theunusuallylongtokenthatjustdragsonandonandonandthendragsonsomemoreeof'
   149         -    );
   150         -  }
   151         -  execsql { SELECT docid FROM ft WHERE ft MATCH 'th*' }
   152         -} -test {
   153         -  faultsim_test_result {0 {1 2}}
          134  +ifcapable fts3_unicode {
          135  +  do_test 5.0 {
          136  +    faultsim_delete_and_reopen
          137  +    execsql {
          138  +      CREATE VIRTUAL TABLE ft USING fts4(a, tokenize=unicode61);
          139  +    }
          140  +    faultsim_save_and_close
          141  +  } {}
          142  +  
          143  +  do_faultsim_test 5.1 -faults oom* -prep {
          144  +    faultsim_restore_and_reopen
          145  +    db eval {SELECT * FROM sqlite_master}
          146  +  } -body {
          147  +    execsql { INSERT INTO ft VALUES('the quick brown fox'); }
          148  +    execsql { INSERT INTO ft VALUES(
          149  +       'theunusuallylongtokenthatjustdragsonandonandonandthendragsonsomemoreeof'
          150  +      );
          151  +    }
          152  +    execsql { SELECT docid FROM ft WHERE ft MATCH 'th*' }
          153  +  } -test {
          154  +    faultsim_test_result {0 {1 2}}
          155  +  }
   154    156   }
   155    157   
   156    158   finish_test

Changes to test/fts4unicode.test.

    10     10   #*************************************************************************
    11     11   #
    12     12   # The tests in this file focus on testing the "unicode" FTS tokenizer.
    13     13   #
    14     14   
    15     15   set testdir [file dirname $argv0]
    16     16   source $testdir/tester.tcl
    17         -ifcapable !fts3 { finish_test ; return }
           17  +ifcapable !fts3_unicode { finish_test ; return }
    18     18   set ::testprefix fts4unicode
    19     19   
    20     20   proc do_unicode_token_test {tn input res} {
    21     21     set input [string map {' ''} $input]
    22     22     uplevel [list do_execsql_test $tn "
    23     23       SELECT fts3_tokenizer_test('unicode61', '$input');
    24     24     " [list [list {*}$res]]]