Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Changes In Branch fts4-unicode Excluding Merge-Ins
This is equivalent to a diff from 6d326d44 to e71495a8
2012-05-26
| ||
18:42 | Merge fts4-unicode branch with trunk. (check-in: 25ba1f84 user: dan tags: trunk) | |
18:28 | If SQLITE_DISABLE_FTS3_UNICODE is defined, do not build the "unicode61" tokenizer. (Closed-Leaf check-in: e71495a8 user: dan tags: fts4-unicode) | |
18:06 | Enable the use of shared cache for an in-memory database, so that separate database connections can share the same in-memory database. (check-in: 4590e433 user: drh tags: shared-cache-memdb) | |
17:57 | Change the format of the tables used by sqlite3FtsUnicodeTolower() to make them a little smaller. (check-in: b89d3834 user: dan tags: fts4-unicode) | |
2012-05-25
| ||
17:50 | Add an experimental tokenizer to fts4 - "unicode". This tokenizer works in the same way except that it understands unicode "simple case folding" and recognizes all characters not classified as "Letters" or "Numbers" by unicode as token separators. (check-in: 0c13570e user: dan tags: fts4-unicode) | |
2012-05-22
| ||
17:39 | Merge the 3.7.12.1 trunk changes into the WinRT branch. (check-in: b9ed0b26 user: drh tags: winrt) | |
13:11 | Version 3.7.12.1 (check-in: 972e75bb user: drh tags: apple-osx) | |
13:01 | Version 3.7.12.1 (check-in: d07b7b67 user: drh tags: sessions) | |
02:45 | Version 3.7.12.1 (check-in: 6d326d44 user: drh tags: trunk, release, version-3.7.12.1) | |
2012-05-21
| ||
22:45 | Increase the version number to 3.7.12.1. Minor changes to test scripts. (check-in: 5519cc5e user: drh tags: trunk) | |
Changes to ext/fts3/README.tokenizers.
1 2 3 4 5 6 7 8 9 10 11 12 13 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 | - + | 1. FTS3 Tokenizers When creating a new full-text table, FTS3 allows the user to select the text tokenizer implementation to be used when indexing text by specifying a "tokenize" clause as part of the CREATE VIRTUAL TABLE statement: CREATE VIRTUAL TABLE <table-name> USING fts3( <columns ...> [, tokenize <tokenizer-name> [<tokenizer-args>]] ); The built-in tokenizers (valid values to pass as <tokenizer name>) are |
︙ |
Changes to ext/fts3/fts3.c.
︙ | |||
3550 3551 3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583 | 3550 3551 3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585 3586 3587 3588 3589 3590 3591 3592 3593 | + + + + + + + + + + | ** ** Calling sqlite3Fts3SimpleTokenizerModule() sets the value pointed ** to by the argument to point to the "simple" tokenizer implementation. ** And so on. */ void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule); void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule); #ifndef SQLITE_DISABLE_FTS3_UNICODE void sqlite3Fts3UnicodeTokenizer(sqlite3_tokenizer_module const**ppModule); #endif #ifdef SQLITE_ENABLE_ICU void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule); #endif /* ** Initialise the fts3 extension. If this extension is built as part ** of the sqlite library, then this function is called directly by ** SQLite. If fts3 is built as a dynamically loadable extension, this ** function is called by the sqlite3_extension_init() entry point. */ int sqlite3Fts3Init(sqlite3 *db){ int rc = SQLITE_OK; Fts3Hash *pHash = 0; const sqlite3_tokenizer_module *pSimple = 0; const sqlite3_tokenizer_module *pPorter = 0; #ifndef SQLITE_DISABLE_FTS3_UNICODE const sqlite3_tokenizer_module *pUnicode = 0; #endif #ifdef SQLITE_ENABLE_ICU const sqlite3_tokenizer_module *pIcu = 0; sqlite3Fts3IcuTokenizerModule(&pIcu); #endif #ifndef SQLITE_DISABLE_FTS3_UNICODE sqlite3Fts3UnicodeTokenizer(&pUnicode); #endif #ifdef SQLITE_TEST rc = sqlite3Fts3InitTerm(db); if( rc!=SQLITE_OK ) return rc; #endif rc = sqlite3Fts3InitAux(db); |
︙ | |||
3594 3595 3596 3597 3598 3599 3600 3601 3602 3603 3604 3605 3606 3607 | 3604 3605 3606 3607 3608 3609 3610 3611 3612 3613 3614 3615 3616 3617 3618 3619 3620 3621 | + + + + | sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1); } /* Load the built-in tokenizers into the hash table */ if( rc==SQLITE_OK ){ if( sqlite3Fts3HashInsert(pHash, "simple", 7, (void *)pSimple) || sqlite3Fts3HashInsert(pHash, "porter", 7, (void *)pPorter) #ifndef SQLITE_DISABLE_FTS3_UNICODE || sqlite3Fts3HashInsert(pHash, "unicode61", 10, (void *)pUnicode) #endif #ifdef SQLITE_ENABLE_ICU || (pIcu && sqlite3Fts3HashInsert(pHash, "icu", 4, (void *)pIcu)) #endif ){ rc = SQLITE_NOMEM; } } |
︙ |
Changes to ext/fts3/fts3Int.h.
︙ | |||
536 537 538 539 540 541 542 543 544 545 | 536 537 538 539 540 541 542 543 544 545 546 547 548 549 | + + + + | int sqlite3Fts3MsrIncrNext( Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *); int sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol, char **); int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *); int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr); int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *); /* fts3_unicode2.c (functions generated by parsing unicode text files) */ int sqlite3FtsUnicodeTolower(int); int sqlite3FtsUnicodeIsalnum(int); #endif /* !SQLITE_CORE || SQLITE_ENABLE_FTS3 */ #endif /* _FTSINT_H */ |
Added ext/fts3/fts3_unicode.c.