Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Remove some redundant code from fts5. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | fts5 |
Files: | files | file ages | folders |
SHA1: |
939b7a5de25e064bdf08e03864c35ab7 |
User & Date: | dan 2015-01-23 06:50:33.338 |
Context
2015-01-23
| ||
17:43 | Fix compression of keys stored on internal segment b-tree nodes by fts5. (check-in: 51444f67c0 user: dan tags: fts5) | |
06:50 | Remove some redundant code from fts5. (check-in: 939b7a5de2 user: dan tags: fts5) | |
2015-01-22
| ||
19:13 | Add further tests and fixes for fts5. (check-in: 5b29589715 user: dan tags: fts5) | |
Changes
Changes to ext/fts5/fts5Int.h.
︙ | ︙ | |||
82 83 84 85 86 87 88 89 90 91 92 93 94 95 | Fts5Tokenizer *pTok; fts5_tokenizer *pTokApi; /* Values loaded from the %_config table */ int iCookie; /* Incremented when %_config is modified */ int pgsz; /* Approximate page size used in %_data */ int nAutomerge; /* 'automerge' setting */ char *zRank; /* Name of rank function */ char *zRankArgs; /* Arguments to rank function */ }; #define FTS5_CONTENT_NORMAL 0 #define FTS5_CONTENT_NONE 1 #define FTS5_CONTENT_EXTERNAL 2 | > | 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 | Fts5Tokenizer *pTok; fts5_tokenizer *pTokApi; /* Values loaded from the %_config table */ int iCookie; /* Incremented when %_config is modified */ int pgsz; /* Approximate page size used in %_data */ int nAutomerge; /* 'automerge' setting */ int nCrisisMerge; /* Maximum allowed segments per level */ char *zRank; /* Name of rank function */ char *zRankArgs; /* Arguments to rank function */ }; #define FTS5_CONTENT_NORMAL 0 #define FTS5_CONTENT_NONE 1 #define FTS5_CONTENT_EXTERNAL 2 |
︙ | ︙ |
Changes to ext/fts5/fts5_config.c.
︙ | ︙ | |||
13 14 15 16 17 18 19 20 21 22 23 24 25 26 | ** This is an SQLite module implementing full-text search. */ #include "fts5Int.h" #define FTS5_DEFAULT_PAGE_SIZE 1000 #define FTS5_DEFAULT_AUTOMERGE 4 /* Maximum allowed page size */ #define FTS5_MAX_PAGE_SIZE (128*1024) static int fts5_iswhitespace(char x){ return (x==' '); } | > | 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 | ** This is an SQLite module implementing full-text search. */ #include "fts5Int.h" #define FTS5_DEFAULT_PAGE_SIZE 1000 #define FTS5_DEFAULT_AUTOMERGE 4 #define FTS5_DEFAULT_CRISISMERGE 16 /* Maximum allowed page size */ #define FTS5_MAX_PAGE_SIZE (128*1024) static int fts5_iswhitespace(char x){ return (x==' '); } |
︙ | ︙ | |||
712 713 714 715 716 717 718 719 720 721 722 723 724 725 | if( nAutomerge<0 || nAutomerge>64 ){ if( pbBadkey ) *pbBadkey = 1; }else{ if( nAutomerge==1 ) nAutomerge = FTS5_DEFAULT_AUTOMERGE; pConfig->nAutomerge = nAutomerge; } } else if( 0==sqlite3_stricmp(zKey, "rank") ){ const char *zIn = (const char*)sqlite3_value_text(pVal); char *zRank; char *zRankArgs; rc = sqlite3Fts5ConfigParseRank(zIn, &zRank, &zRankArgs); if( rc==SQLITE_OK ){ | > > > > > > > > > > > > > | 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 | if( nAutomerge<0 || nAutomerge>64 ){ if( pbBadkey ) *pbBadkey = 1; }else{ if( nAutomerge==1 ) nAutomerge = FTS5_DEFAULT_AUTOMERGE; pConfig->nAutomerge = nAutomerge; } } else if( 0==sqlite3_stricmp(zKey, "crisismerge") ){ int nCrisisMerge = -1; if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ nCrisisMerge = sqlite3_value_int(pVal); } if( nCrisisMerge<0 ){ if( pbBadkey ) *pbBadkey = 1; }else{ if( nCrisisMerge<=1 ) nCrisisMerge = FTS5_DEFAULT_CRISISMERGE; pConfig->nCrisisMerge = nCrisisMerge; } } else if( 0==sqlite3_stricmp(zKey, "rank") ){ const char *zIn = (const char*)sqlite3_value_text(pVal); char *zRank; char *zRankArgs; rc = sqlite3Fts5ConfigParseRank(zIn, &zRank, &zRankArgs); if( rc==SQLITE_OK ){ |
︙ | ︙ | |||
745 746 747 748 749 750 751 752 753 754 755 756 757 758 | char *zSql; sqlite3_stmt *p = 0; int rc; /* Set default values */ pConfig->pgsz = FTS5_DEFAULT_PAGE_SIZE; pConfig->nAutomerge = FTS5_DEFAULT_AUTOMERGE; zSql = sqlite3_mprintf(zSelect, pConfig->zDb, pConfig->zName); if( zSql==0 ){ rc = SQLITE_NOMEM; }else{ rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &p, 0); sqlite3_free(zSql); | > | 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 | char *zSql; sqlite3_stmt *p = 0; int rc; /* Set default values */ pConfig->pgsz = FTS5_DEFAULT_PAGE_SIZE; pConfig->nAutomerge = FTS5_DEFAULT_AUTOMERGE; pConfig->nCrisisMerge = FTS5_DEFAULT_CRISISMERGE; zSql = sqlite3_mprintf(zSelect, pConfig->zDb, pConfig->zName); if( zSql==0 ){ rc = SQLITE_NOMEM; }else{ rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &p, 0); sqlite3_free(zSql); |
︙ | ︙ |
Changes to ext/fts5/fts5_index.c.
︙ | ︙ | |||
39 40 41 42 43 44 45 | ** * extra fields in the "structure record" record the state of ongoing ** incremental merge operations. ** */ #define FTS5_OPT_WORK_UNIT 1000 /* Number of leaf pages per optimize step */ #define FTS5_WORK_UNIT 64 /* Number of leaf pages in unit of work */ | < | 39 40 41 42 43 44 45 46 47 48 49 50 51 52 | ** * extra fields in the "structure record" record the state of ongoing ** incremental merge operations. ** */ #define FTS5_OPT_WORK_UNIT 1000 /* Number of leaf pages per optimize step */ #define FTS5_WORK_UNIT 64 /* Number of leaf pages in unit of work */ #define FTS5_MIN_DLIDX_SIZE 4 /* Add dlidx if this many empty pages */ /* ** Details: ** ** The %_data table managed by this module, |
︙ | ︙ | |||
289 290 291 292 293 294 295 | /* ** One object per %_data table. */ struct Fts5Index { Fts5Config *pConfig; /* Virtual table configuration */ char *zDataTbl; /* Name of %_data table */ | < | 288 289 290 291 292 293 294 295 296 297 298 299 300 301 | /* ** One object per %_data table. */ struct Fts5Index { Fts5Config *pConfig; /* Virtual table configuration */ char *zDataTbl; /* Name of %_data table */ int nWorkUnit; /* Leaf pages in a "unit" of work */ /* ** Variables related to the accumulation of tokens and doclists within the ** in-memory hash tables before they are flushed to disk. */ Fts5Hash **apHash; /* Array of hash tables */ |
︙ | ︙ | |||
1101 1102 1103 1104 1105 1106 1107 | int iCookie; /* Configuration cookie */ assert( iIdx<=pConfig->nPrefix ); pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID(iIdx)); if( !pData ) return 0; p->rc = fts5StructureDecode(pData->p, pData->n, &iCookie, &pRet); | | | | 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 | int iCookie; /* Configuration cookie */ assert( iIdx<=pConfig->nPrefix ); pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID(iIdx)); if( !pData ) return 0; p->rc = fts5StructureDecode(pData->p, pData->n, &iCookie, &pRet); if( p->rc==SQLITE_OK && pConfig->iCookie!=iCookie ){ p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie); } fts5DataRelease(pData); if( p->rc!=SQLITE_OK ){ fts5StructureRelease(pRet); pRet = 0; } |
︙ | ︙ | |||
1246 1247 1248 1249 1250 1251 1252 | Fts5Structure *pStruct /* Index structure */ ){ if( p->rc==SQLITE_OK ){ int iTst; int iPromote = -1; int szPromote; /* Promote anything this size or smaller */ Fts5StructureSegment *pSeg; /* Segment just written */ | < < < > > | 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 | Fts5Structure *pStruct /* Index structure */ ){ if( p->rc==SQLITE_OK ){ int iTst; int iPromote = -1; int szPromote; /* Promote anything this size or smaller */ Fts5StructureSegment *pSeg; /* Segment just written */ int szSeg; /* Size of segment just written */ pSeg = &pStruct->aLevel[iLvl].aSeg[pStruct->aLevel[iLvl].nSeg-1]; szSeg = (1 + pSeg->pgnoLast - pSeg->pgnoFirst); /* Check for condition (a) */ for(iTst=iLvl-1; iTst>=0 && pStruct->aLevel[iTst].nSeg==0; iTst--); if( iTst>=0 ){ int i; int szMax = 0; Fts5StructureLevel *pTst = &pStruct->aLevel[iTst]; assert( pTst->nMerge==0 ); for(i=0; i<pTst->nSeg; i++){ int sz = pTst->aSeg[i].pgnoLast - pTst->aSeg[i].pgnoFirst + 1; if( sz>szMax ) szMax = sz; } if( szMax>=szSeg ){ /* Condition (a) is true. Promote the newest segment on level ** iLvl to level iTst. */ |
︙ | ︙ | |||
2479 2480 2481 2482 2483 2484 2485 | ** Return true if the position iterator passed as the second argument is ** at EOF. Or if an error has already occurred. Otherwise, return false. */ static int fts5PosIterEof(Fts5Index *p, Fts5PosIter *pIter){ return (p->rc || pIter->chunk.pLeaf==0); } | < < < < < < < < < < < < < < < < < < < < < < | 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 | ** Return true if the position iterator passed as the second argument is ** at EOF. Or if an error has already occurred. Otherwise, return false. */ static int fts5PosIterEof(Fts5Index *p, Fts5PosIter *pIter){ return (p->rc || pIter->chunk.pLeaf==0); } /* ** Allocate a new segment-id for the structure pStruct. ** ** If an error has already occurred, this function is a no-op. 0 is ** returned in this case. */ static int fts5AllocateSegid(Fts5Index *p, Fts5Structure *pStruct){ |
︙ | ︙ | |||
3224 3225 3226 3227 3228 3229 3230 | int iIdx, /* Index to work on */ Fts5Structure **ppStruct /* IN/OUT: Current structure of index */ ){ Fts5Structure *pStruct = *ppStruct; int iLvl = 0; while( p->rc==SQLITE_OK && iLvl<pStruct->nLevel | | | 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 | int iIdx, /* Index to work on */ Fts5Structure **ppStruct /* IN/OUT: Current structure of index */ ){ Fts5Structure *pStruct = *ppStruct; int iLvl = 0; while( p->rc==SQLITE_OK && iLvl<pStruct->nLevel && pStruct->aLevel[iLvl].nSeg>=p->pConfig->nCrisisMerge ){ fts5IndexMergeLevel(p, iIdx, &pStruct, iLvl, 0); fts5StructurePromote(p, iLvl+1, pStruct); iLvl++; } *ppStruct = pStruct; } |
︙ | ︙ | |||
3996 3997 3998 3999 4000 4001 4002 4003 4004 4005 4006 4007 4008 4009 | /* ** Indicate that all subsequent calls to sqlite3Fts5IndexWrite() pertain ** to the document with rowid iRowid. */ int sqlite3Fts5IndexBeginWrite(Fts5Index *p, i64 iRowid){ assert( p->rc==SQLITE_OK ); if( iRowid<=p->iWriteRowid || (p->nPendingData > p->nMaxPendingData) ){ fts5IndexFlush(p); } p->iWriteRowid = iRowid; return fts5IndexReturn(p); } | > > > > > > > > > > > > > > > > > > > > > > > | 3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986 3987 3988 3989 3990 3991 3992 3993 3994 3995 3996 3997 3998 3999 4000 4001 4002 4003 4004 4005 4006 4007 | /* ** Indicate that all subsequent calls to sqlite3Fts5IndexWrite() pertain ** to the document with rowid iRowid. */ int sqlite3Fts5IndexBeginWrite(Fts5Index *p, i64 iRowid){ assert( p->rc==SQLITE_OK ); /* Allocate hash tables if they have not already been allocated */ if( p->apHash==0 ){ int i; int rc = SQLITE_OK; int nHash = p->pConfig->nPrefix + 1; Fts5Hash **apNew; apNew = (Fts5Hash**)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Hash*)*nHash); for(i=0; rc==SQLITE_OK && i<nHash; i++){ rc = sqlite3Fts5HashNew(&apNew[i], &p->nPendingData); } if( rc==SQLITE_OK ){ p->apHash = apNew; }else{ for(i=0; i<nHash; i++){ sqlite3Fts5HashFree(apNew[i]); } sqlite3_free(apNew); return rc; } } if( iRowid<=p->iWriteRowid || (p->nPendingData > p->nMaxPendingData) ){ fts5IndexFlush(p); } p->iWriteRowid = iRowid; return fts5IndexReturn(p); } |
︙ | ︙ | |||
4067 4068 4069 4070 4071 4072 4073 | Fts5Index *p; /* New object */ *pp = p = (Fts5Index*)sqlite3_malloc(sizeof(Fts5Index)); if( !p ) return SQLITE_NOMEM; memset(p, 0, sizeof(Fts5Index)); p->pConfig = pConfig; | < | 4065 4066 4067 4068 4069 4070 4071 4072 4073 4074 4075 4076 4077 4078 | Fts5Index *p; /* New object */ *pp = p = (Fts5Index*)sqlite3_malloc(sizeof(Fts5Index)); if( !p ) return SQLITE_NOMEM; memset(p, 0, sizeof(Fts5Index)); p->pConfig = pConfig; p->nWorkUnit = FTS5_WORK_UNIT; p->nMaxPendingData = 1024*1024; p->zDataTbl = sqlite3_mprintf("%s_data", pConfig->zName); if( p->zDataTbl==0 ){ rc = SQLITE_NOMEM; }else if( bCreate ){ rc = sqlite3Fts5CreateTable( |
︙ | ︙ | |||
4192 4193 4194 4195 4196 4197 4198 4199 | int sqlite3Fts5IndexWrite( Fts5Index *p, /* Index to write to */ int iCol, /* Column token appears in (-ve -> delete) */ int iPos, /* Position of token within column */ const char *pToken, int nToken /* Token to add or remove to or from index */ ){ int i; /* Used to iterate through indexes */ Fts5Config *pConfig = p->pConfig; | > < < < < < | < < < > | > | > | > | | 4189 4190 4191 4192 4193 4194 4195 4196 4197 4198 4199 4200 4201 4202 4203 4204 4205 4206 4207 4208 4209 4210 4211 4212 4213 4214 4215 4216 4217 4218 4219 4220 4221 4222 | int sqlite3Fts5IndexWrite( Fts5Index *p, /* Index to write to */ int iCol, /* Column token appears in (-ve -> delete) */ int iPos, /* Position of token within column */ const char *pToken, int nToken /* Token to add or remove to or from index */ ){ int i; /* Used to iterate through indexes */ int rc; /* Return code */ Fts5Config *pConfig = p->pConfig; assert( p->rc==SQLITE_OK ); /* Add the new token to the main terms hash table. And to each of the ** prefix hash tables that it is large enough for. */ rc = sqlite3Fts5HashWrite( p->apHash[0], p->iWriteRowid, iCol, iPos, pToken, nToken ); for(i=0; i<pConfig->nPrefix && rc==SQLITE_OK; i++){ int nByte = fts5IndexCharlenToBytelen(pToken, nToken, pConfig->aPrefix[i]); if( nByte ){ rc = sqlite3Fts5HashWrite( p->apHash[i+1], p->iWriteRowid, iCol, iPos, pToken, nByte ); } } return rc; } /* ** Open a new iterator to iterate though all docids that match the ** specified token or token prefix. */ int sqlite3Fts5IndexQuery( |
︙ | ︙ |
Changes to ext/fts5/tool/loadfts5.tcl.
︙ | ︙ | |||
25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 | puts stderr "" puts stderr "Switches are:" puts stderr " -fts4 (use fts4 instead of fts5)" puts stderr " -fts5 (use fts5)" puts stderr " -porter (use porter tokenizer)" puts stderr " -limit N (load no more than N documents)" puts stderr " -automerge N (set the automerge parameter to N)" exit 1 } set O(vtab) fts5 set O(tok) "" set O(limit) 0 set O(automerge) -1 if {[llength $argv]<2} usage set nOpt [expr {[llength $argv]-2}] for {set i 0} {$i < $nOpt} {incr i} { set arg [lindex $argv $i] switch -- [lindex $argv $i] { -fts4 { | > > | 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 | puts stderr "" puts stderr "Switches are:" puts stderr " -fts4 (use fts4 instead of fts5)" puts stderr " -fts5 (use fts5)" puts stderr " -porter (use porter tokenizer)" puts stderr " -limit N (load no more than N documents)" puts stderr " -automerge N (set the automerge parameter to N)" puts stderr " -crisismerge N (set the crisismerge parameter to N)" exit 1 } set O(vtab) fts5 set O(tok) "" set O(limit) 0 set O(automerge) -1 set O(crisismerge) -1 if {[llength $argv]<2} usage set nOpt [expr {[llength $argv]-2}] for {set i 0} {$i < $nOpt} {incr i} { set arg [lindex $argv $i] switch -- [lindex $argv $i] { -fts4 { |
︙ | ︙ | |||
59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 | set O(limit) [lindex $argv $i] } -automerge { if { [incr i]>=$nOpt } usage set O(automerge) [lindex $argv $i] } default { usage } } } sqlite3 db [lindex $argv end-1] db func loadfile loadfile db transaction { db eval "CREATE VIRTUAL TABLE t1 USING $O(vtab) (path, content$O(tok))" if {$O(automerge)>=0} { if {$O(vtab) == "fts5"} { db eval { INSERT INTO t1(t1, rank) VALUES('automerge', $O(automerge)) } } else { db eval { INSERT INTO t1(t1) VALUES('automerge=' || $O(automerge)) } } } load_hierachy [lindex $argv end] } | > > > > > > > > > > > | 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 | set O(limit) [lindex $argv $i] } -automerge { if { [incr i]>=$nOpt } usage set O(automerge) [lindex $argv $i] } -crisismerge { if { [incr i]>=$nOpt } usage set O(crisismerge) [lindex $argv $i] } default { usage } } } sqlite3 db [lindex $argv end-1] db func loadfile loadfile db transaction { db eval "CREATE VIRTUAL TABLE t1 USING $O(vtab) (path, content$O(tok))" if {$O(automerge)>=0} { if {$O(vtab) == "fts5"} { db eval { INSERT INTO t1(t1, rank) VALUES('automerge', $O(automerge)) } } else { db eval { INSERT INTO t1(t1) VALUES('automerge=' || $O(automerge)) } } } if {$O(crisismerge)>=0} { if {$O(vtab) == "fts5"} { db eval {INSERT INTO t1(t1, rank) VALUES('crisismerge', $O(crisismerge))} } else { } } load_hierachy [lindex $argv end] } |