Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Add the 'hashsize' configuration option to fts5, for configuring the amount of memory allocated to the in-memory hash table while writing. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | trunk |
Files: | files | file ages | folders |
SHA1: |
445480095e6877cce8220b1c095f334b |
User & Date: | dan 2015-11-05 18:09:16.776 |
Context
2015-11-05
| ||
21:49 | Fix typo in help information for an FTS5 script. (check-in: 777ae8007f user: mistachkin tags: trunk) | |
20:25 | The top of an index equality loop normally starts with OP_SeekGE and OP_IdxGT. This check-in adds a flag to OP_SeekGE such that it fails immediately if the key is not equal, then jumps over the OP_IdxGT, saving a call to the key comparison functions. Consider this check-in a proof-of-concept. It needs improvement before going on trunk. Some tests fail, but only because they new use fewer key comparisons than expected (which is a good thing!). (check-in: 32e31b9bc8 user: drh tags: seekeq-experiment) | |
18:09 | Add the 'hashsize' configuration option to fts5, for configuring the amount of memory allocated to the in-memory hash table while writing. (check-in: 445480095e user: dan tags: trunk) | |
11:47 | Remove a #pragma used to work around an issues with MSVC 2012 that has been overcome but subsequent changes. (check-in: 8303e4cfed user: drh tags: trunk) | |
Changes
Changes to ext/fts5/fts5Int.h.
︙ | ︙ | |||
156 157 158 159 160 161 162 163 164 165 166 167 168 169 | fts5_tokenizer *pTokApi; /* Values loaded from the %_config table */ int iCookie; /* Incremented when %_config is modified */ int pgsz; /* Approximate page size used in %_data */ int nAutomerge; /* 'automerge' setting */ int nCrisisMerge; /* Maximum allowed segments per level */ char *zRank; /* Name of rank function */ char *zRankArgs; /* Arguments to rank function */ /* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */ char **pzErrmsg; #ifdef SQLITE_DEBUG | > | 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 | fts5_tokenizer *pTokApi; /* Values loaded from the %_config table */ int iCookie; /* Incremented when %_config is modified */ int pgsz; /* Approximate page size used in %_data */ int nAutomerge; /* 'automerge' setting */ int nCrisisMerge; /* Maximum allowed segments per level */ int nHashSize; /* Bytes of memory for in-memory hash */ char *zRank; /* Name of rank function */ char *zRankArgs; /* Arguments to rank function */ /* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */ char **pzErrmsg; #ifdef SQLITE_DEBUG |
︙ | ︙ |
Changes to ext/fts5/fts5_config.c.
︙ | ︙ | |||
16 17 18 19 20 21 22 23 24 25 26 27 28 29 | #include "fts5Int.h" #define FTS5_DEFAULT_PAGE_SIZE 4050 #define FTS5_DEFAULT_AUTOMERGE 4 #define FTS5_DEFAULT_CRISISMERGE 16 /* Maximum allowed page size */ #define FTS5_MAX_PAGE_SIZE (128*1024) static int fts5_iswhitespace(char x){ return (x==' '); } | > | 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 | #include "fts5Int.h" #define FTS5_DEFAULT_PAGE_SIZE 4050 #define FTS5_DEFAULT_AUTOMERGE 4 #define FTS5_DEFAULT_CRISISMERGE 16 #define FTS5_DEFAULT_HASHSIZE (1024*1024) /* Maximum allowed page size */ #define FTS5_MAX_PAGE_SIZE (128*1024) static int fts5_iswhitespace(char x){ return (x==' '); } |
︙ | ︙ | |||
762 763 764 765 766 767 768 769 770 771 772 773 774 775 | } if( pgsz<=0 || pgsz>FTS5_MAX_PAGE_SIZE ){ *pbBadkey = 1; }else{ pConfig->pgsz = pgsz; } } else if( 0==sqlite3_stricmp(zKey, "automerge") ){ int nAutomerge = -1; if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ nAutomerge = sqlite3_value_int(pVal); } if( nAutomerge<0 || nAutomerge>64 ){ | > > > > > > > > > > > > | 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 | } if( pgsz<=0 || pgsz>FTS5_MAX_PAGE_SIZE ){ *pbBadkey = 1; }else{ pConfig->pgsz = pgsz; } } else if( 0==sqlite3_stricmp(zKey, "hashsize") ){ int nHashSize = -1; if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ nHashSize = sqlite3_value_int(pVal); } if( nHashSize<=0 ){ *pbBadkey = 1; }else{ pConfig->nHashSize = nHashSize; } } else if( 0==sqlite3_stricmp(zKey, "automerge") ){ int nAutomerge = -1; if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ nAutomerge = sqlite3_value_int(pVal); } if( nAutomerge<0 || nAutomerge>64 ){ |
︙ | ︙ | |||
823 824 825 826 827 828 829 830 831 832 833 834 835 836 | int rc = SQLITE_OK; int iVersion = 0; /* Set default values */ pConfig->pgsz = FTS5_DEFAULT_PAGE_SIZE; pConfig->nAutomerge = FTS5_DEFAULT_AUTOMERGE; pConfig->nCrisisMerge = FTS5_DEFAULT_CRISISMERGE; zSql = sqlite3Fts5Mprintf(&rc, zSelect, pConfig->zDb, pConfig->zName); if( zSql ){ rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &p, 0); sqlite3_free(zSql); } | > | 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 | int rc = SQLITE_OK; int iVersion = 0; /* Set default values */ pConfig->pgsz = FTS5_DEFAULT_PAGE_SIZE; pConfig->nAutomerge = FTS5_DEFAULT_AUTOMERGE; pConfig->nCrisisMerge = FTS5_DEFAULT_CRISISMERGE; pConfig->nHashSize = FTS5_DEFAULT_HASHSIZE; zSql = sqlite3Fts5Mprintf(&rc, zSelect, pConfig->zDb, pConfig->zName); if( zSql ){ rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &p, 0); sqlite3_free(zSql); } |
︙ | ︙ |
Changes to ext/fts5/fts5_index.c.
︙ | ︙ | |||
284 285 286 287 288 289 290 | int nWorkUnit; /* Leaf pages in a "unit" of work */ /* ** Variables related to the accumulation of tokens and doclists within the ** in-memory hash tables before they are flushed to disk. */ Fts5Hash *pHash; /* Hash table for in-memory data */ | < | 284 285 286 287 288 289 290 291 292 293 294 295 296 297 | int nWorkUnit; /* Leaf pages in a "unit" of work */ /* ** Variables related to the accumulation of tokens and doclists within the ** in-memory hash tables before they are flushed to disk. */ Fts5Hash *pHash; /* Hash table for in-memory data */ int nPendingData; /* Current bytes of pending data */ i64 iWriteRowid; /* Rowid for current doc being written */ int bDelete; /* Current write is a delete */ /* Error state. */ int rc; /* Current error code */ |
︙ | ︙ | |||
4444 4445 4446 4447 4448 4449 4450 4451 4452 4453 4454 4455 | */ int sqlite3Fts5IndexBeginWrite(Fts5Index *p, int bDelete, i64 iRowid){ assert( p->rc==SQLITE_OK ); /* Allocate the hash table if it has not already been allocated */ if( p->pHash==0 ){ p->rc = sqlite3Fts5HashNew(&p->pHash, &p->nPendingData); } /* Flush the hash table to disk if required */ if( iRowid<p->iWriteRowid || (iRowid==p->iWriteRowid && p->bDelete==0) | > > > | | 4443 4444 4445 4446 4447 4448 4449 4450 4451 4452 4453 4454 4455 4456 4457 4458 4459 4460 4461 4462 4463 4464 4465 | */ int sqlite3Fts5IndexBeginWrite(Fts5Index *p, int bDelete, i64 iRowid){ assert( p->rc==SQLITE_OK ); /* Allocate the hash table if it has not already been allocated */ if( p->pHash==0 ){ p->rc = sqlite3Fts5HashNew(&p->pHash, &p->nPendingData); /* Force the configuration to be loaded */ fts5StructureRelease(fts5StructureRead(p)); } /* Flush the hash table to disk if required */ if( iRowid<p->iWriteRowid || (iRowid==p->iWriteRowid && p->bDelete==0) || (p->nPendingData > p->pConfig->nHashSize) ){ fts5IndexFlush(p); } p->iWriteRowid = iRowid; p->bDelete = bDelete; return fts5IndexReturn(p); |
︙ | ︙ | |||
4515 4516 4517 4518 4519 4520 4521 | int rc = SQLITE_OK; Fts5Index *p; /* New object */ *pp = p = (Fts5Index*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Index)); if( rc==SQLITE_OK ){ p->pConfig = pConfig; p->nWorkUnit = FTS5_WORK_UNIT; | < | 4517 4518 4519 4520 4521 4522 4523 4524 4525 4526 4527 4528 4529 4530 | int rc = SQLITE_OK; Fts5Index *p; /* New object */ *pp = p = (Fts5Index*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Index)); if( rc==SQLITE_OK ){ p->pConfig = pConfig; p->nWorkUnit = FTS5_WORK_UNIT; p->zDataTbl = sqlite3Fts5Mprintf(&rc, "%s_data", pConfig->zName); if( p->zDataTbl && bCreate ){ rc = sqlite3Fts5CreateTable( pConfig, "data", "id INTEGER PRIMARY KEY, block BLOB", 0, pzErr ); if( rc==SQLITE_OK ){ rc = sqlite3Fts5CreateTable(pConfig, "idx", |
︙ | ︙ |
Changes to ext/fts5/test/fts5simple.test.
︙ | ︙ | |||
317 318 319 320 321 322 323 324 325 326 | 1 {1 2 3} 3 {2 3 4} } do_execsql_test 13.3 { INSERT INTO xy(xy) VALUES('integrity-check'); } finish_test | > > > > > > > > > > > > > > | 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 | 1 {1 2 3} 3 {2 3 4} } do_execsql_test 13.3 { INSERT INTO xy(xy) VALUES('integrity-check'); } #------------------------------------------------------------------------- # do_execsql_test 14.1 { CREATE VIRTUAL TABLE ttt USING fts5(x); BEGIN; INSERT INTO ttt(rowid, x) VALUES(1, 'a b c'); INSERT INTO ttt(rowid, x) VALUES(2, 'a b c'); INSERT INTO ttt(rowid, x) VALUES(3, 'a b c'); COMMIT; } do_test 14.2 { fts5_level_segs ttt } {1} finish_test |
Changes to ext/fts5/tool/loadfts5.tcl.
︙ | ︙ | |||
44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 | puts stderr " -porter (use porter tokenizer)" puts stderr " -delete (delete the database file before starting)" puts stderr " -limit N (load no more than N documents)" puts stderr " -automerge N (set the automerge parameter to N)" puts stderr " -crisismerge N (set the crisismerge parameter to N)" puts stderr " -prefix PREFIX (comma separated prefix= argument)" puts stderr " -trans N (commit after N inserts - 0 == never)" exit 1 } set O(vtab) fts5 set O(tok) "" set O(limit) 0 set O(delete) 0 set O(automerge) -1 set O(crisismerge) -1 set O(prefix) "" set O(trans) 0 if {[llength $argv]<2} usage set nOpt [expr {[llength $argv]-2}] for {set i 0} {$i < $nOpt} {incr i} { set arg [lindex $argv $i] switch -- [lindex $argv $i] { -fts4 { | > > | 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 | puts stderr " -porter (use porter tokenizer)" puts stderr " -delete (delete the database file before starting)" puts stderr " -limit N (load no more than N documents)" puts stderr " -automerge N (set the automerge parameter to N)" puts stderr " -crisismerge N (set the crisismerge parameter to N)" puts stderr " -prefix PREFIX (comma separated prefix= argument)" puts stderr " -trans N (commit after N inserts - 0 == never)" puts stderr " -hashsize N (set the fts5 hashsize parameteger to N)" exit 1 } set O(vtab) fts5 set O(tok) "" set O(limit) 0 set O(delete) 0 set O(automerge) -1 set O(crisismerge) -1 set O(prefix) "" set O(trans) 0 set O(hashsize) -1 if {[llength $argv]<2} usage set nOpt [expr {[llength $argv]-2}] for {set i 0} {$i < $nOpt} {incr i} { set arg [lindex $argv $i] switch -- [lindex $argv $i] { -fts4 { |
︙ | ︙ | |||
101 102 103 104 105 106 107 108 109 110 111 112 113 114 | set O(crisismerge) [lindex $argv $i] } -prefix { if { [incr i]>=$nOpt } usage set O(prefix) [lindex $argv $i] } default { usage } } } | > > > > > | 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 | set O(crisismerge) [lindex $argv $i] } -prefix { if { [incr i]>=$nOpt } usage set O(prefix) [lindex $argv $i] } -hashsize { if { [incr i]>=$nOpt } usage set O(hashsize) [lindex $argv $i] } default { usage } } } |
︙ | ︙ | |||
122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 | db eval BEGIN set pref "" if {$O(prefix)!=""} { set pref ", prefix='$O(prefix)'" } catch { db eval "CREATE VIRTUAL TABLE t1 USING $O(vtab) (path, content$O(tok)$pref)" db eval "INSERT INTO t1(t1, rank) VALUES('pgsz', 4050);" } if {$O(automerge)>=0} { if {$O(vtab) == "fts5"} { db eval { INSERT INTO t1(t1, rank) VALUES('automerge', $O(automerge)) } } else { db eval { INSERT INTO t1(t1) VALUES('automerge=' || $O(automerge)) } } } if {$O(crisismerge)>=0} { if {$O(vtab) == "fts5"} { db eval {INSERT INTO t1(t1, rank) VALUES('crisismerge', $O(crisismerge))} } else { } } load_hierachy [lindex $argv end] db eval COMMIT | > > > > > > > > > | 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 | db eval BEGIN set pref "" if {$O(prefix)!=""} { set pref ", prefix='$O(prefix)'" } catch { db eval "CREATE VIRTUAL TABLE t1 USING $O(vtab) (path, content$O(tok)$pref)" db eval "INSERT INTO t1(t1, rank) VALUES('pgsz', 4050);" } if {$O(hashsize)>=0} { catch { db eval "INSERT INTO t1(t1, rank) VALUES('hashsize', $O(hashsize));" } } if {$O(automerge)>=0} { if {$O(vtab) == "fts5"} { db eval { INSERT INTO t1(t1, rank) VALUES('automerge', $O(automerge)) } } else { db eval { INSERT INTO t1(t1) VALUES('automerge=' || $O(automerge)) } } } if {$O(crisismerge)>=0} { if {$O(vtab) == "fts5"} { db eval {INSERT INTO t1(t1, rank) VALUES('crisismerge', $O(crisismerge))} } else { } } load_hierachy [lindex $argv end] db eval COMMIT puts "" |