Index: ext/fts5/fts5Int.h ================================================================== --- ext/fts5/fts5Int.h +++ ext/fts5/fts5Int.h @@ -158,10 +158,11 @@ /* Values loaded from the %_config table */ int iCookie; /* Incremented when %_config is modified */ int pgsz; /* Approximate page size used in %_data */ int nAutomerge; /* 'automerge' setting */ int nCrisisMerge; /* Maximum allowed segments per level */ + int nHashSize; /* Bytes of memory for in-memory hash */ char *zRank; /* Name of rank function */ char *zRankArgs; /* Arguments to rank function */ /* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */ char **pzErrmsg; Index: ext/fts5/fts5_config.c ================================================================== --- ext/fts5/fts5_config.c +++ ext/fts5/fts5_config.c @@ -18,10 +18,11 @@ #include "fts5Int.h" #define FTS5_DEFAULT_PAGE_SIZE 4050 #define FTS5_DEFAULT_AUTOMERGE 4 #define FTS5_DEFAULT_CRISISMERGE 16 +#define FTS5_DEFAULT_HASHSIZE (1024*1024) /* Maximum allowed page size */ #define FTS5_MAX_PAGE_SIZE (128*1024) static int fts5_iswhitespace(char x){ @@ -764,10 +765,22 @@ *pbBadkey = 1; }else{ pConfig->pgsz = pgsz; } } + + else if( 0==sqlite3_stricmp(zKey, "hashsize") ){ + int nHashSize = -1; + if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ + nHashSize = sqlite3_value_int(pVal); + } + if( nHashSize<=0 ){ + *pbBadkey = 1; + }else{ + pConfig->nHashSize = nHashSize; + } + } else if( 0==sqlite3_stricmp(zKey, "automerge") ){ int nAutomerge = -1; if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ nAutomerge = sqlite3_value_int(pVal); @@ -825,10 +838,11 @@ /* Set default values */ pConfig->pgsz = FTS5_DEFAULT_PAGE_SIZE; pConfig->nAutomerge = FTS5_DEFAULT_AUTOMERGE; pConfig->nCrisisMerge = FTS5_DEFAULT_CRISISMERGE; + pConfig->nHashSize = FTS5_DEFAULT_HASHSIZE; zSql = sqlite3Fts5Mprintf(&rc, zSelect, pConfig->zDb, pConfig->zName); if( zSql ){ rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &p, 0); sqlite3_free(zSql); Index: ext/fts5/fts5_index.c ================================================================== --- ext/fts5/fts5_index.c +++ ext/fts5/fts5_index.c @@ -286,11 +286,10 @@ /* ** Variables related to the accumulation of tokens and doclists within the ** in-memory hash tables before they are flushed to disk. */ Fts5Hash *pHash; /* Hash table for in-memory data */ - int nMaxPendingData; /* Max pending data before flush to disk */ int nPendingData; /* Current bytes of pending data */ i64 iWriteRowid; /* Rowid for current doc being written */ int bDelete; /* Current write is a delete */ /* Error state. */ @@ -4446,16 +4445,19 @@ assert( p->rc==SQLITE_OK ); /* Allocate the hash table if it has not already been allocated */ if( p->pHash==0 ){ p->rc = sqlite3Fts5HashNew(&p->pHash, &p->nPendingData); + + /* Force the configuration to be loaded */ + fts5StructureRelease(fts5StructureRead(p)); } /* Flush the hash table to disk if required */ if( iRowidiWriteRowid || (iRowid==p->iWriteRowid && p->bDelete==0) - || (p->nPendingData > p->nMaxPendingData) + || (p->nPendingData > p->pConfig->nHashSize) ){ fts5IndexFlush(p); } p->iWriteRowid = iRowid; @@ -4517,11 +4519,10 @@ *pp = p = (Fts5Index*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Index)); if( rc==SQLITE_OK ){ p->pConfig = pConfig; p->nWorkUnit = FTS5_WORK_UNIT; - p->nMaxPendingData = 1024*1024; p->zDataTbl = sqlite3Fts5Mprintf(&rc, "%s_data", pConfig->zName); if( p->zDataTbl && bCreate ){ rc = sqlite3Fts5CreateTable( pConfig, "data", "id INTEGER PRIMARY KEY, block BLOB", 0, pzErr ); Index: ext/fts5/test/fts5simple.test ================================================================== --- ext/fts5/test/fts5simple.test +++ ext/fts5/test/fts5simple.test @@ -16,11 +16,11 @@ # If SQLITE_ENABLE_FTS5 is defined, omit this file. ifcapable !fts5 { finish_test return } - + #------------------------------------------------------------------------- # set doc "x x [string repeat {y } 50]z z" do_execsql_test 1.0 { CREATE VIRTUAL TABLE t1 USING fts5(x); @@ -319,8 +319,22 @@ } do_execsql_test 13.3 { INSERT INTO xy(xy) VALUES('integrity-check'); } + +#------------------------------------------------------------------------- +# +do_execsql_test 14.1 { + CREATE VIRTUAL TABLE ttt USING fts5(x); + BEGIN; + INSERT INTO ttt(rowid, x) VALUES(1, 'a b c'); + INSERT INTO ttt(rowid, x) VALUES(2, 'a b c'); + INSERT INTO ttt(rowid, x) VALUES(3, 'a b c'); + COMMIT; +} +do_test 14.2 { + fts5_level_segs ttt +} {1} finish_test Index: ext/fts5/tool/loadfts5.tcl ================================================================== --- ext/fts5/tool/loadfts5.tcl +++ ext/fts5/tool/loadfts5.tcl @@ -46,10 +46,11 @@ puts stderr " -limit N (load no more than N documents)" puts stderr " -automerge N (set the automerge parameter to N)" puts stderr " -crisismerge N (set the crisismerge parameter to N)" puts stderr " -prefix PREFIX (comma separated prefix= argument)" puts stderr " -trans N (commit after N inserts - 0 == never)" + puts stderr " -hashsize N (set the fts5 hashsize parameteger to N)" exit 1 } set O(vtab) fts5 set O(tok) "" @@ -57,10 +58,11 @@ set O(delete) 0 set O(automerge) -1 set O(crisismerge) -1 set O(prefix) "" set O(trans) 0 +set O(hashsize) -1 if {[llength $argv]<2} usage set nOpt [expr {[llength $argv]-2}] for {set i 0} {$i < $nOpt} {incr i} { set arg [lindex $argv $i] @@ -103,10 +105,15 @@ -prefix { if { [incr i]>=$nOpt } usage set O(prefix) [lindex $argv $i] } + + -hashsize { + if { [incr i]>=$nOpt } usage + set O(hashsize) [lindex $argv $i] + } default { usage } } @@ -124,10 +131,18 @@ if {$O(prefix)!=""} { set pref ", prefix='$O(prefix)'" } catch { db eval "CREATE VIRTUAL TABLE t1 USING $O(vtab) (path, content$O(tok)$pref)" db eval "INSERT INTO t1(t1, rank) VALUES('pgsz', 4050);" } + + if {$O(hashsize)>=0} { + catch { + db eval "INSERT INTO t1(t1, rank) VALUES('hashsize', $O(hashsize));" + } + } + + if {$O(automerge)>=0} { if {$O(vtab) == "fts5"} { db eval { INSERT INTO t1(t1, rank) VALUES('automerge', $O(automerge)) } } else { db eval { INSERT INTO t1(t1) VALUES('automerge=' || $O(automerge)) } @@ -139,8 +154,9 @@ } else { } } load_hierachy [lindex $argv end] db eval COMMIT +puts ""