Index: ext/fts3/fts3_write.c ================================================================== --- ext/fts3/fts3_write.c +++ ext/fts3/fts3_write.c @@ -508,14 +508,17 @@ Fts3Table *p, /* FTS3 table handle */ int iLangid, /* Language id */ int iIndex, /* Index in p->aIndex[] */ int iLevel /* Level of segments */ ){ + sqlite3_int64 iBase; /* First absolute level for iLangid/iIndex */ assert( iLangid>=0 ); assert( p->nIndex>0 ); assert( iIndex>=0 && iIndexnIndex ); - return (iLangid * p->nIndex + iIndex) * FTS3_SEGDIR_MAXLEVEL + iLevel; + + iBase = ((sqlite3_int64)iLangid * p->nIndex + iIndex) * FTS3_SEGDIR_MAXLEVEL; + return iBase + iLevel; } /* ** Given an absolute level number, determine the langauge-id, index ** and relative level that it corresponds to. @@ -554,11 +557,11 @@ */ int sqlite3Fts3AllSegdirs( Fts3Table *p, /* FTS3 table */ int iLangid, /* Language being queried */ int iIndex, /* Index for p->aIndex[] */ - int iLevel, /* Level to select */ + int iLevel, /* Level to select (relative level) */ sqlite3_stmt **ppStmt /* OUT: Compiled statement */ ){ int rc; sqlite3_stmt *pStmt = 0; @@ -569,19 +572,19 @@ if( iLevel<0 ){ /* "SELECT * FROM %_segdir WHERE level BETWEEN ? AND ? ORDER BY ..." */ rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE, &pStmt, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); - sqlite3_bind_int(pStmt, 2, + sqlite3_bind_int64(pStmt, 2, getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) ); } }else{ /* "SELECT * FROM %_segdir WHERE level = ? ORDER BY ..." */ rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); if( rc==SQLITE_OK ){ - sqlite3_bind_int(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, iLevel)); + sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex,iLevel)); } } *ppStmt = pStmt; return rc; } @@ -1852,11 +1855,11 @@ /* ** Insert a record into the %_segdir table. */ static int fts3WriteSegdir( Fts3Table *p, /* Virtual table handle */ - int iLevel, /* Value for "level" field */ + sqlite3_int64 iLevel, /* Value for "level" field (absolute level) */ int iIdx, /* Value for "idx" field */ sqlite3_int64 iStartBlock, /* Value for "start_block" field */ sqlite3_int64 iLeafEndBlock, /* Value for "leaves_end_block" field */ sqlite3_int64 iEndBlock, /* Value for "end_block" field */ char *zRoot, /* Blob value for "root" field */ @@ -1863,11 +1866,11 @@ int nRoot /* Number of bytes in buffer zRoot */ ){ sqlite3_stmt *pStmt; int rc = fts3SqlStmt(p, SQL_INSERT_SEGDIR, &pStmt, 0); if( rc==SQLITE_OK ){ - sqlite3_bind_int(pStmt, 1, iLevel); + sqlite3_bind_int64(pStmt, 1, iLevel); sqlite3_bind_int(pStmt, 2, iIdx); sqlite3_bind_int64(pStmt, 3, iStartBlock); sqlite3_bind_int64(pStmt, 4, iLeafEndBlock); sqlite3_bind_int64(pStmt, 5, iEndBlock); sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC); @@ -2246,11 +2249,11 @@ ** returned. Otherwise, an SQLite error code. */ static int fts3SegWriterFlush( Fts3Table *p, /* Virtual table handle */ SegmentWriter *pWriter, /* SegmentWriter to flush to the db */ - int iLevel, /* Value for 'level' column of %_segdir */ + sqlite3_int64 iLevel, /* Value for 'level' column of %_segdir */ int iIdx /* Value for 'idx' column of %_segdir */ ){ int rc; /* Return code */ if( pWriter->pTree ){ sqlite3_int64 iLast = 0; /* Largest block id written to database */ @@ -2328,11 +2331,11 @@ */ static int fts3SegmentMaxLevel( Fts3Table *p, int iLangid, int iIndex, - int *pnMax + sqlite3_int64 *pnMax ){ sqlite3_stmt *pStmt; int rc; assert( iIndex>=0 && iIndexnIndex ); @@ -2342,16 +2345,16 @@ ** ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR). */ rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0); if( rc!=SQLITE_OK ) return rc; - sqlite3_bind_int(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); - sqlite3_bind_int(pStmt, 2, + sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); + sqlite3_bind_int64(pStmt, 2, getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) ); if( SQLITE_ROW==sqlite3_step(pStmt) ){ - *pnMax = sqlite3_column_int(pStmt, 0); + *pnMax = sqlite3_column_int64(pStmt, 0); } return sqlite3_reset(pStmt); } /* @@ -2412,19 +2415,21 @@ assert( iLevel>=0 || iLevel==FTS3_SEGCURSOR_ALL ); if( iLevel==FTS3_SEGCURSOR_ALL ){ rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_RANGE, &pDelete, 0); if( rc==SQLITE_OK ){ - sqlite3_bind_int(pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); - sqlite3_bind_int(pDelete, 2, + sqlite3_bind_int64(pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); + sqlite3_bind_int64(pDelete, 2, getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) ); } }else{ rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pDelete, 0); if( rc==SQLITE_OK ){ - sqlite3_bind_int(pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex,iLevel)); + sqlite3_bind_int64( + pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex, iLevel) + ); } } if( rc==SQLITE_OK ){ sqlite3_step(pDelete); @@ -2897,11 +2902,11 @@ int iIndex, /* Index in p->aIndex[] to merge */ int iLevel /* Level to merge */ ){ int rc; /* Return code */ int iIdx = 0; /* Index of new segment */ - int iNewLevel = 0; /* Level/index to create new segment at */ + sqlite3_int64 iNewLevel = 0; /* Level/index to create new segment at */ SegmentWriter *pWriter = 0; /* Used to write the new, merged, segment */ Fts3SegFilter filter; /* Segment term filter condition */ Fts3MultiSegReader csr; /* Cursor to iterate through level(s) */ int bIgnoreEmpty = 0; /* True to ignore empty segments */ @@ -3862,11 +3867,11 @@ const char *zKey = pCsr->zTerm; int nKey = pCsr->nTerm; rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pOutputIdx, 0); if( rc==SQLITE_OK ){ - sqlite3_bind_int(pOutputIdx, 1, iAbsLevel+1); + sqlite3_bind_int64(pOutputIdx, 1, iAbsLevel+1); sqlite3_step(pOutputIdx); iIdx = sqlite3_column_int(pOutputIdx, 0) - 1; rc = sqlite3_reset(pOutputIdx); } if( rc!=SQLITE_OK ) return rc; Index: test/fts4langid.test ================================================================== --- test/fts4langid.test +++ test/fts4langid.test @@ -379,7 +379,109 @@ } [expr 0==($i%2)] } do_catchsql_test 4.1.5 { INSERT INTO t4(content, lid) VALUES('hello world', 101) } {1 {SQL logic error or missing database}} + +#------------------------------------------------------------------------- +# Test cases 5.* +# +# The following test cases are designed to detect a 32-bit overflow bug +# that existed at one point. +# +proc build_multilingual_db_3 {db} { + $db eval { + CREATE VIRTUAL TABLE t5 USING fts4(languageid=lid); + } + set languages [list 0 1 2 [expr 1<<30]] + + foreach lid $languages { + execsql { + INSERT INTO t5(docid, content, lid) VALUES( + $lid, 'My language is ' || $lid, $lid + ) + } + } +} + +do_test 5.1.0 { + reset_db + build_multilingual_db_3 db +} {} + +do_execsql_test 5.1.1 { + SELECT level FROM t5_segdir; +} [list 0 1024 2048 [expr 1<<40]] + +do_execsql_test 5.1.2 {SELECT docid FROM t5 WHERE t5 MATCH 'language'} 0 +foreach langid [list 0 1 2 [expr 1<<30]] { + do_execsql_test 5.2.$langid { + SELECT docid FROM t5 WHERE t5 MATCH 'language' AND lid = $langid + } $langid +} + +set lid [expr 1<<30] +do_execsql_test 5.3.1 { + CREATE VIRTUAL TABLE t6 USING fts4(languageid=lid); + INSERT INTO t6 VALUES('I belong to language 0!'); +} +do_test 5.3.2 { + for {set i 0} {$i < 20} {incr i} { + execsql { + INSERT INTO t6(content, lid) VALUES( + 'I (row '||$i||') belong to langauge N!', $lid + ); + } + } + execsql { SELECT docid FROM t6 WHERE t6 MATCH 'belong' } +} {1} + +do_test 5.3.3 { + execsql { SELECT docid FROM t6 WHERE t6 MATCH 'belong' AND lid=$lid} +} {2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21} + +do_execsql_test 5.3.4 { INSERT INTO t6(t6) VALUES('optimize') } {} +do_execsql_test 5.3.5 { SELECT docid FROM t6 WHERE t6 MATCH 'belong' } {1} +do_execsql_test 5.3.6 { + SELECT docid FROM t6 WHERE t6 MATCH 'belong' AND lid=$lid +} {2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21} + + +set lid [expr 1<<30] +foreach lid [list 4 [expr 1<<30]] { + do_execsql_test 5.4.$lid.1 { + DELETE FROM t6; + SELECT count(*) FROM t6_segdir; + SELECT count(*) FROM t6_segments; + } {0 0} + do_execsql_test 5.4.$lid.2 { + INSERT INTO t6(content, lid) VALUES('zero zero zero', $lid); + INSERT INTO t6(content, lid) VALUES('zero zero one', $lid); + INSERT INTO t6(content, lid) VALUES('zero one zero', $lid); + INSERT INTO t6(content, lid) VALUES('zero one one', $lid); + INSERT INTO t6(content, lid) VALUES('one zero zero', $lid); + INSERT INTO t6(content, lid) VALUES('one zero one', $lid); + INSERT INTO t6(content, lid) VALUES('one one zero', $lid); + INSERT INTO t6(content, lid) VALUES('one one one', $lid); + + SELECT docid FROM t6 WHERE t6 MATCH '"zero zero"' AND lid=$lid; + } {1 2 5} + + do_execsql_test 5.4.$lid.3 { + SELECT count(*) FROM t6_segdir; + SELECT count(*) FROM t6_segments; + } {8 0} + + do_execsql_test 5.4.$lid.4 { + INSERT INTO t6(t6) VALUES('merge=100,3'); + INSERT INTO t6(t6) VALUES('merge=100,3'); + SELECT docid FROM t6 WHERE t6 MATCH '"zero zero"' AND lid=$lid; + } {1 2 5} + + do_execsql_test 5.4.$lid.5 { + SELECT count(*) FROM t6_segdir; + SELECT count(*) FROM t6_segments; + } {4 4} +} + finish_test