Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Cherrypick the [3475092cff] fix for 32-bit overflow with large language-ids into trunk. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | trunk |
Files: | files | file ages | folders |
SHA1: |
2755edc7f1186ba8955f2c3550a90f40 |
User & Date: | drh 2012-03-16 16:52:54.032 |
Context
2012-03-16
| ||
17:00 | Update the trace2.test script for languageid. (check-in: a11528ac57 user: drh tags: trunk) | |
16:52 | Cherrypick the [3475092cff] fix for 32-bit overflow with large language-ids into trunk. (check-in: 2755edc7f1 user: drh tags: trunk) | |
00:28 | Fix harmless compiler warnings. (check-in: 74eadeec34 user: drh tags: trunk) | |
Changes
Changes to ext/fts3/fts3_write.c.
︙ | ︙ | |||
430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 | }else{ rc = SQLITE_OK; } return rc; } static sqlite3_int64 getAbsoluteLevel( Fts3Table *p, int iLangid, int iIndex, int iLevel ){ assert( iLangid>=0 ); assert( p->nIndex>0 ); assert( iIndex>=0 && iIndex<p->nIndex ); | > > > > > > > > > > > > > > > > > > > > > > > > > | > | 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 | }else{ rc = SQLITE_OK; } return rc; } /* ** FTS maintains a separate indexes for each language-id (a 32-bit integer). ** Within each language id, a separate index is maintained to store the ** document terms, and each configured prefix size (configured the FTS ** "prefix=" option). And each index consists of multiple levels ("relative ** levels"). ** ** All three of these values (the language id, the specific index and the ** level within the index) are encoded in 64-bit integer values stored ** in the %_segdir table on disk. This function is used to convert three ** separate component values into the single 64-bit integer value that ** can be used to query the %_segdir table. ** ** Specifically, each language-id/index combination is allocated 1024 ** 64-bit integer level values ("absolute levels"). The main terms index ** for language-id 0 is allocate values 0-1023. The first prefix index ** (if any) for language-id 0 is allocated values 1024-2047. And so on. ** Language 1 indexes are allocated immediately following language 0. ** ** So, for a system with nPrefix prefix indexes configured, the block of ** absolute levels that corresponds to language-id iLangid and index ** iIndex starts at absolute level ((iLangid * (nPrefix+1) + iIndex) * 1024). */ static sqlite3_int64 getAbsoluteLevel( Fts3Table *p, int iLangid, int iIndex, int iLevel ){ sqlite3_int64 iBase; /* First absolute level for iLangid/iIndex */ assert( iLangid>=0 ); assert( p->nIndex>0 ); assert( iIndex>=0 && iIndex<p->nIndex ); iBase = ((sqlite3_int64)iLangid * p->nIndex + iIndex) * FTS3_SEGDIR_MAXLEVEL; return iBase + iLevel; } /* ** Set *ppStmt to a statement handle that may be used to iterate through ** all rows in the %_segdir table, from oldest to newest. If successful, ** return SQLITE_OK. If an error occurs while preparing the statement, |
︙ | ︙ | |||
464 465 466 467 468 469 470 | ** 3: end_block ** 4: root */ int sqlite3Fts3AllSegdirs( Fts3Table *p, /* FTS3 table */ int iLangid, /* Language being queried */ int iIndex, /* Index for p->aIndex[] */ | | | | | 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 | ** 3: end_block ** 4: root */ int sqlite3Fts3AllSegdirs( Fts3Table *p, /* FTS3 table */ int iLangid, /* Language being queried */ int iIndex, /* Index for p->aIndex[] */ int iLevel, /* Level to select (relative level) */ sqlite3_stmt **ppStmt /* OUT: Compiled statement */ ){ int rc; sqlite3_stmt *pStmt = 0; assert( iLevel==FTS3_SEGCURSOR_ALL || iLevel>=0 ); assert( iLevel<FTS3_SEGDIR_MAXLEVEL ); assert( iIndex>=0 && iIndex<p->nIndex ); if( iLevel<0 ){ /* "SELECT * FROM %_segdir WHERE level BETWEEN ? AND ? ORDER BY ..." */ rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE, &pStmt, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); sqlite3_bind_int64(pStmt, 2, getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) ); } }else{ /* "SELECT * FROM %_segdir WHERE level = ? ORDER BY ..." */ rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex,iLevel)); } } *ppStmt = pStmt; return rc; } |
︙ | ︙ | |||
1759 1760 1761 1762 1763 1764 1765 | } /* ** Insert a record into the %_segdir table. */ static int fts3WriteSegdir( Fts3Table *p, /* Virtual table handle */ | | | | 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 | } /* ** Insert a record into the %_segdir table. */ static int fts3WriteSegdir( Fts3Table *p, /* Virtual table handle */ sqlite3_int64 iLevel, /* Value for "level" field (absolute level) */ int iIdx, /* Value for "idx" field */ sqlite3_int64 iStartBlock, /* Value for "start_block" field */ sqlite3_int64 iLeafEndBlock, /* Value for "leaves_end_block" field */ sqlite3_int64 iEndBlock, /* Value for "end_block" field */ char *zRoot, /* Blob value for "root" field */ int nRoot /* Number of bytes in buffer zRoot */ ){ sqlite3_stmt *pStmt; int rc = fts3SqlStmt(p, SQL_INSERT_SEGDIR, &pStmt, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int64(pStmt, 1, iLevel); sqlite3_bind_int(pStmt, 2, iIdx); sqlite3_bind_int64(pStmt, 3, iStartBlock); sqlite3_bind_int64(pStmt, 4, iLeafEndBlock); sqlite3_bind_int64(pStmt, 5, iEndBlock); sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC); sqlite3_step(pStmt); rc = sqlite3_reset(pStmt); |
︙ | ︙ | |||
2153 2154 2155 2156 2157 2158 2159 | ** database. This function must be called after all terms have been added ** to the segment using fts3SegWriterAdd(). If successful, SQLITE_OK is ** returned. Otherwise, an SQLite error code. */ static int fts3SegWriterFlush( Fts3Table *p, /* Virtual table handle */ SegmentWriter *pWriter, /* SegmentWriter to flush to the db */ | | | 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 | ** database. This function must be called after all terms have been added ** to the segment using fts3SegWriterAdd(). If successful, SQLITE_OK is ** returned. Otherwise, an SQLite error code. */ static int fts3SegWriterFlush( Fts3Table *p, /* Virtual table handle */ SegmentWriter *pWriter, /* SegmentWriter to flush to the db */ sqlite3_int64 iLevel, /* Value for 'level' column of %_segdir */ int iIdx /* Value for 'idx' column of %_segdir */ ){ int rc; /* Return code */ if( pWriter->pTree ){ sqlite3_int64 iLast = 0; /* Largest block id written to database */ sqlite3_int64 iLastLeaf; /* Largest leaf block id written to db */ char *zRoot = NULL; /* Pointer to buffer containing root node */ |
︙ | ︙ | |||
2235 2236 2237 2238 2239 2240 2241 | ** ** Return SQLITE_OK if successful, or an SQLite error code if not. */ static int fts3SegmentMaxLevel( Fts3Table *p, int iLangid, int iIndex, | | | | | | 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 | ** ** Return SQLITE_OK if successful, or an SQLite error code if not. */ static int fts3SegmentMaxLevel( Fts3Table *p, int iLangid, int iIndex, sqlite3_int64 *pnMax ){ sqlite3_stmt *pStmt; int rc; assert( iIndex>=0 && iIndex<p->nIndex ); /* Set pStmt to the compiled version of: ** ** SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? ** ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR). */ rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0); if( rc!=SQLITE_OK ) return rc; sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); sqlite3_bind_int64(pStmt, 2, getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) ); if( SQLITE_ROW==sqlite3_step(pStmt) ){ *pnMax = sqlite3_column_int64(pStmt, 0); } return sqlite3_reset(pStmt); } /* ** This function is used after merging multiple segments into a single large ** segment to delete the old, now redundant, segment b-trees. Specifically, |
︙ | ︙ | |||
2303 2304 2305 2306 2307 2308 2309 | return rc; } assert( iLevel>=0 || iLevel==FTS3_SEGCURSOR_ALL ); if( iLevel==FTS3_SEGCURSOR_ALL ){ rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_RANGE, &pDelete, 0); if( rc==SQLITE_OK ){ | | | > | > | 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 | return rc; } assert( iLevel>=0 || iLevel==FTS3_SEGCURSOR_ALL ); if( iLevel==FTS3_SEGCURSOR_ALL ){ rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_RANGE, &pDelete, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int64(pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); sqlite3_bind_int64(pDelete, 2, getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) ); } }else{ rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pDelete, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int64( pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex, iLevel) ); } } if( rc==SQLITE_OK ){ sqlite3_step(pDelete); rc = sqlite3_reset(pDelete); } |
︙ | ︙ | |||
2788 2789 2790 2791 2792 2793 2794 | Fts3Table *p, int iLangid, /* Language id to merge */ int iIndex, /* Index in p->aIndex[] to merge */ int iLevel /* Level to merge */ ){ int rc; /* Return code */ int iIdx = 0; /* Index of new segment */ | | | 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829 2830 | Fts3Table *p, int iLangid, /* Language id to merge */ int iIndex, /* Index in p->aIndex[] to merge */ int iLevel /* Level to merge */ ){ int rc; /* Return code */ int iIdx = 0; /* Index of new segment */ sqlite3_int64 iNewLevel = 0; /* Level/index to create new segment at */ SegmentWriter *pWriter = 0; /* Used to write the new, merged, segment */ Fts3SegFilter filter; /* Segment term filter condition */ Fts3MultiSegReader csr; /* Cursor to iterate through level(s) */ int bIgnoreEmpty = 0; /* True to ignore empty segments */ assert( iLevel==FTS3_SEGCURSOR_ALL || iLevel==FTS3_SEGCURSOR_PENDING |
︙ | ︙ |
Changes to test/fts4langid.test.
︙ | ︙ | |||
377 378 379 380 381 382 383 384 385 | do_execsql_test 4.1.4.$i { SELECT count(*) FROM t4 WHERE t4 MATCH 'fox' AND lid=$i; } [expr 0==($i%2)] } do_catchsql_test 4.1.5 { INSERT INTO t4(content, lid) VALUES('hello world', 101) } {1 {SQL logic error or missing database}} finish_test | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 | do_execsql_test 4.1.4.$i { SELECT count(*) FROM t4 WHERE t4 MATCH 'fox' AND lid=$i; } [expr 0==($i%2)] } do_catchsql_test 4.1.5 { INSERT INTO t4(content, lid) VALUES('hello world', 101) } {1 {SQL logic error or missing database}} #------------------------------------------------------------------------- # Test cases 5.* # # The following test cases are designed to detect a 32-bit overflow bug # that existed at one point. # proc build_multilingual_db_3 {db} { $db eval { CREATE VIRTUAL TABLE t5 USING fts4(languageid=lid); } set languages [list 0 1 2 [expr 1<<30]] foreach lid $languages { execsql { INSERT INTO t5(docid, content, lid) VALUES( $lid, 'My language is ' || $lid, $lid ) } } } do_test 5.1.0 { reset_db build_multilingual_db_3 db } {} do_execsql_test 5.1.1 { SELECT level FROM t5_segdir; } [list 0 1024 2048 [expr 1<<40]] do_execsql_test 5.1.2 {SELECT docid FROM t5 WHERE t5 MATCH 'language'} 0 foreach langid [list 0 1 2 [expr 1<<30]] { do_execsql_test 5.2.$langid { SELECT docid FROM t5 WHERE t5 MATCH 'language' AND lid = $langid } $langid } set lid [expr 1<<30] do_execsql_test 5.3.1 { CREATE VIRTUAL TABLE t6 USING fts4(languageid=lid); INSERT INTO t6 VALUES('I belong to language 0!'); } do_test 5.3.2 { for {set i 0} {$i < 20} {incr i} { execsql { INSERT INTO t6(content, lid) VALUES( 'I (row '||$i||') belong to langauge N!', $lid ); } } execsql { SELECT docid FROM t6 WHERE t6 MATCH 'belong' } } {1} do_test 5.3.3 { execsql { SELECT docid FROM t6 WHERE t6 MATCH 'belong' AND lid=$lid} } {2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21} do_execsql_test 5.3.4 { INSERT INTO t6(t6) VALUES('optimize') } {} do_execsql_test 5.3.5 { SELECT docid FROM t6 WHERE t6 MATCH 'belong' } {1} do_execsql_test 5.3.6 { SELECT docid FROM t6 WHERE t6 MATCH 'belong' AND lid=$lid } {2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21} set lid [expr 1<<30] foreach lid [list 4 [expr 1<<30]] { do_execsql_test 5.4.$lid.1 { DELETE FROM t6; SELECT count(*) FROM t6_segdir; SELECT count(*) FROM t6_segments; } {0 0} do_execsql_test 5.4.$lid.2 { INSERT INTO t6(content, lid) VALUES('zero zero zero', $lid); INSERT INTO t6(content, lid) VALUES('zero zero one', $lid); INSERT INTO t6(content, lid) VALUES('zero one zero', $lid); INSERT INTO t6(content, lid) VALUES('zero one one', $lid); INSERT INTO t6(content, lid) VALUES('one zero zero', $lid); INSERT INTO t6(content, lid) VALUES('one zero one', $lid); INSERT INTO t6(content, lid) VALUES('one one zero', $lid); INSERT INTO t6(content, lid) VALUES('one one one', $lid); SELECT docid FROM t6 WHERE t6 MATCH '"zero zero"' AND lid=$lid; } {1 2 5} do_execsql_test 5.4.$lid.3 { SELECT count(*) FROM t6_segdir; SELECT count(*) FROM t6_segments; } {8 0} do_execsql_test 5.4.$lid.4 { INSERT INTO t6(t6) VALUES('optimize'); SELECT docid FROM t6 WHERE t6 MATCH '"zero zero"' AND lid=$lid; } {1 2 5} do_execsql_test 5.4.$lid.5 { SELECT count(*) FROM t6_segdir; SELECT count(*) FROM t6_segments; } {1 0} } finish_test |