/ Check-in [3475092c]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix some integer overflow problems that can occur when using large langauge id values.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts4-incr-merge
Files: files | file ages | folders
SHA1: 3475092cff862080a020d386076d739f0d22c9b2
User & Date: dan 2012-03-16 15:54:19
References
2012-03-16
16:52
Cherrypick the [3475092cff] fix for 32-bit overflow with large language-ids into trunk. check-in: 2755edc7 user: drh tags: trunk
Context
2012-03-17
16:56
Fix various incorrect and missing comments and other style issues in and around the FTS incremental merge code. check-in: 7aabb62c user: dan tags: fts4-incr-merge
2012-03-16
15:54
Fix some integer overflow problems that can occur when using large langauge id values. check-in: 3475092c user: dan tags: fts4-incr-merge
14:54
Add a comment to the FTS getAbsoluteLevel() function. No actual code changes. check-in: 7e0f861b user: dan tags: fts4-incr-merge
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts3/fts3_write.c.

   506    506   */
   507    507   static sqlite3_int64 getAbsoluteLevel(
   508    508     Fts3Table *p,                   /* FTS3 table handle */
   509    509     int iLangid,                    /* Language id */
   510    510     int iIndex,                     /* Index in p->aIndex[] */
   511    511     int iLevel                      /* Level of segments */
   512    512   ){
          513  +  sqlite3_int64 iBase;            /* First absolute level for iLangid/iIndex */
   513    514     assert( iLangid>=0 );
   514    515     assert( p->nIndex>0 );
   515    516     assert( iIndex>=0 && iIndex<p->nIndex );
   516         -  return (iLangid * p->nIndex + iIndex) * FTS3_SEGDIR_MAXLEVEL + iLevel;
          517  +
          518  +  iBase = ((sqlite3_int64)iLangid * p->nIndex + iIndex) * FTS3_SEGDIR_MAXLEVEL;
          519  +  return iBase + iLevel;
   517    520   }
   518    521   
   519    522   /*
   520    523   ** Given an absolute level number, determine the langauge-id, index
   521    524   ** and relative level that it corresponds to.
   522    525   **
   523    526   ** The return value is the relative level. The language-id and index
................................................................................
   552    555   **   3: end_block
   553    556   **   4: root
   554    557   */
   555    558   int sqlite3Fts3AllSegdirs(
   556    559     Fts3Table *p,                   /* FTS3 table */
   557    560     int iLangid,                    /* Language being queried */
   558    561     int iIndex,                     /* Index for p->aIndex[] */
   559         -  int iLevel,                     /* Level to select */
          562  +  int iLevel,                     /* Level to select (relative level) */
   560    563     sqlite3_stmt **ppStmt           /* OUT: Compiled statement */
   561    564   ){
   562    565     int rc;
   563    566     sqlite3_stmt *pStmt = 0;
   564    567   
   565    568     assert( iLevel==FTS3_SEGCURSOR_ALL || iLevel>=0 );
   566    569     assert( iLevel<FTS3_SEGDIR_MAXLEVEL );
................................................................................
   567    570     assert( iIndex>=0 && iIndex<p->nIndex );
   568    571   
   569    572     if( iLevel<0 ){
   570    573       /* "SELECT * FROM %_segdir WHERE level BETWEEN ? AND ? ORDER BY ..." */
   571    574       rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE, &pStmt, 0);
   572    575       if( rc==SQLITE_OK ){ 
   573    576         sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0));
   574         -      sqlite3_bind_int(pStmt, 2, 
          577  +      sqlite3_bind_int64(pStmt, 2, 
   575    578             getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1)
   576    579         );
   577    580       }
   578    581     }else{
   579    582       /* "SELECT * FROM %_segdir WHERE level = ? ORDER BY ..." */
   580    583       rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0);
   581    584       if( rc==SQLITE_OK ){ 
   582         -      sqlite3_bind_int(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, iLevel));
          585  +      sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex,iLevel));
   583    586       }
   584    587     }
   585    588     *ppStmt = pStmt;
   586    589     return rc;
   587    590   }
   588    591   
   589    592   
................................................................................
  1850   1853   }
  1851   1854   
  1852   1855   /* 
  1853   1856   ** Insert a record into the %_segdir table.
  1854   1857   */
  1855   1858   static int fts3WriteSegdir(
  1856   1859     Fts3Table *p,                   /* Virtual table handle */
  1857         -  int iLevel,                     /* Value for "level" field */
         1860  +  sqlite3_int64 iLevel,           /* Value for "level" field (absolute level) */
  1858   1861     int iIdx,                       /* Value for "idx" field */
  1859   1862     sqlite3_int64 iStartBlock,      /* Value for "start_block" field */
  1860   1863     sqlite3_int64 iLeafEndBlock,    /* Value for "leaves_end_block" field */
  1861   1864     sqlite3_int64 iEndBlock,        /* Value for "end_block" field */
  1862   1865     char *zRoot,                    /* Blob value for "root" field */
  1863   1866     int nRoot                       /* Number of bytes in buffer zRoot */
  1864   1867   ){
  1865   1868     sqlite3_stmt *pStmt;
  1866   1869     int rc = fts3SqlStmt(p, SQL_INSERT_SEGDIR, &pStmt, 0);
  1867   1870     if( rc==SQLITE_OK ){
  1868         -    sqlite3_bind_int(pStmt, 1, iLevel);
         1871  +    sqlite3_bind_int64(pStmt, 1, iLevel);
  1869   1872       sqlite3_bind_int(pStmt, 2, iIdx);
  1870   1873       sqlite3_bind_int64(pStmt, 3, iStartBlock);
  1871   1874       sqlite3_bind_int64(pStmt, 4, iLeafEndBlock);
  1872   1875       sqlite3_bind_int64(pStmt, 5, iEndBlock);
  1873   1876       sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC);
  1874   1877       sqlite3_step(pStmt);
  1875   1878       rc = sqlite3_reset(pStmt);
................................................................................
  2244   2247   ** database. This function must be called after all terms have been added
  2245   2248   ** to the segment using fts3SegWriterAdd(). If successful, SQLITE_OK is
  2246   2249   ** returned. Otherwise, an SQLite error code.
  2247   2250   */
  2248   2251   static int fts3SegWriterFlush(
  2249   2252     Fts3Table *p,                   /* Virtual table handle */
  2250   2253     SegmentWriter *pWriter,         /* SegmentWriter to flush to the db */
  2251         -  int iLevel,                     /* Value for 'level' column of %_segdir */
         2254  +  sqlite3_int64 iLevel,           /* Value for 'level' column of %_segdir */
  2252   2255     int iIdx                        /* Value for 'idx' column of %_segdir */
  2253   2256   ){
  2254   2257     int rc;                         /* Return code */
  2255   2258     if( pWriter->pTree ){
  2256   2259       sqlite3_int64 iLast = 0;      /* Largest block id written to database */
  2257   2260       sqlite3_int64 iLastLeaf;      /* Largest leaf block id written to db */
  2258   2261       char *zRoot = NULL;           /* Pointer to buffer containing root node */
................................................................................
  2326   2329   **
  2327   2330   ** Return SQLITE_OK if successful, or an SQLite error code if not.
  2328   2331   */
  2329   2332   static int fts3SegmentMaxLevel(
  2330   2333     Fts3Table *p, 
  2331   2334     int iLangid,
  2332   2335     int iIndex, 
  2333         -  int *pnMax
         2336  +  sqlite3_int64 *pnMax
  2334   2337   ){
  2335   2338     sqlite3_stmt *pStmt;
  2336   2339     int rc;
  2337   2340     assert( iIndex>=0 && iIndex<p->nIndex );
  2338   2341   
  2339   2342     /* Set pStmt to the compiled version of:
  2340   2343     **
  2341   2344     **   SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?
  2342   2345     **
  2343   2346     ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR).
  2344   2347     */
  2345   2348     rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0);
  2346   2349     if( rc!=SQLITE_OK ) return rc;
  2347         -  sqlite3_bind_int(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0));
  2348         -  sqlite3_bind_int(pStmt, 2, 
         2350  +  sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0));
         2351  +  sqlite3_bind_int64(pStmt, 2, 
  2349   2352         getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1)
  2350   2353     );
  2351   2354     if( SQLITE_ROW==sqlite3_step(pStmt) ){
  2352         -    *pnMax = sqlite3_column_int(pStmt, 0);
         2355  +    *pnMax = sqlite3_column_int64(pStmt, 0);
  2353   2356     }
  2354   2357     return sqlite3_reset(pStmt);
  2355   2358   }
  2356   2359   
  2357   2360   /*
  2358   2361   ** Delete all entries in the %_segments table associated with the segment
  2359   2362   ** opened with seg-reader pSeg. This function does not affect the contents
................................................................................
  2410   2413       return rc;
  2411   2414     }
  2412   2415   
  2413   2416     assert( iLevel>=0 || iLevel==FTS3_SEGCURSOR_ALL );
  2414   2417     if( iLevel==FTS3_SEGCURSOR_ALL ){
  2415   2418       rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_RANGE, &pDelete, 0);
  2416   2419       if( rc==SQLITE_OK ){
  2417         -      sqlite3_bind_int(pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex, 0));
  2418         -      sqlite3_bind_int(pDelete, 2, 
         2420  +      sqlite3_bind_int64(pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex, 0));
         2421  +      sqlite3_bind_int64(pDelete, 2, 
  2419   2422             getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1)
  2420   2423         );
  2421   2424       }
  2422   2425     }else{
  2423   2426       rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pDelete, 0);
  2424   2427       if( rc==SQLITE_OK ){
  2425         -      sqlite3_bind_int(pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex,iLevel));
         2428  +      sqlite3_bind_int64(
         2429  +          pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex, iLevel)
         2430  +      );
  2426   2431       }
  2427   2432     }
  2428   2433   
  2429   2434     if( rc==SQLITE_OK ){
  2430   2435       sqlite3_step(pDelete);
  2431   2436       rc = sqlite3_reset(pDelete);
  2432   2437     }
................................................................................
  2895   2900     Fts3Table *p, 
  2896   2901     int iLangid,                    /* Language id to merge */
  2897   2902     int iIndex,                     /* Index in p->aIndex[] to merge */
  2898   2903     int iLevel                      /* Level to merge */
  2899   2904   ){
  2900   2905     int rc;                         /* Return code */
  2901   2906     int iIdx = 0;                   /* Index of new segment */
  2902         -  int iNewLevel = 0;              /* Level/index to create new segment at */
         2907  +  sqlite3_int64 iNewLevel = 0;    /* Level/index to create new segment at */
  2903   2908     SegmentWriter *pWriter = 0;     /* Used to write the new, merged, segment */
  2904   2909     Fts3SegFilter filter;           /* Segment term filter condition */
  2905   2910     Fts3MultiSegReader csr;         /* Cursor to iterate through level(s) */
  2906   2911     int bIgnoreEmpty = 0;           /* True to ignore empty segments */
  2907   2912   
  2908   2913     assert( iLevel==FTS3_SEGCURSOR_ALL
  2909   2914          || iLevel==FTS3_SEGCURSOR_PENDING
................................................................................
  3860   3865     sqlite3_stmt *pFirstBlock = 0;  /* SQL used to determine first block */
  3861   3866     sqlite3_stmt *pOutputIdx = 0;   /* SQL used to find output index */
  3862   3867     const char *zKey = pCsr->zTerm;
  3863   3868     int nKey = pCsr->nTerm;
  3864   3869   
  3865   3870     rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pOutputIdx, 0);
  3866   3871     if( rc==SQLITE_OK ){
  3867         -    sqlite3_bind_int(pOutputIdx, 1, iAbsLevel+1);
         3872  +    sqlite3_bind_int64(pOutputIdx, 1, iAbsLevel+1);
  3868   3873       sqlite3_step(pOutputIdx);
  3869   3874       iIdx = sqlite3_column_int(pOutputIdx, 0) - 1;
  3870   3875       rc = sqlite3_reset(pOutputIdx);
  3871   3876     }
  3872   3877     if( rc!=SQLITE_OK ) return rc;
  3873   3878   
  3874   3879     assert( zKey );

Changes to test/fts4langid.test.

   377    377     do_execsql_test 4.1.4.$i {
   378    378       SELECT count(*) FROM t4 WHERE t4 MATCH 'fox' AND lid=$i;
   379    379     } [expr 0==($i%2)]
   380    380   }
   381    381   do_catchsql_test 4.1.5 {
   382    382     INSERT INTO t4(content, lid) VALUES('hello world', 101)
   383    383   } {1 {SQL logic error or missing database}}
          384  +
          385  +#-------------------------------------------------------------------------
          386  +# Test cases 5.*
          387  +#
          388  +# The following test cases are designed to detect a 32-bit overflow bug
          389  +# that existed at one point.
          390  +#
          391  +proc build_multilingual_db_3 {db} {
          392  +  $db eval {
          393  +    CREATE VIRTUAL TABLE t5 USING fts4(languageid=lid);
          394  +  }
          395  +  set languages [list 0 1 2 [expr 1<<30]]
          396  +
          397  +  foreach lid $languages {
          398  +    execsql {
          399  +      INSERT INTO t5(docid, content, lid) VALUES(
          400  +          $lid, 'My language is ' || $lid, $lid
          401  +      ) 
          402  +    }
          403  +  }
          404  +}
          405  +
          406  +do_test 5.1.0 {
          407  +  reset_db
          408  +  build_multilingual_db_3 db
          409  +} {}
          410  +
          411  +do_execsql_test 5.1.1 {
          412  +  SELECT level FROM t5_segdir;
          413  +} [list 0 1024 2048 [expr 1<<40]]
          414  +
          415  +do_execsql_test 5.1.2 {SELECT docid FROM t5 WHERE t5 MATCH 'language'} 0
          416  +foreach langid [list 0 1 2 [expr 1<<30]] {
          417  +  do_execsql_test 5.2.$langid { 
          418  +    SELECT docid FROM t5 WHERE t5 MATCH 'language' AND lid = $langid
          419  +  } $langid
          420  +}
          421  +
          422  +set lid [expr 1<<30]
          423  +do_execsql_test 5.3.1 {
          424  +  CREATE VIRTUAL TABLE t6 USING fts4(languageid=lid);
          425  +  INSERT INTO t6 VALUES('I belong to language 0!');
          426  +}
          427  +do_test 5.3.2 {
          428  +  for {set i 0} {$i < 20} {incr i} {
          429  +    execsql {
          430  +      INSERT INTO t6(content, lid) VALUES(
          431  +        'I (row '||$i||') belong to langauge N!', $lid
          432  +      );
          433  +    }
          434  +  }
          435  +  execsql { SELECT docid FROM t6 WHERE t6 MATCH 'belong' }
          436  +} {1}
          437  +
          438  +do_test 5.3.3 {
          439  +  execsql { SELECT docid FROM t6 WHERE t6 MATCH 'belong' AND lid=$lid}
          440  +} {2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21}
          441  +
          442  +do_execsql_test 5.3.4 { INSERT INTO t6(t6) VALUES('optimize') } {}
          443  +do_execsql_test 5.3.5 { SELECT docid FROM t6 WHERE t6 MATCH 'belong' } {1}
          444  +do_execsql_test 5.3.6 { 
          445  +  SELECT docid FROM t6 WHERE t6 MATCH 'belong' AND lid=$lid
          446  +} {2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21}
          447  +
          448  +
          449  +set lid [expr 1<<30]
          450  +foreach lid [list 4 [expr 1<<30]] {
          451  +  do_execsql_test 5.4.$lid.1 {
          452  +    DELETE FROM t6;
          453  +    SELECT count(*) FROM t6_segdir;
          454  +    SELECT count(*) FROM t6_segments;
          455  +  } {0 0}
          456  +  do_execsql_test 5.4.$lid.2 {
          457  +    INSERT INTO t6(content, lid) VALUES('zero zero zero', $lid);
          458  +    INSERT INTO t6(content, lid) VALUES('zero zero one', $lid);
          459  +    INSERT INTO t6(content, lid) VALUES('zero one zero', $lid);
          460  +    INSERT INTO t6(content, lid) VALUES('zero one one', $lid);
          461  +    INSERT INTO t6(content, lid) VALUES('one zero zero', $lid);
          462  +    INSERT INTO t6(content, lid) VALUES('one zero one', $lid);
          463  +    INSERT INTO t6(content, lid) VALUES('one one zero', $lid);
          464  +    INSERT INTO t6(content, lid) VALUES('one one one', $lid);
          465  +
          466  +    SELECT docid FROM t6 WHERE t6 MATCH '"zero zero"' AND lid=$lid;
          467  +  } {1 2 5}
          468  +
          469  +  do_execsql_test 5.4.$lid.3 {
          470  +    SELECT count(*) FROM t6_segdir;
          471  +    SELECT count(*) FROM t6_segments;
          472  +  } {8 0}
          473  +
          474  +  do_execsql_test 5.4.$lid.4 {
          475  +    INSERT INTO t6(t6) VALUES('merge=100,3');
          476  +    INSERT INTO t6(t6) VALUES('merge=100,3');
          477  +    SELECT docid FROM t6 WHERE t6 MATCH '"zero zero"' AND lid=$lid;
          478  +  } {1 2 5}
          479  +
          480  +  do_execsql_test 5.4.$lid.5 {
          481  +    SELECT count(*) FROM t6_segdir;
          482  +    SELECT count(*) FROM t6_segments;
          483  +  } {4 4}
          484  +}
          485  +
   384    486   
   385    487   finish_test