/ Check-in [e22252e1]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix an fts5 problem with corrupt database handling found by address-sanitizer.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: e22252e1da4cd9e41b970970a1c4f466aa6cc133
User & Date: dan 2016-08-13 10:34:12
Context
2016-08-16
16:46
Fix a bug in destructor processing of Lemon. That has no impact on the SQLite grammar. The bug was introduced by prior work to optimize the Lemon-generated parser used by SQLite. check-in: f9035b8e user: drh tags: trunk
2016-08-13
10:34
Fix an fts5 problem with corrupt database handling found by address-sanitizer. check-in: e22252e1 user: dan tags: trunk
06:38
Fix a buffer overread in fts5. check-in: fcfbee6c user: dan tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5_index.c.

   698    698       p->nRead++;
   699    699     }
   700    700   
   701    701     assert( (pRet==0)==(p->rc!=SQLITE_OK) );
   702    702     return pRet;
   703    703   }
   704    704   
   705         -
   706    705   /*
   707    706   ** Release a reference to data record returned by an earlier call to
   708    707   ** fts5DataRead().
   709    708   */
   710    709   static void fts5DataRelease(Fts5Data *pData){
   711    710     sqlite3_free(pData);
   712    711   }
          712  +
          713  +static Fts5Data *fts5LeafRead(Fts5Index *p, i64 iRowid){
          714  +  Fts5Data *pRet = fts5DataRead(p, iRowid);
          715  +  if( pRet ){
          716  +    if( pRet->szLeaf>pRet->nn ){
          717  +      p->rc = FTS5_CORRUPT;
          718  +      fts5DataRelease(pRet);
          719  +      pRet = 0;
          720  +    }
          721  +  }
          722  +  return pRet;
          723  +}
   713    724   
   714    725   static int fts5IndexPrepareStmt(
   715    726     Fts5Index *p,
   716    727     sqlite3_stmt **ppStmt,
   717    728     char *zSql
   718    729   ){
   719    730     if( p->rc==SQLITE_OK ){
................................................................................
  1515   1526     Fts5StructureSegment *pSeg = pIter->pSeg;
  1516   1527     fts5DataRelease(pIter->pLeaf);
  1517   1528     pIter->iLeafPgno++;
  1518   1529     if( pIter->pNextLeaf ){
  1519   1530       pIter->pLeaf = pIter->pNextLeaf;
  1520   1531       pIter->pNextLeaf = 0;
  1521   1532     }else if( pIter->iLeafPgno<=pSeg->pgnoLast ){
  1522         -    pIter->pLeaf = fts5DataRead(p, 
         1533  +    pIter->pLeaf = fts5LeafRead(p, 
  1523   1534           FTS5_SEGMENT_ROWID(pSeg->iSegid, pIter->iLeafPgno)
  1524   1535       );
  1525   1536     }else{
  1526   1537       pIter->pLeaf = 0;
  1527   1538     }
  1528   1539     pLeaf = pIter->pLeaf;
  1529   1540   
................................................................................
  2018   2029         if( (iOff = fts5LeafFirstRowidOff(pLeaf)) && iOff<pLeaf->szLeaf ){
  2019   2030           iOff += sqlite3Fts5GetVarint(&pLeaf->p[iOff], (u64*)&pIter->iRowid);
  2020   2031           pIter->iLeafOffset = iOff;
  2021   2032   
  2022   2033           if( pLeaf->nn>pLeaf->szLeaf ){
  2023   2034             pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32(
  2024   2035                 &pLeaf->p[pLeaf->szLeaf], pIter->iEndofDoclist
  2025         -              );
         2036  +          );
  2026   2037           }
  2027         -
  2028   2038         }
  2029   2039         else if( pLeaf->nn>pLeaf->szLeaf ){
  2030   2040           pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32(
  2031   2041               &pLeaf->p[pLeaf->szLeaf], iOff
  2032   2042               );
  2033   2043           pIter->iLeafOffset = iOff;
  2034   2044           pIter->iEndofDoclist = iOff;
................................................................................
  2264   2274         bEndOfPage = 1;
  2265   2275         break;
  2266   2276       }
  2267   2277   
  2268   2278       iPgidx += fts5GetVarint32(&a[iPgidx], nKeep);
  2269   2279       iTermOff += nKeep;
  2270   2280       iOff = iTermOff;
         2281  +
         2282  +    if( iOff>=n ){
         2283  +      p->rc = FTS5_CORRUPT;
         2284  +      return;
         2285  +    }
  2271   2286   
  2272   2287       /* Read the nKeep field of the next term. */
  2273   2288       fts5FastGetVarint32(a, iOff, nKeep);
  2274   2289     }
  2275   2290   
  2276   2291    search_failed:
  2277   2292     if( bGe==0 ){

Changes to ext/fts5/test/fts5corrupt2.test.

    33     33     CREATE VIRTUAL TABLE t1 USING fts5(x);
    34     34     INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
    35     35     WITH ii(i) AS (SELECT 1 UNION SELECT i+1 FROM ii WHERE i<100)
    36     36     INSERT INTO t1 SELECT rnddoc(10) FROM ii;
    37     37   }
    38     38   set mask [expr 31 << 31]
    39     39   
    40         -if 1 {
           40  +if 0 {
    41     41   
    42     42   # Test 1:
    43     43   #
    44     44   #   For each page in the t1_data table, open a transaction and DELETE
    45     45   #   the t1_data entry. Then run:
    46     46   #
    47     47   #     * an integrity-check, and
................................................................................
    78     78       }
    79     79     
    80     80       do_execsql_test 1.$tno.$tn.3.$rowid {
    81     81         ROLLBACK;
    82     82         INSERT INTO t1(t1) VALUES('integrity-check');
    83     83       } {}
    84     84     }
           85  +}
           86  +
    85     87   }
    86     88   
    87     89   # Using the same database as the 1.* tests.
    88     90   #
    89     91   # Run N-1 tests, where N is the number of bytes in the rightmost leaf page
    90     92   # of the fts index. For test $i, truncate the rightmost leafpage to $i
    91     93   # bytes. Then test both the integrity-check detects the corruption.
................................................................................
   206    208         set {} 1
   207    209       } {1}
   208    210   
   209    211       execsql ROLLBACK
   210    212     }
   211    213   
   212    214     # do_test 4.$tn.x { expr $nCorrupt>0 } 1
   213         -}
   214         -
   215    215   }
   216    216   
   217    217   set doc [string repeat "A B C " 1000]
   218    218   do_execsql_test 5.0 {
   219    219     CREATE VIRTUAL TABLE x5 USING fts5(tt);
   220    220     INSERT INTO x5(x5, rank) VALUES('pgsz', 32);
   221    221     WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<10)