/ Check-in [940f2adc]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add extra defenses against strategically corrupt databases to fts3/4.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | branch-3.25
Files: files | file ages | folders
SHA3-256:940f2adc8541a838c19ceeb62105074117c906efa0f36610a0d3e8d6f859154f
User & Date: drh 2018-11-05 13:43:03
Context
2018-11-05
13:48
Disable the IS NOT NULL optimization when the IS NOT NULL operator is part of the ON clause of a LEFT JOIN. Fix for ticket [65eb38f6e46de8c75e188a17ec]. check-in: 8d09ce5d user: drh tags: branch-3.25
13:43
Add extra defenses against strategically corrupt databases to fts3/4. check-in: 940f2adc user: drh tags: branch-3.25
13:37
Increase the version number to 3.25.3. check-in: 1250ab8f user: drh tags: branch-3.25
2018-11-03
16:51
Add extra defenses against strategically corrupt databases to fts3/4. check-in: d44318f5 user: dan tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts3/fts3.c.

  1817   1817     sqlite3_int64 *piFirst,         /* OUT: Selected child node */
  1818   1818     sqlite3_int64 *piLast           /* OUT: Selected child node */
  1819   1819   ){
  1820   1820     int rc = SQLITE_OK;             /* Return code */
  1821   1821     const char *zCsr = zNode;       /* Cursor to iterate through node */
  1822   1822     const char *zEnd = &zCsr[nNode];/* End of interior node buffer */
  1823   1823     char *zBuffer = 0;              /* Buffer to load terms into */
  1824         -  int nAlloc = 0;                 /* Size of allocated buffer */
         1824  +  i64 nAlloc = 0;                 /* Size of allocated buffer */
  1825   1825     int isFirstTerm = 1;            /* True when processing first term on page */
  1826   1826     sqlite3_int64 iChild;           /* Block id of child node to descend to */
  1827   1827   
  1828   1828     /* Skip over the 'height' varint that occurs at the start of every 
  1829   1829     ** interior node. Then load the blockid of the left-child of the b-tree
  1830   1830     ** node into variable iChild.  
  1831   1831     **
................................................................................
  1855   1855       if( !isFirstTerm ){
  1856   1856         zCsr += fts3GetVarint32(zCsr, &nPrefix);
  1857   1857       }
  1858   1858       isFirstTerm = 0;
  1859   1859       zCsr += fts3GetVarint32(zCsr, &nSuffix);
  1860   1860       
  1861   1861       assert( nPrefix>=0 && nSuffix>=0 );
  1862         -    if( &zCsr[nSuffix]>zEnd ){
         1862  +    if( nPrefix>zCsr-zNode || nSuffix>zEnd-zCsr ){
  1863   1863         rc = FTS_CORRUPT_VTAB;
  1864   1864         goto finish_scan;
  1865   1865       }
  1866         -    if( nPrefix+nSuffix>nAlloc ){
         1866  +    if( (i64)nPrefix+nSuffix>nAlloc ){
  1867   1867         char *zNew;
  1868         -      nAlloc = (nPrefix+nSuffix) * 2;
  1869         -      zNew = (char *)sqlite3_realloc(zBuffer, nAlloc);
         1868  +      nAlloc = ((i64)nPrefix+nSuffix) * 2;
         1869  +      zNew = (char *)sqlite3_realloc64(zBuffer, nAlloc);
  1870   1870         if( !zNew ){
  1871   1871           rc = SQLITE_NOMEM;
  1872   1872           goto finish_scan;
  1873   1873         }
  1874   1874         zBuffer = zNew;
  1875   1875       }
  1876   1876       assert( zBuffer );

Changes to ext/fts3/fts3_write.c.

  1370   1370     rc = fts3SegReaderRequire(pReader, pNext, FTS3_VARINT_MAX*2);
  1371   1371     if( rc!=SQLITE_OK ) return rc;
  1372   1372     
  1373   1373     /* Because of the FTS3_NODE_PADDING bytes of padding, the following is 
  1374   1374     ** safe (no risk of overread) even if the node data is corrupted. */
  1375   1375     pNext += fts3GetVarint32(pNext, &nPrefix);
  1376   1376     pNext += fts3GetVarint32(pNext, &nSuffix);
  1377         -  if( nPrefix<0 || nSuffix<=0 
  1378         -   || &pNext[nSuffix]>&pReader->aNode[pReader->nNode] 
         1377  +  if( nSuffix<=0 
         1378  +   || (&pReader->aNode[pReader->nNode] - pNext)<nSuffix
         1379  +   || nPrefix>pReader->nTermAlloc
  1379   1380     ){
  1380   1381       return FTS_CORRUPT_VTAB;
  1381   1382     }
  1382   1383   
  1383         -  if( nPrefix+nSuffix>pReader->nTermAlloc ){
  1384         -    int nNew = (nPrefix+nSuffix)*2;
  1385         -    char *zNew = sqlite3_realloc(pReader->zTerm, nNew);
         1384  +  /* Both nPrefix and nSuffix were read by fts3GetVarint32() and so are
         1385  +  ** between 0 and 0x7FFFFFFF. But the sum of the two may cause integer
         1386  +  ** overflow - hence the (i64) casts.  */
         1387  +  if( (i64)nPrefix+nSuffix>(i64)pReader->nTermAlloc ){
         1388  +    i64 nNew = ((i64)nPrefix+nSuffix)*2;
         1389  +    char *zNew = sqlite3_realloc64(pReader->zTerm, nNew);
  1386   1390       if( !zNew ){
  1387   1391         return SQLITE_NOMEM;
  1388   1392       }
  1389   1393       pReader->zTerm = zNew;
  1390   1394       pReader->nTermAlloc = nNew;
  1391   1395     }
  1392   1396   
................................................................................
  1400   1404     pReader->aDoclist = pNext;
  1401   1405     pReader->pOffsetList = 0;
  1402   1406   
  1403   1407     /* Check that the doclist does not appear to extend past the end of the
  1404   1408     ** b-tree node. And that the final byte of the doclist is 0x00. If either 
  1405   1409     ** of these statements is untrue, then the data structure is corrupt.
  1406   1410     */
  1407         -  if( &pReader->aDoclist[pReader->nDoclist]>&pReader->aNode[pReader->nNode] 
         1411  +  if( (&pReader->aNode[pReader->nNode] - pReader->aDoclist)<pReader->nDoclist
  1408   1412      || (pReader->nPopulate==0 && pReader->aDoclist[pReader->nDoclist-1])
  1409   1413     ){
  1410   1414       return FTS_CORRUPT_VTAB;
  1411   1415     }
  1412   1416     return SQLITE_OK;
  1413   1417   }
  1414   1418   
................................................................................
  3726   3730       p->aNode = 0;
  3727   3731     }else{
  3728   3732       if( bFirst==0 ){
  3729   3733         p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nPrefix);
  3730   3734       }
  3731   3735       p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nSuffix);
  3732   3736   
         3737  +    if( nPrefix>p->iOff || nSuffix>p->nNode-p->iOff ){
         3738  +      return SQLITE_CORRUPT_VTAB;
         3739  +    }
  3733   3740       blobGrowBuffer(&p->term, nPrefix+nSuffix, &rc);
  3734   3741       if( rc==SQLITE_OK ){
  3735   3742         memcpy(&p->term.a[nPrefix], &p->aNode[p->iOff], nSuffix);
  3736   3743         p->term.n = nPrefix+nSuffix;
  3737   3744         p->iOff += nSuffix;
  3738   3745         if( p->iChild==0 ){
  3739   3746           p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &p->nDoclist);
         3747  +        if( (p->nNode-p->iOff)<p->nDoclist ){
         3748  +          return SQLITE_CORRUPT_VTAB;
         3749  +        }
  3740   3750           p->aDoclist = &p->aNode[p->iOff];
  3741   3751           p->iOff += p->nDoclist;
  3742   3752         }
  3743   3753       }
  3744   3754     }
  3745   3755   
  3746   3756     assert( p->iOff<=p->nNode );
  3747         -
  3748   3757     return rc;
  3749   3758   }
  3750   3759   
  3751   3760   /*
  3752   3761   ** Release all dynamic resources held by node-reader object *p.
  3753   3762   */
  3754   3763   static void nodeReaderRelease(NodeReader *p){

Added test/fts3corrupt4.test.

            1  +# 2006 September 9
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this script is testing the FTS3 module.
           13  +#
           14  +# $Id: fts3aa.test,v 1.1 2007/08/20 17:38:42 shess Exp $
           15  +#
           16  +
           17  +set testdir [file dirname $argv0]
           18  +source $testdir/tester.tcl
           19  +set testprefix fts3corrupt4
           20  +
           21  +# If SQLITE_ENABLE_FTS3 is defined, omit this file.
           22  +ifcapable !fts3 {
           23  +  finish_test
           24  +  return
           25  +}
           26  +
           27  +do_execsql_test 1.0 {
           28  +  BEGIN;
           29  +    CREATE VIRTUAL TABLE ft USING fts3;
           30  +    INSERT INTO ft VALUES('aback');
           31  +    INSERT INTO ft VALUES('abaft');
           32  +    INSERT INTO ft VALUES('abandon');
           33  +  COMMIT;
           34  +}
           35  +
           36  +proc blob {a} { binary decode hex $a }
           37  +db func blob blob
           38  +
           39  +do_execsql_test 1.1 {
           40  +  SELECT quote(root) FROM ft_segdir;
           41  +} {X'0005616261636B03010200030266740302020003046E646F6E03030200'}
           42  +
           43  +do_execsql_test 1.2 {
           44  +  UPDATE ft_segdir SET root = blob(
           45  +    '0005616261636B03010200 FFFFFFFF0702 66740302020003046E646F6E03030200'
           46  +  );
           47  +}
           48  +
           49  +do_catchsql_test 1.3 {
           50  +  SELECT * FROM ft WHERE ft MATCH 'abandon';
           51  +} {1 {database disk image is malformed}}
           52  +
           53  +#-------------------------------------------------------------------------
           54  +reset_db
           55  +do_execsql_test 2.0.0 {
           56  +  CREATE VIRTUAL TABLE ft USING fts3;
           57  +  INSERT INTO ft(ft) VALUES('nodesize=32');
           58  +}
           59  +do_test 2.0.1 {
           60  +  for {set i 0} {$i < 12} {incr i} {
           61  +    execsql {
           62  +      BEGIN;
           63  +        INSERT INTO ft VALUES('abc' || $i);
           64  +        INSERT INTO ft VALUES('abc' || $i || 'x' );
           65  +        INSERT INTO ft VALUES('abc' || $i || 'xx' );
           66  +      COMMIT
           67  +    }
           68  +  }
           69  +  execsql {
           70  +    SELECT count(*) FROM ft_segdir;
           71  +    SELECT count(*) FROM ft_segments;
           72  +  }
           73  +} {12 0}
           74  +
           75  +do_execsql_test 2.1 {
           76  +  INSERT INTO ft(ft) VALUES('merge=1,4');
           77  +  SELECT count(*) FROM ft_segdir;
           78  +  SELECT count(*) FROM ft_segments;
           79  +} {12 3}
           80  +
           81  +do_execsql_test 2.2 {
           82  +  SELECT quote(block) FROM ft_segments WHERE blockid=2
           83  +} {X'00056162633130031F0200'}
           84  +
           85  +db func blob blob
           86  +do_execsql_test 2.3.1 {
           87  +  UPDATE ft_segments SET block = 
           88  +    blob('00056162633130031F0200 FFFFFFFF07FF55 66740302020003046E646F6E03030200')
           89  +    WHERE blockid=2;
           90  +} {}
           91  +do_catchsql_test 2.3.2 {
           92  +  INSERT INTO ft(ft) VALUES('merge=1,4');
           93  +} {1 {database disk image is malformed}}
           94  +
           95  +do_execsql_test 2.4.1 {
           96  +  UPDATE ft_segments SET block = 
           97  +    blob('00056162633130031F0200 02FFFFFFFF07 66740302020003046E646F6E03030200')
           98  +    WHERE blockid=2;
           99  +} {}
          100  +do_catchsql_test 2.4.2 {
          101  +  INSERT INTO ft(ft) VALUES('merge=1,4');
          102  +} {1 {database disk image is malformed}}
          103  +
          104  +do_execsql_test 2.5.1 {
          105  +  UPDATE ft_segments SET block = 
          106  +    blob('00056162633130031F0200 0202 6674 FFFFFF070302020003046E646F6E030200')
          107  +    WHERE blockid=2;
          108  +} {}
          109  +do_catchsql_test 2.5.2 {
          110  +  INSERT INTO ft(ft) VALUES('merge=1,4');
          111  +} {1 {database disk image is malformed}}
          112  +
          113  +#-------------------------------------------------------------------------
          114  +reset_db
          115  +do_execsql_test 3.0.0 {
          116  +  CREATE VIRTUAL TABLE ft USING fts3;
          117  +  INSERT INTO ft(ft) VALUES('nodesize=32');
          118  +}
          119  +do_test 3.0.1 {
          120  +  execsql BEGIN
          121  +  for {set i 0} {$i < 20} {incr i} {
          122  +    execsql { INSERT INTO ft VALUES('abc' || $i) }
          123  +  }
          124  +  execsql {
          125  +    COMMIT;
          126  +    SELECT count(*) FROM ft_segdir;
          127  +    SELECT count(*) FROM ft_segments;
          128  +  }
          129  +} {1 5}
          130  +
          131  +do_execsql_test 3.1 {
          132  +  SELECT quote(root) FROM ft_segdir
          133  +} {X'0101056162633132040136030132030136'}
          134  +
          135  +db func blob blob
          136  +do_execsql_test 3.2 {
          137  +  UPDATE ft_segdir 
          138  +  SET root = blob('0101056162633132FFFFFFFF070236030132030136');
          139  +}
          140  +
          141  +do_catchsql_test 3.1 {
          142  +  SELECT * FROM ft WHERE ft MATCH 'abc20'
          143  +} {1 {database disk image is malformed}}
          144  +
          145  +finish_test
          146  +
          147  +

Changes to test/permutations.test.

   251    251   } -files {
   252    252     fts3aa.test fts3ab.test fts3ac.test fts3ad.test
   253    253     fts3ae.test fts3af.test fts3ag.test fts3ah.test
   254    254     fts3ai.test fts3aj.test fts3ak.test fts3al.test
   255    255     fts3am.test fts3an.test fts3ao.test fts3atoken.test
   256    256     fts3auto.test fts3aux1.test fts3aux2.test fts3b.test
   257    257     fts3comp1.test fts3conf.test fts3corrupt2.test fts3corrupt.test
          258  +  fts3corrupt4.test
   258    259     fts3cov.test fts3c.test fts3defer2.test fts3defer3.test
   259    260     fts3defer.test fts3drop.test fts3d.test fts3e.test
   260    261     fts3expr2.test fts3expr3.test fts3expr4.test fts3expr5.test
   261    262     fts3expr.test fts3fault2.test fts3fault.test fts3first.test
   262    263     fts3join.test fts3malloc.test fts3matchinfo.test fts3near.test
   263    264     fts3offsets.test fts3prefix2.test fts3prefix.test fts3query.test
   264    265     fts3shared.test fts3snippet.test fts3sort.test fts3tok1.test