/ Check-in [8f5b199e]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Better detection and handling of corrupt database files. (CVS 1922)
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 8f5b199e845fa7ae3444ef69bd840716d305cf73
User & Date: drh 2004-08-30 16:52:18
Context
2004-08-31
00:52
Combine the implementation of LIKE and GLOB into a single parameterized function. (CVS 1923) check-in: 0a47c8f8 user: drh tags: trunk
2004-08-30
16:52
Better detection and handling of corrupt database files. (CVS 1922) check-in: 8f5b199e user: drh tags: trunk
14:58
Documentation updates (CVS 1921) check-in: 9322c439 user: drh tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to src/btree.c.

     5      5   ** a legal notice, here is a blessing:
     6      6   **
     7      7   **    May you do good and not evil.
     8      8   **    May you find forgiveness for yourself and forgive others.
     9      9   **    May you share freely, never taking more than you give.
    10     10   **
    11     11   *************************************************************************
    12         -** $Id: btree.c,v 1.182 2004/08/14 19:20:10 drh Exp $
           12  +** $Id: btree.c,v 1.183 2004/08/30 16:52:18 drh Exp $
    13     13   **
    14     14   ** This file implements a external (disk-based) database using BTrees.
    15     15   ** For a detailed discussion of BTrees, refer to
    16     16   **
    17     17   **     Donald E. Knuth, THE ART OF COMPUTER PROGRAMMING, Volume 3:
    18     18   **     "Sorting And Searching", pages 473-480. Addison-Wesley
    19     19   **     Publishing Company, Reading, Massachusetts.
................................................................................
   818    818     int nFree;         /* Number of unused bytes on the page */
   819    819     int top;           /* First byte of the cell content area */
   820    820   
   821    821     assert( pPage->pBt!=0 );
   822    822     assert( pParent==0 || pParent->pBt==pPage->pBt );
   823    823     assert( pPage->pgno==sqlite3pager_pagenumber(pPage->aData) );
   824    824     assert( pPage->aData == &((unsigned char*)pPage)[-pPage->pBt->pageSize] );
   825         -  assert( pPage->pParent==0 || pPage->pParent==pParent );
   826         -  assert( pPage->pParent==pParent || !pPage->isInit );
          825  +  if( pPage->pParent!=pParent && (pPage->pParent!=0 || pPage->isInit) ){
          826  +    /* The parent page should never change unless the file is corrupt */
          827  +    return SQLITE_CORRUPT; /* bkpt-CORRUPT */
          828  +  }
   827    829     if( pPage->isInit ) return SQLITE_OK;
   828    830     if( pPage->pParent==0 && pParent!=0 ){
   829    831       pPage->pParent = pParent;
   830    832       sqlite3pager_ref(pParent->aData);
   831    833     }
   832    834     hdr = pPage->hdrOffset;
   833    835     data = pPage->aData;
................................................................................
   834    836     decodeFlags(pPage, data[hdr]);
   835    837     pPage->nOverflow = 0;
   836    838     pPage->idxShift = 0;
   837    839     usableSize = pPage->pBt->usableSize;
   838    840     pPage->cellOffset = cellOffset = hdr + 12 - 4*pPage->leaf;
   839    841     top = get2byte(&data[hdr+5]);
   840    842     pPage->nCell = get2byte(&data[hdr+3]);
          843  +  if( pPage->nCell>MX_CELL ){
          844  +    /* To many cells for a single page.  The page must be corrupt */
          845  +    return SQLITE_CORRUPT; /* bkpt-CORRUPT */
          846  +  }
          847  +  if( pPage->nCell==0 && pParent!=0 && pParent->pgno!=1 ){
          848  +    /* All pages must have at least one cell, except for root pages */
          849  +    return SQLITE_CORRUPT; /* bkpt-CORRUPT */
          850  +  }
   841    851   
   842    852     /* Compute the total free space on the page */
   843    853     pc = get2byte(&data[hdr+1]);
   844    854     nFree = data[hdr+7] + top - (cellOffset + 2*pPage->nCell);
   845    855     i = 0;
   846    856     while( pc>0 ){
   847    857       int next, size;
   848         -    if( pc>=usableSize ) return SQLITE_CORRUPT;
   849         -    if( i++>SQLITE_MAX_PAGE_SIZE ) return SQLITE_CORRUPT;
          858  +    if( pc>usableSize-4 ){
          859  +      /* Free block is off the page */
          860  +      return SQLITE_CORRUPT;  /* bkpt-CORRUPT */
          861  +    }
          862  +    if( i++>SQLITE_MAX_PAGE_SIZE/4 ){
          863  +      /* The free block list forms an infinite loop */
          864  +      return SQLITE_CORRUPT;  /* bkpt-CORRUPT */
          865  +    }
   850    866       next = get2byte(&data[pc]);
   851    867       size = get2byte(&data[pc+2]);
   852         -    if( next>0 && next<=pc+size+3 ) return SQLITE_CORRUPT;
          868  +    if( next>0 && next<=pc+size+3 ){
          869  +      /* Free blocks must be in accending order */
          870  +      return SQLITE_CORRUPT;  /* bkpt-CORRUPT */
          871  +    }
   853    872       nFree += size;
   854    873       pc = next;
   855    874     }
   856    875     pPage->nFree = nFree;
   857         -  if( nFree>=usableSize ) return SQLITE_CORRUPT;
          876  +  if( nFree>=usableSize ){
          877  +    /* Free space cannot exceed total page size */
          878  +    return SQLITE_CORRUPT;  /* bkpt-CORRUPT */
          879  +  }
   858    880   
   859    881     pPage->isInit = 1;
   860    882     pageIntegrity(pPage);
   861    883     return SQLITE_OK;
   862    884   }
   863    885   
   864    886   /*
................................................................................
   918    940   static int getAndInitPage(
   919    941     Btree *pBt,          /* The database file */
   920    942     Pgno pgno,           /* Number of the page to get */
   921    943     MemPage **ppPage,    /* Write the page pointer here */
   922    944     MemPage *pParent     /* Parent of the page */
   923    945   ){
   924    946     int rc;
          947  +  if( pgno==0 ){
          948  +    return SQLITE_CORRUPT;  /* bkpt-CORRUPT */
          949  +  }
   925    950     rc = getPage(pBt, pgno, ppPage);
   926    951     if( rc==SQLITE_OK && (*ppPage)->isInit==0 ){
   927    952       rc = initPage(*ppPage, pParent);
   928    953     }
   929    954     return rc;
   930    955   }
   931    956   
................................................................................
  1786   1811           offset -= ovflSize;
  1787   1812         }
  1788   1813         sqlite3pager_unref(aPayload);
  1789   1814       }
  1790   1815     }
  1791   1816   
  1792   1817     if( amt>0 ){
  1793         -    return SQLITE_CORRUPT;
         1818  +    return SQLITE_CORRUPT; /* bkpt-CORRUPT */
  1794   1819     }
  1795   1820     return SQLITE_OK;
  1796   1821   }
  1797   1822   
  1798   1823   /*
  1799   1824   ** Read part of the key associated with cursor pCur.  Exactly
  1800   1825   ** "amt" bytes will be transfered into pBuf[].  The transfer
................................................................................
  1927   1952     pOldPage = pCur->pPage;
  1928   1953     pOldPage->idxShift = 0;
  1929   1954     releasePage(pOldPage);
  1930   1955     pCur->pPage = pNewPage;
  1931   1956     pCur->idx = 0;
  1932   1957     pCur->info.nSize = 0;
  1933   1958     if( pNewPage->nCell<1 ){
  1934         -    return SQLITE_CORRUPT;
         1959  +    return SQLITE_CORRUPT; /* bkpt-CORRUPT */
  1935   1960     }
  1936   1961     return SQLITE_OK;
  1937   1962   }
  1938   1963   
  1939   1964   /*
  1940   1965   ** Return true if the page is the virtual root of its table.
  1941   1966   **
................................................................................
  2382   2407       if( k==0 ){
  2383   2408         /* The trunk has no leaves.  So extract the trunk page itself and
  2384   2409         ** use it as the newly allocated page */
  2385   2410         *pPgno = get4byte(&pPage1->aData[32]);
  2386   2411         memcpy(&pPage1->aData[32], &pTrunk->aData[0], 4);
  2387   2412         *ppPage = pTrunk;
  2388   2413         TRACE(("ALLOCATE: %d trunk - %d free pages left\n", *pPgno, n-1));
         2414  +    }else if( k>pBt->usableSize/4 - 8 ){
         2415  +      /* Value of k is out of range.  Database corruption */
         2416  +      return SQLITE_CORRUPT; /* bkpt-CORRUPT */
  2389   2417       }else{
  2390   2418         /* Extract a leaf from the trunk */
  2391   2419         int closest;
  2392   2420         unsigned char *aData = pTrunk->aData;
  2393   2421         if( nearby>0 ){
  2394   2422           int i, dist;
  2395   2423           closest = 0;
................................................................................
  2400   2428             if( d2<0 ) d2 = -d2;
  2401   2429             if( d2<dist ) closest = i;
  2402   2430           }
  2403   2431         }else{
  2404   2432           closest = 0;
  2405   2433         }
  2406   2434         *pPgno = get4byte(&aData[8+closest*4]);
         2435  +      if( *pPgno>sqlite3pager_pagecount(pBt->pPager) ){
         2436  +        /* Free page off the end of the file */
         2437  +        return SQLITE_CORRUPT; /* bkpt-CORRUPT */
         2438  +      }
  2407   2439         TRACE(("ALLOCATE: %d was leaf %d of %d on trunk %d: %d more free pages\n",
  2408   2440                *pPgno, closest+1, k, pTrunk->pgno, n-1));
  2409   2441         if( closest<k-1 ){
  2410   2442           memcpy(&aData[8+closest*4], &aData[4+k*4], 4);
  2411   2443         }
  2412   2444         put4byte(&aData[4], k-1);
  2413   2445         rc = getPage(pBt, *pPgno, ppPage);
................................................................................
  2461   2493     }else{
  2462   2494       /* Other free pages already exist.  Retrive the first trunk page
  2463   2495       ** of the freelist and find out how many leaves it has. */
  2464   2496       MemPage *pTrunk;
  2465   2497       rc = getPage(pBt, get4byte(&pPage1->aData[32]), &pTrunk);
  2466   2498       if( rc ) return rc;
  2467   2499       k = get4byte(&pTrunk->aData[4]);
  2468         -    if( k==pBt->usableSize/4 - 8 ){
         2500  +    if( k>=pBt->usableSize/4 - 8 ){
  2469   2501         /* The trunk is full.  Turn the page being freed into a new
  2470   2502         ** trunk page with no leaves. */
  2471   2503         rc = sqlite3pager_write(pPage->aData);
  2472   2504         if( rc ) return rc;
  2473   2505         put4byte(pPage->aData, pTrunk->pgno);
  2474   2506         put4byte(&pPage->aData[4], 0);
  2475   2507         put4byte(&pPage1->aData[32], pPage->pgno);
................................................................................
  3561   3593       int szNext;
  3562   3594       int notUsed;
  3563   3595       unsigned char tempCell[MX_CELL_SIZE];
  3564   3596       assert( !pPage->leafData );
  3565   3597       getTempCursor(pCur, &leafCur);
  3566   3598       rc = sqlite3BtreeNext(&leafCur, &notUsed);
  3567   3599       if( rc!=SQLITE_OK ){
  3568         -      if( rc!=SQLITE_NOMEM ) rc = SQLITE_CORRUPT;
         3600  +      if( rc!=SQLITE_NOMEM ){
         3601  +        rc = SQLITE_CORRUPT;  /* bkpt-CORRUPT */
         3602  +      }
  3569   3603         return rc;
  3570   3604       }
  3571   3605       rc = sqlite3pager_write(leafCur.pPage->aData);
  3572   3606       if( rc ) return rc;
  3573   3607       TRACE(("DELETE: table=%d delete internal from %d replace from leaf %d\n",
  3574   3608          pCur->pgnoRoot, pPage->pgno, leafCur.pPage->pgno));
  3575   3609       dropCell(pPage, pCur->idx, cellSizePtr(pPage, pCell));
................................................................................
  4037   4071       if( sqlite3pager_get(pCheck->pPager, (Pgno)iPage, (void**)&pOvfl) ){
  4038   4072         sprintf(zMsg, "failed to get page %d", iPage);
  4039   4073         checkAppendMsg(pCheck, zContext, zMsg);
  4040   4074         break;
  4041   4075       }
  4042   4076       if( isFreeList ){
  4043   4077         int n = get4byte(&pOvfl[4]);
  4044         -      for(i=0; i<n; i++){
  4045         -        checkRef(pCheck, get4byte(&pOvfl[8+i*4]), zContext);
         4078  +      if( n>=pCheck->pBt->usableSize/4-8 ){
         4079  +        sprintf(zMsg, "freelist leaf count too big on page %d", iPage);
         4080  +        checkAppendMsg(pCheck, zContext, zMsg);
         4081  +        N--;
         4082  +      }else{
         4083  +        for(i=0; i<n; i++){
         4084  +          checkRef(pCheck, get4byte(&pOvfl[8+i*4]), zContext);
         4085  +        }
         4086  +        N -= n;
  4046   4087         }
  4047         -      N -= n;
  4048   4088       }
  4049   4089       iPage = get4byte(pOvfl);
  4050   4090       sqlite3pager_unref(pOvfl);
  4051   4091     }
  4052   4092   }
  4053   4093   
  4054   4094   /*

Changes to src/pager.c.

    14     14   ** The pager is used to access a database disk file.  It implements
    15     15   ** atomic commit and rollback through the use of a journal file that
    16     16   ** is separate from the database file.  The pager also implements file
    17     17   ** locking to prevent two processes from writing the same database
    18     18   ** file simultaneously, or one process from reading the database while
    19     19   ** another is writing.
    20     20   **
    21         -** @(#) $Id: pager.c,v 1.160 2004/08/21 19:20:42 drh Exp $
           21  +** @(#) $Id: pager.c,v 1.161 2004/08/30 16:52:18 drh Exp $
    22     22   */
    23     23   #include "os.h"         /* Must be first to enable large file support */
    24     24   #include "sqliteInt.h"
    25     25   #include "pager.h"
    26     26   #include <assert.h>
    27     27   #include <string.h>
    28     28   
................................................................................
  1304   1304     }
  1305   1305   
  1306   1306     pPager->journalOff = szJ;
  1307   1307     
  1308   1308   end_stmt_playback:
  1309   1309     if( rc!=SQLITE_OK ){
  1310   1310       pPager->errMask |= PAGER_ERR_CORRUPT;
  1311         -    rc = SQLITE_CORRUPT;
         1311  +    rc = SQLITE_CORRUPT;  /* bkpt-CORRUPT */
  1312   1312     }else{
  1313   1313       pPager->journalOff = szJ;
  1314   1314       /* pager_reload_cache(pPager); */
  1315   1315     }
  1316   1316     return rc;
  1317   1317   }
  1318   1318   
................................................................................
  2878   2878         rc = rc2;
  2879   2879         if( rc3 ) rc = rc3;
  2880   2880       }
  2881   2881     }else{
  2882   2882       rc = pager_playback(pPager);
  2883   2883     }
  2884   2884     if( rc!=SQLITE_OK ){
  2885         -    rc = SQLITE_CORRUPT;
         2885  +    rc = SQLITE_CORRUPT;  /* bkpt-CORRUPT */
  2886   2886       pPager->errMask |= PAGER_ERR_CORRUPT;
  2887   2887     }
  2888   2888     pPager->dbSize = -1;
  2889   2889     return rc;
  2890   2890   }
  2891   2891   
  2892   2892   /*

Changes to src/vdbemem.c.

   628    628   
   629    629       if( (flags & MEM_Str) ){
   630    630         assert( pMem->enc==SQLITE_UTF8 || 
   631    631                 pMem->enc==SQLITE_UTF16BE ||
   632    632                 pMem->enc==SQLITE_UTF16LE 
   633    633         );
   634    634         /* If the string is UTF-8 encoded and nul terminated, then pMem->n
   635         -      ** must be the length of the string.
          635  +      ** must be the length of the string.  (Later:)  If the database file
          636  +      ** has been corrupted, '\000' characters might have been inserted
          637  +      ** into the middle of the string.  In that case, the strlen() might
          638  +      ** be less.
   636    639         */
   637    640         if( pMem->enc==SQLITE_UTF8 && (flags & MEM_Term) ){ 
   638         -        assert( strlen(pMem->z)==pMem->n );
          641  +        assert( strlen(pMem->z)<=pMem->n );
          642  +        assert( pMem->z[pMem->n]==0 );
   639    643         }
   640    644       }
   641    645     }else{
   642    646       /* Cannot define a string subtype for non-string objects */
   643    647       assert( (pMem->flags & (MEM_Static|MEM_Dyn|MEM_Ephem|MEM_Short))==0 );
   644    648       assert( pMem->xDel==0 );
   645    649     }

Added test/corrupt.test.

            1  +# 2004 August 30
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +# This file implements regression tests for SQLite library.
           12  +#
           13  +# This file implements tests to make sure SQLite does not crash or
           14  +# segfault if it sees a corrupt database file.
           15  +#
           16  +# $Id: corrupt.test,v 1.1 2004/08/30 16:52:19 drh Exp $
           17  +
           18  +set testdir [file dirname $argv0]
           19  +source $testdir/tester.tcl
           20  +
           21  +# Construct a large database for testing.
           22  +#
           23  +do_test corrupt-1.1 {
           24  +  execsql {
           25  +    BEGIN;
           26  +    CREATE TABLE t1(x);
           27  +    INSERT INTO t1 VALUES(randstr(10,100));
           28  +    INSERT INTO t1 VALUES(randstr(10,100));
           29  +    INSERT INTO t1 VALUES(randstr(10,100));
           30  +    INSERT INTO t1 SELECT x || randstr(5,10) FROM t1;
           31  +    INSERT INTO t1 SELECT x || randstr(5,10) FROM t1;
           32  +    INSERT INTO t1 SELECT x || randstr(5,10) FROM t1;
           33  +    INSERT INTO t1 SELECT x || randstr(5,10) FROM t1;
           34  +    INSERT INTO t1 VALUES(randstr(2100,3000));
           35  +    INSERT INTO t1 SELECT x || randstr(5,10) FROM t1;
           36  +    INSERT INTO t1 SELECT x || randstr(5,10) FROM t1;
           37  +    INSERT INTO t1 SELECT x || randstr(5,10) FROM t1;
           38  +    INSERT INTO t1 SELECT x || randstr(5,10) FROM t1;
           39  +    CREATE INDEX t1i1 ON t1(x);
           40  +    CREATE TABLE t2 AS SELECT * FROM t1;
           41  +    DELETE FROM t2 WHERE rowid%5!=0;
           42  +    COMMIT;
           43  +    PRAGMA integrity_check;
           44  +  }
           45  +} {ok}
           46  +
           47  +# Copy a file 
           48  +#
           49  +proc copy_file {from to} {
           50  +  set f [open $from]
           51  +  fconfigure $f -translation binary
           52  +  set t [open $to w]
           53  +  fconfigure $t -translation binary
           54  +  puts -nonewline $t [read $f [file size $from]]
           55  +  close $t
           56  +  close $f
           57  +}
           58  +
           59  +# Setup for the tests.
           60  +copy_file test.db test.bu
           61  +set fsize [file size test.db]
           62  +set junk "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
           63  +while {[string length $junk]<256} {append junk $junk}
           64  +set junk [string range $junk 0 255]
           65  +
           66  +
           67  +for {set i [expr {1*256}]} {$i<$fsize-256} {incr i 256} {
           68  +  set tn [expr {$i/256}]
           69  +  db close
           70  +  copy_file test.bu test.db
           71  +  set fd [open test.db r+]
           72  +  fconfigure $fd -translation binary
           73  +  seek $fd $i
           74  +  puts -nonewline $fd $junk
           75  +  close $fd
           76  +  sqlite3 db test.db
           77  +  do_test corrupt-2.$tn.1 {
           78  +    sqlite3 db test.db
           79  +    catchsql {SELECT count(*) FROM sqlite_master}
           80  +    set x {}
           81  +  } {}
           82  +  do_test corrupt-2.$tn.2 {
           83  +    catchsql {SELECT count(*) FROM t1}
           84  +    set x {}
           85  +  } {}
           86  +  do_test corrupt-2.$tn.3 {
           87  +    catchsql {SELECT count(*) FROM t1 WHERE x>'abcdef'}
           88  +    set x {}
           89  +  } {}
           90  +  do_test corrupt-2.$tn.4 {
           91  +    catchsql {SELECT count(*) FROM t2}
           92  +    set x {}
           93  +  } {}
           94  +  do_test corrupt-2.$tn.5 {
           95  +    catchsql {CREATE TABLE t3 AS SELECT * FROM t1}
           96  +    set x {}
           97  +  } {}
           98  +  do_test corrupt-2.$tn.6 {
           99  +    catchsql {DROP TABLE t1}
          100  +    set x {}
          101  +  } {}
          102  +if {$tn==724} btree_breakpoint
          103  +  do_test corrupt-2.$tn.7 {
          104  +    catchsql {PRAGMA integrity_check}
          105  +    set x {}
          106  +  } {}
          107  +}  
          108  +
          109  +finish_test

Changes to test/quick.test.

     6      6   #    May you do good and not evil.
     7      7   #    May you find forgiveness for yourself and forgive others.
     8      8   #    May you share freely, never taking more than you give.
     9      9   #
    10     10   #***********************************************************************
    11     11   # This file runs all tests.
    12     12   #
    13         -# $Id: quick.test,v 1.28 2004/07/22 15:02:26 drh Exp $
           13  +# $Id: quick.test,v 1.29 2004/08/30 16:52:19 drh Exp $
    14     14   
    15     15   set testdir [file dirname $argv0]
    16     16   source $testdir/tester.tcl
    17     17   rename finish_test really_finish_test
    18     18   proc finish_test {} {}
    19     19   set ISQUICK 1
    20     20   
    21     21   set EXCLUDE {
    22     22     all.test
    23         -  quick.test
    24     23     btree2.test
           24  +  corrupt.test
           25  +  crash.test
    25     26     malloc.test
    26     27     memleak.test
    27     28     misuse.test
    28         -  crash.test
           29  +  quick.test
    29     30     utf16.test
    30     31   }
    31     32   
    32     33   if {[sqlite3 -has-codec]} {
    33     34     # lappend EXCLUDE \
    34     35     #  conflict.test
    35     36   }