SQLite

Check-in [30825f74d6]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Make the btree layer robust when faced with a corrupt database that contains duplicate entries on the freelist. Ticket #3209. (CVS 5392)
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 30825f74d60d8ace39bafd06814017ceefeb4fa4
User & Date: drh 2008-07-11 03:34:10.000
Context
2008-07-11
03:38
Remove an extra zeroPage() call that was left in the previous check-in by mistake. Ticket #3209. (CVS 5393) (check-in: c45d578e44 user: drh tags: trunk)
03:34
Make the btree layer robust when faced with a corrupt database that contains duplicate entries on the freelist. Ticket #3209. (CVS 5392) (check-in: 30825f74d6 user: drh tags: trunk)
02:21
Additional test coverage in btree.c. Added corruption tests for the ptrmap pages of an autovacuumed database (corrupt8.test). (CVS 5391) (check-in: 620b472133 user: drh tags: trunk)
Changes
Unified Diff Ignore Whitespace Patch
Changes to src/btree.c.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
/*
** 2004 April 6
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
** $Id: btree.c,v 1.476 2008/07/11 02:21:41 drh Exp $
**
** This file implements a external (disk-based) database using BTrees.
** See the header comment on "btreeInt.h" for additional information.
** Including a description of file format and an overview of operation.
*/
#include "btreeInt.h"












|







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
/*
** 2004 April 6
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
** $Id: btree.c,v 1.477 2008/07/11 03:34:10 drh Exp $
**
** This file implements a external (disk-based) database using BTrees.
** See the header comment on "btreeInt.h" for additional information.
** Including a description of file format and an overview of operation.
*/
#include "btreeInt.h"

5298
5299
5300
5301
5302
5303
5304

5305
5306
5307
5308
5309
5310
5311
  */
  j = 0;
  for(i=0; i<nNew; i++){
    /* Assemble the new sibling page. */
    MemPage *pNew = apNew[i];
    assert( j<nMaxCells );
    assert( pNew->pgno==pgnoNew[i] );

    assemblePage(pNew, cntNew[i]-j, &apCell[j], &szCell[j]);
    assert( pNew->nCell>0 || (nNew==1 && cntNew[0]==0) );
    assert( pNew->nOverflow==0 );

#ifndef SQLITE_OMIT_AUTOVACUUM
    /* If this is an auto-vacuum database, update the pointer map entries
    ** that point to the siblings that were rearranged. These can be: left







>







5298
5299
5300
5301
5302
5303
5304
5305
5306
5307
5308
5309
5310
5311
5312
  */
  j = 0;
  for(i=0; i<nNew; i++){
    /* Assemble the new sibling page. */
    MemPage *pNew = apNew[i];
    assert( j<nMaxCells );
    assert( pNew->pgno==pgnoNew[i] );
    zeroPage(pNew, pageFlags);
    assemblePage(pNew, cntNew[i]-j, &apCell[j], &szCell[j]);
    assert( pNew->nCell>0 || (nNew==1 && cntNew[0]==0) );
    assert( pNew->nOverflow==0 );

#ifndef SQLITE_OMIT_AUTOVACUUM
    /* If this is an auto-vacuum database, update the pointer map entries
    ** that point to the siblings that were rearranged. These can be: left
5574
5575
5576
5577
5578
5579
5580
5581
5582
5583
5584
5585
5586
5587
5588
  usableSize = pBt->usableSize;
  data = pPage->aData;
  hdr = pPage->hdrOffset;
  brk = get2byte(&data[hdr+5]);
  cdata = pChild->aData;
  memcpy(cdata, &data[hdr], pPage->cellOffset+2*pPage->nCell-hdr);
  memcpy(&cdata[brk], &data[brk], usableSize-brk);
  assert( pChild->isInit==0 );
  rc = sqlite3BtreeInitPage(pChild, pPage);
  if( rc ) goto balancedeeper_out;
  memcpy(pChild->aOvfl, pPage->aOvfl, pPage->nOverflow*sizeof(pPage->aOvfl[0]));
  pChild->nOverflow = pPage->nOverflow;
  if( pChild->nOverflow ){
    pChild->nFree = 0;
  }







|







5575
5576
5577
5578
5579
5580
5581
5582
5583
5584
5585
5586
5587
5588
5589
  usableSize = pBt->usableSize;
  data = pPage->aData;
  hdr = pPage->hdrOffset;
  brk = get2byte(&data[hdr+5]);
  cdata = pChild->aData;
  memcpy(cdata, &data[hdr], pPage->cellOffset+2*pPage->nCell-hdr);
  memcpy(&cdata[brk], &data[brk], usableSize-brk);
  if( pChild->isInit ) return SQLITE_CORRUPT;
  rc = sqlite3BtreeInitPage(pChild, pPage);
  if( rc ) goto balancedeeper_out;
  memcpy(pChild->aOvfl, pPage->aOvfl, pPage->nOverflow*sizeof(pPage->aOvfl[0]));
  pChild->nOverflow = pPage->nOverflow;
  if( pChild->nOverflow ){
    pChild->nFree = 0;
  }
Changes to src/pager.c.
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
** The pager is used to access a database disk file.  It implements
** atomic commit and rollback through the use of a journal file that
** is separate from the database file.  The pager also implements file
** locking to prevent two processes from writing the same database
** file simultaneously, or one process from reading the database while
** another is writing.
**
** @(#) $Id: pager.c,v 1.464 2008/07/10 00:32:42 drh Exp $
*/
#ifndef SQLITE_OMIT_DISKIO
#include "sqliteInt.h"
#include <assert.h>
#include <string.h>

/*







|







14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
** The pager is used to access a database disk file.  It implements
** atomic commit and rollback through the use of a journal file that
** is separate from the database file.  The pager also implements file
** locking to prevent two processes from writing the same database
** file simultaneously, or one process from reading the database while
** another is writing.
**
** @(#) $Id: pager.c,v 1.465 2008/07/11 03:34:10 drh Exp $
*/
#ifndef SQLITE_OMIT_DISKIO
#include "sqliteInt.h"
#include <assert.h>
#include <string.h>

/*
4543
4544
4545
4546
4547
4548
4549




4550
4551
4552
4553
4554
4555
4556
4557
4558
#endif

  /* If SECURE_DELETE is disabled, then there is no way that this
  ** routine can be called on a page for which sqlite3PagerDontWrite()
  ** has not been previously called during the same transaction.
  ** And if DontWrite() has previously been called, the following
  ** conditions must be met.




  */
  assert( !pPg->inJournal && (int)pPg->pgno <= pPager->origDbSize );

  assert( pPager->pInJournal!=0 );
  sqlite3BitvecSet(pPager->pInJournal, pPg->pgno);
  pPg->inJournal = 1;
  pPg->needRead = 0;
  if( pPager->stmtInUse ){
    assert( pPager->stmtSize >= pPager->origDbSize );







>
>
>
>

|







4543
4544
4545
4546
4547
4548
4549
4550
4551
4552
4553
4554
4555
4556
4557
4558
4559
4560
4561
4562
#endif

  /* If SECURE_DELETE is disabled, then there is no way that this
  ** routine can be called on a page for which sqlite3PagerDontWrite()
  ** has not been previously called during the same transaction.
  ** And if DontWrite() has previously been called, the following
  ** conditions must be met.
  **
  ** (Later:)  Not true.  If the database is corrupted by having duplicate
  ** pages on the freelist (ex: corrupt9.test) then the following is not
  ** necessarily true:
  */
  /* assert( !pPg->inJournal && (int)pPg->pgno <= pPager->origDbSize ); */

  assert( pPager->pInJournal!=0 );
  sqlite3BitvecSet(pPager->pInJournal, pPg->pgno);
  pPg->inJournal = 1;
  pPg->needRead = 0;
  if( pPager->stmtInUse ){
    assert( pPager->stmtSize >= pPager->origDbSize );
Changes to test/corrupt8.test.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
# 2008 June 11
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file implements regression tests for SQLite library.
#
# This file implements tests to make sure SQLite does not crash or
# segfault if it sees a corrupt database file.  It specifically focuses
# on corrupt pointer map pages.
#
# $Id: corrupt8.test,v 1.1 2008/07/11 02:21:41 drh Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# We must have the page_size pragma for these tests to work.
#
ifcapable !pager_pragmas||!autovacuum {
|















|







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
# 2008 July 9
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file implements regression tests for SQLite library.
#
# This file implements tests to make sure SQLite does not crash or
# segfault if it sees a corrupt database file.  It specifically focuses
# on corrupt pointer map pages.
#
# $Id: corrupt8.test,v 1.2 2008/07/11 03:34:10 drh Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# We must have the page_size pragma for these tests to work.
#
ifcapable !pager_pragmas||!autovacuum {
Added test/corrupt9.test.




































































































































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
# 2008 July 9
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file implements regression tests for SQLite library.
#
# This file implements tests to make sure SQLite does not crash or
# segfault if it sees a corrupt database file.  It specifically focuses
# on corruption in the form of duplicate entries no the freelist.
#
# $Id: corrupt9.test,v 1.1 2008/07/11 03:34:10 drh Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# We must have the page_size pragma for these tests to work.
#
ifcapable !pager_pragmas {
  finish_test
  return
}

# Return the offset to the first (trunk) page of the freelist.  Return
# zero of the freelist is empty.
#
proc freelist_trunk_offset {filename} {
  if {[hexio_read $filename 36 4]==0} {return 0}
  set pgno [hexio_get_int [hexio_read $filename 32 4]]
  return [expr {($pgno-1)*[hexio_get_int [hexio_read $filename 16 2]]}]
}

# This procedure looks at the first trunk page of the freelist and
# corrupts that page by overwriting up to N entries with duplicates
# of the first entry.
#
proc corrupt_freelist {filename N} {
  set offset [freelist_trunk_offset $filename]
  if {$offset==0} {error "Freelist is empty"}
  set cnt [hexio_get_int [hexio_read $filename [expr {$offset+4}] 4]]
  set pgno [hexio_read $filename [expr {$offset+8}] 4]
  for {set i 12} {$N>0 && $i<8+4*$cnt} {incr i 4; incr N -1} {
    hexio_write $filename [expr {$offset+$i}] $pgno
  }
}

# Create a database to work with.  Make sure there are plenty of
# entries on the freelist.
#
do_test corrupt9-1.1 {
  execsql {
    PRAGMA page_size=1024;
    CREATE TABLE t1(x);
    INSERT INTO t1(x) VALUES(1);
    INSERT INTO t1(x) VALUES(2);
    INSERT INTO t1(x) SELECT x+2 FROM t1;
    INSERT INTO t1(x) SELECT x+4 FROM t1;
    INSERT INTO t1(x) SELECT x+8 FROM t1;
    INSERT INTO t1(x) SELECT x+16 FROM t1;
    INSERT INTO t1(x) SELECT x+32 FROM t1;
    INSERT INTO t1(x) SELECT x+64 FROM t1;
    INSERT INTO t1(x) SELECT x+128 FROM t1;
    INSERT INTO t1(x) SELECT x+256 FROM t1;
    CREATE TABLE t2(a,b);
    INSERT INTO t2 SELECT x, x*x FROM t1;
    CREATE INDEX i1 ON t1(x);
    CREATE INDEX i2 ON t2(b,a);
    DROP INDEX i2;
  }
  expr {[file size test.db]>1024*24}
} {1}
integrity_check corrupt9-1.2

# Corrupt the freelist by adding duplicate entries to the freelist.
# Make sure the corruption is detected.
#
db close
file copy -force test.db test.db-template

corrupt_freelist test.db 1
sqlite3 db test.db
do_test corrupt9-2.1 {
  set x [db eval {PRAGMA integrity_check}]
  expr {$x!="ok"}
} {1}
do_test corrupt9-2.2 {
  catchsql {
    CREATE INDEX i2 ON t2(b,a);
    REINDEX;
  }
} {1 {database disk image is malformed}}


db close
file copy -force test.db-template test.db
corrupt_freelist test.db 2
sqlite3 db test.db
do_test corrupt9-3.1 {
  set x [db eval {PRAGMA integrity_check}]
  expr {$x!="ok"}
} {1}
do_test corrupt9-3.2 {
  catchsql {
    CREATE INDEX i2 ON t2(b,a);
    REINDEX;
  }
} {1 {database disk image is malformed}}

db close
file copy -force test.db-template test.db
corrupt_freelist test.db 3
sqlite3 db test.db
do_test corrupt9-4.1 {
  set x [db eval {PRAGMA integrity_check}]
  expr {$x!="ok"}
} {1}
do_test corrupt9-4.2 {
  catchsql {
    CREATE INDEX i2 ON t2(b,a);
    REINDEX;
  }
} {1 {database disk image is malformed}}
 

finish_test