/ Check-in [65245d99]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add a test (and fix) for possible corruption if malloc() fails during a CREATE INDEX statement, the application continues with the transaction, then crashes. (CVS 4725)
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 65245d9904db19568d5092926b27f0c193c9ef0b
User & Date: danielk1977 2008-01-18 13:42:55
Context
2008-01-18
14:08
Remove the OP_HexBlob instruction and code OP_Blob directly. Reduce the amount of memory allocation required to encode blob literals. Remove the "out2" instruction type. Other minor optimizations. (CVS 4726) check-in: 0e50c020 user: drh tags: trunk
13:42
Add a test (and fix) for possible corruption if malloc() fails during a CREATE INDEX statement, the application continues with the transaction, then crashes. (CVS 4725) check-in: 65245d99 user: danielk1977 tags: trunk
11:33
Add a couple of missing calls to pagerLeave(). (CVS 4724) check-in: 87534dff user: danielk1977 tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/pager.c.

14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
....
5047
5048
5049
5050
5051
5052
5053







5054
5055
5056
5057
5058
5059
5060
5061
5062



5063
5064
5065
5066
5067
5068
5069
** The pager is used to access a database disk file.  It implements
** atomic commit and rollback through the use of a journal file that
** is separate from the database file.  The pager also implements file
** locking to prevent two processes from writing the same database
** file simultaneously, or one process from reading the database while
** another is writing.
**
** @(#) $Id: pager.c,v 1.401 2008/01/18 11:33:16 danielk1977 Exp $
*/
#ifndef SQLITE_OMIT_DISKIO
#include "sqliteInt.h"
#include <assert.h>
#include <string.h>

/*
................................................................................

  if( needSyncPgno ){
    /* If needSyncPgno is non-zero, then the journal file needs to be 
    ** sync()ed before any data is written to database file page needSyncPgno.
    ** Currently, no such page exists in the page-cache and the 
    ** Pager.aInJournal bit has been set. This needs to be remedied by loading
    ** the page into the pager-cache and setting the PgHdr.needSync flag.







    **
    ** The sqlite3PagerGet() call may cause the journal to sync. So make
    ** sure the Pager.needSync flag is set too.
    */
    int rc;
    PgHdr *pPgHdr;
    assert( pPager->needSync );
    rc = sqlite3PagerGet(pPager, needSyncPgno, &pPgHdr);
    if( rc!=SQLITE_OK ){



      pagerLeave(pPager);
      return rc;
    }
    pPager->needSync = 1;
    pPgHdr->needSync = 1;
    pPgHdr->inJournal = 1;
    makeDirty(pPgHdr);







|







 







>
>
>
>
>
>
>









>
>
>







14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
....
5047
5048
5049
5050
5051
5052
5053
5054
5055
5056
5057
5058
5059
5060
5061
5062
5063
5064
5065
5066
5067
5068
5069
5070
5071
5072
5073
5074
5075
5076
5077
5078
5079
** The pager is used to access a database disk file.  It implements
** atomic commit and rollback through the use of a journal file that
** is separate from the database file.  The pager also implements file
** locking to prevent two processes from writing the same database
** file simultaneously, or one process from reading the database while
** another is writing.
**
** @(#) $Id: pager.c,v 1.402 2008/01/18 13:42:55 danielk1977 Exp $
*/
#ifndef SQLITE_OMIT_DISKIO
#include "sqliteInt.h"
#include <assert.h>
#include <string.h>

/*
................................................................................

  if( needSyncPgno ){
    /* If needSyncPgno is non-zero, then the journal file needs to be 
    ** sync()ed before any data is written to database file page needSyncPgno.
    ** Currently, no such page exists in the page-cache and the 
    ** Pager.aInJournal bit has been set. This needs to be remedied by loading
    ** the page into the pager-cache and setting the PgHdr.needSync flag.
    **
    ** If the attempt to load the page into the page-cache fails, (due
    ** to a malloc() or IO failure), clear the bit in the aInJournal[]
    ** array. Otherwise, if the page is loaded and written again in
    ** this transaction, it may be written to the database file before
    ** it is synced into the journal file. This way, it may end up in
    ** the journal file twice, but that is not a problem.
    **
    ** The sqlite3PagerGet() call may cause the journal to sync. So make
    ** sure the Pager.needSync flag is set too.
    */
    int rc;
    PgHdr *pPgHdr;
    assert( pPager->needSync );
    rc = sqlite3PagerGet(pPager, needSyncPgno, &pPgHdr);
    if( rc!=SQLITE_OK ){
      if( pPager->aInJournal && (int)needSyncPgno<=pPager->origDbSize ){
        pPager->aInJournal[needSyncPgno/8] &= ~(1<<(needSyncPgno&7));
      }
      pagerLeave(pPager);
      return rc;
    }
    pPager->needSync = 1;
    pPgHdr->needSync = 1;
    pPgHdr->inJournal = 1;
    makeDirty(pPgHdr);

Changes to src/test6.c.

150
151
152
153
154
155
156










157
158
159
160
161
162
163
...
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
...
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
...
334
335
336
337
338
339
340
341
342
343
344

345
346
347
348
349
350
351
...
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
...
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559

static CrashGlobal g = {0, 0, SQLITE_DEFAULT_SECTOR_SIZE, 0, 0};

/*
** Set this global variable to 1 to enable crash testing.
*/
static int sqlite3CrashTestEnable = 0;











/*
** Flush the write-list as if xSync() had been called on file handle
** pFile. If isCrash is true, simulate a crash.
*/
static int writeListSync(CrashFile *pFile, int isCrash){
  int rc = SQLITE_OK;
................................................................................
#ifdef TRACE_CRASHTEST
        if( isCrash ){
          printf("Writing %d bytes @ %d (%s)\n", 
            pWrite->nBuf, (int)pWrite->iOffset, pWrite->pFile->zName
          );
        }
#endif
        sqlite3_free(pWrite);
        break;
      }
      case 2: {               /* Do nothing */
        ppPtr = &pWrite->pNext;
#ifdef TRACE_CRASHTEST
        if( isCrash ){
          printf("Omiting %d bytes @ %d (%s)\n", 
................................................................................

#ifdef TRACE_CRASHTEST
        printf("Trashing %d sectors @ sector %d (%s)\n", 
            1+iLast-iFirst, iFirst, pWrite->pFile->zName
        );
#endif

        zGarbage = sqlite3_malloc(g.iSectorSize);
        if( zGarbage ){
          sqlite3_int64 i;
          for(i=iFirst; rc==SQLITE_OK && i<=iLast; i++){
            sqlite3Randomness(g.iSectorSize, zGarbage); 
            rc = sqlite3OsWrite(
              pRealFile, zGarbage, g.iSectorSize, i*g.iSectorSize
            );
          }
          sqlite3_free(zGarbage);
        }else{
          rc = SQLITE_NOMEM;
        }

        ppPtr = &pWrite->pNext;
        break;
      }
................................................................................
  const u8 *zBuf,
  int nBuf
){
  WriteBuffer *pNew;

  assert((zBuf && nBuf) || (!nBuf && !zBuf));

  pNew = (WriteBuffer *)sqlite3MallocZero(sizeof(WriteBuffer) + nBuf);
  if( pNew==0 ){
    fprintf(stderr, "out of memory in the crash simulator\n");
  }

  pNew->iOffset = iOffset;
  pNew->nBuf = nBuf;
  pNew->pFile = (CrashFile *)pFile;
  if( zBuf ){
    pNew->zBuf = (u8 *)&pNew[1];
    memcpy(pNew->zBuf, zBuf, nBuf);
  }
................................................................................
  CrashFile *pCrash = (CrashFile *)pFile;
  if( iAmt+iOfst>pCrash->iSize ){
    pCrash->iSize = iAmt+iOfst;
  }
  while( pCrash->iSize>pCrash->nData ){
    u8 *zNew;
    int nNew = (pCrash->nData*2) + 4096;
    zNew = sqlite3_realloc(pCrash->zData, nNew);
    if( !zNew ){
      return SQLITE_NOMEM;
    }
    memset(&zNew[pCrash->nData], 0, nNew-pCrash->nData);
    pCrash->nData = nNew;
    pCrash->zData = zNew;
  }
................................................................................
    pWrapper->zName = (char *)zName;
    pWrapper->pRealFile = pReal;
    rc = sqlite3OsFileSize(pReal, &iSize);
    pWrapper->iSize = (int)iSize;
  }
  if( rc==SQLITE_OK ){
    pWrapper->nData = (4096 + pWrapper->iSize);
    pWrapper->zData = sqlite3_malloc(pWrapper->nData);
    if( pWrapper->zData ){
      memset(pWrapper->zData, 0, pWrapper->nData);
      rc = sqlite3OsRead(pReal, pWrapper->zData, pWrapper->iSize, 0); 
    }else{
      rc = SQLITE_NOMEM;
    }
  }







>
>
>
>
>
>
>
>
>
>







 







|







 







|








|







 







|



>







 







|







 







|







150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
...
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
...
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
...
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
...
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
...
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570

static CrashGlobal g = {0, 0, SQLITE_DEFAULT_SECTOR_SIZE, 0, 0};

/*
** Set this global variable to 1 to enable crash testing.
*/
static int sqlite3CrashTestEnable = 0;

static void *crash_malloc(int nByte){
  return (void *)Tcl_Alloc((size_t)nByte);
}
static void crash_free(void *p){
  Tcl_Free(p);
}
static void *crash_realloc(void *p, int n){
  return (void *)Tcl_Realloc(p, (size_t)n);
}

/*
** Flush the write-list as if xSync() had been called on file handle
** pFile. If isCrash is true, simulate a crash.
*/
static int writeListSync(CrashFile *pFile, int isCrash){
  int rc = SQLITE_OK;
................................................................................
#ifdef TRACE_CRASHTEST
        if( isCrash ){
          printf("Writing %d bytes @ %d (%s)\n", 
            pWrite->nBuf, (int)pWrite->iOffset, pWrite->pFile->zName
          );
        }
#endif
        crash_free(pWrite);
        break;
      }
      case 2: {               /* Do nothing */
        ppPtr = &pWrite->pNext;
#ifdef TRACE_CRASHTEST
        if( isCrash ){
          printf("Omiting %d bytes @ %d (%s)\n", 
................................................................................

#ifdef TRACE_CRASHTEST
        printf("Trashing %d sectors @ sector %d (%s)\n", 
            1+iLast-iFirst, iFirst, pWrite->pFile->zName
        );
#endif

        zGarbage = crash_malloc(g.iSectorSize);
        if( zGarbage ){
          sqlite3_int64 i;
          for(i=iFirst; rc==SQLITE_OK && i<=iLast; i++){
            sqlite3Randomness(g.iSectorSize, zGarbage); 
            rc = sqlite3OsWrite(
              pRealFile, zGarbage, g.iSectorSize, i*g.iSectorSize
            );
          }
          crash_free(zGarbage);
        }else{
          rc = SQLITE_NOMEM;
        }

        ppPtr = &pWrite->pNext;
        break;
      }
................................................................................
  const u8 *zBuf,
  int nBuf
){
  WriteBuffer *pNew;

  assert((zBuf && nBuf) || (!nBuf && !zBuf));

  pNew = (WriteBuffer *)crash_malloc(sizeof(WriteBuffer) + nBuf);
  if( pNew==0 ){
    fprintf(stderr, "out of memory in the crash simulator\n");
  }
  memset(pNew, 0, sizeof(WriteBuffer)+nBuf);
  pNew->iOffset = iOffset;
  pNew->nBuf = nBuf;
  pNew->pFile = (CrashFile *)pFile;
  if( zBuf ){
    pNew->zBuf = (u8 *)&pNew[1];
    memcpy(pNew->zBuf, zBuf, nBuf);
  }
................................................................................
  CrashFile *pCrash = (CrashFile *)pFile;
  if( iAmt+iOfst>pCrash->iSize ){
    pCrash->iSize = iAmt+iOfst;
  }
  while( pCrash->iSize>pCrash->nData ){
    u8 *zNew;
    int nNew = (pCrash->nData*2) + 4096;
    zNew = crash_realloc(pCrash->zData, nNew);
    if( !zNew ){
      return SQLITE_NOMEM;
    }
    memset(&zNew[pCrash->nData], 0, nNew-pCrash->nData);
    pCrash->nData = nNew;
    pCrash->zData = zNew;
  }
................................................................................
    pWrapper->zName = (char *)zName;
    pWrapper->pRealFile = pReal;
    rc = sqlite3OsFileSize(pReal, &iSize);
    pWrapper->iSize = (int)iSize;
  }
  if( rc==SQLITE_OK ){
    pWrapper->nData = (4096 + pWrapper->iSize);
    pWrapper->zData = crash_malloc(pWrapper->nData);
    if( pWrapper->zData ){
      memset(pWrapper->zData, 0, pWrapper->nData);
      rc = sqlite3OsRead(pReal, pWrapper->zData, pWrapper->iSize, 0); 
    }else{
      rc = SQLITE_NOMEM;
    }
  }

Added test/crash5.test.































































































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111

# 2007 Aug 13
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# 
# This file tests aspects of recovery from a malloc() failure
# in a CREATE INDEX statement.
#
# $Id: crash5.test,v 1.1 2008/01/18 13:42:55 danielk1977 Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# Only run these tests if memory debugging is turned on.
#
ifcapable !memdebug||!crashtest||!memorymanage {
   puts "Skipping crash5 tests: not compiled with -DSQLITE_MEMDEBUG..."
   finish_test
   return
}

db close

for {set ii 0} {$ii < 10} {incr ii} {
  for {set jj 50} {$jj < 100} {incr jj} {

    # Set up the database so that it is an auto-vacuum database 
    # containing a single table (root page 3) with a single row. 
    # The row has an overflow page (page 4).
    file delete -force test.db test.db-journal
    sqlite3 db test.db
    set c [string repeat 3 1500]
    db eval {
      pragma auto_vacuum = 1;
      CREATE TABLE t1(a, b, c);
      INSERT INTO t1 VALUES('1111111111', '2222222222', $c);
    }
    db close

    do_test crash5-$ii.$jj.1 {
      crashsql -delay 1 -file test.db-journal -seed $ii -tclbody [join [list \
        [list set iFail $jj] {
        sqlite3_crashparams 0 /Users/dan/Desktop/sqlite/bld/test.db-journal
      
        # Begin a transaction and evaluate a "CREATE INDEX" statement
        # with the iFail'th malloc() set to fail. This operation will
        # have to move the current contents of page 4 (the overflow
        # page) to make room for the new root page. The bug is that
        # if malloc() fails at a particular point in sqlite3PagerMovepage(),
        # sqlite mistakenly thinks that the page being moved (page 4) has 
        # been safely synced into the journal. If the page is written
        # to later in the transaction, it may be written out to the database
        # before the relevant part of the journal has been synced.
        #
        db eval BEGIN
        sqlite3_memdebug_fail $iFail -repeat 0
        catch {db eval { CREATE UNIQUE INDEX i1 ON t1(a); }} msg
        # puts "$n $msg ac=[sqlite3_get_autocommit db]"
      
        # If the transaction is still active (it may not be if the malloc()
        # failure occured in the OS layer), write to the database. Make sure
        # page 4 is among those written.
        #
        if {![sqlite3_get_autocommit db]} {
          db eval {
            DELETE FROM t1;  -- This will put page 4 on the free list.
            INSERT INTO t1 VALUES('111111111', '2222222222', '33333333');
            INSERT INTO t1 SELECT * FROM t1;                     -- 2
            INSERT INTO t1 SELECT * FROM t1;                     -- 4
            INSERT INTO t1 SELECT * FROM t1;                     -- 8
            INSERT INTO t1 SELECT * FROM t1;                     -- 16
            INSERT INTO t1 SELECT * FROM t1;                     -- 32
            INSERT INTO t1 SELECT * FROM t1 WHERE rowid%2;       -- 48
          }
        }
        
        # If the right malloc() failed during the 'CREATE INDEX' above and
        # the transaction was not rolled back, then the sqlite cache now 
        # has a dirty page 4 that it incorrectly believes is already safely
        # in the synced part of the journal file. When 
        # sqlite3_release_memory() is called sqlite tries to free memory
        # by writing page 4 out to the db file. If it crashes later on,
        # before syncing the journal... Corruption!
        #
        sqlite3_crashparams 1 /Users/dan/Desktop/sqlite/bld/test.db-journal
        sqlite3_release_memory 8092
      }]] {}
      expr 1
    } {1}
  
    sqlite3 db test.db
    do_test crash5-$ii.$jj.2 {
      db eval {pragma integrity_check}
    } {ok}
    do_test crash5-$ii.$jj.3 {
      db eval {SELECT * FROM t1}
    } [list 1111111111 2222222222 $::c]
    db close
  }
}


finish_test