/ Check-in [5f0fb894]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add some further tests and a bugfix for the atomic-write optimization. (CVS 4276)
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 5f0fb894f44069c4aa9b8dba62b4d8a262c991de
User & Date: danielk1977 2007-08-23 11:07:10
Context
2007-08-23
11:47
Ensure temporary files are deleted when they are closed. (CVS 4277) check-in: cf4e3c15 user: danielk1977 tags: trunk
11:07
Add some further tests and a bugfix for the atomic-write optimization. (CVS 4276) check-in: 5f0fb894 user: danielk1977 tags: trunk
08:06
Add some tests for the atomic-write optimization. (CVS 4275) check-in: e2cc7b4a user: danielk1977 tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/pager.c.

14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
...
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
** The pager is used to access a database disk file.  It implements
** atomic commit and rollback through the use of a journal file that
** is separate from the database file.  The pager also implements file
** locking to prevent two processes from writing the same database
** file simultaneously, or one process from reading the database while
** another is writing.
**
** @(#) $Id: pager.c,v 1.368 2007/08/22 22:04:37 drh Exp $
*/
#ifndef SQLITE_OMIT_DISKIO
#include "sqliteInt.h"
#include <assert.h>
#include <string.h>

/*
................................................................................
    nSector = sqlite3OsSectorSize(fd);
    nPage = pPager->pageSize;
  }

  assert(SQLITE_IOCAP_ATOMIC512==(512>>8));
  assert(SQLITE_IOCAP_ATOMIC64K==(65536>>8));

  if( !fd->pMethods || (dc&(SQLITE_IOCAP_ATOMIC|(nPage<<8))&&nSector<=nPage) ){
    return JOURNAL_HDR_SZ(pPager) + JOURNAL_PG_SZ(pPager);
  }
  return 0;
}
#endif

/*







|







 







|







14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
...
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
** The pager is used to access a database disk file.  It implements
** atomic commit and rollback through the use of a journal file that
** is separate from the database file.  The pager also implements file
** locking to prevent two processes from writing the same database
** file simultaneously, or one process from reading the database while
** another is writing.
**
** @(#) $Id: pager.c,v 1.369 2007/08/23 11:07:10 danielk1977 Exp $
*/
#ifndef SQLITE_OMIT_DISKIO
#include "sqliteInt.h"
#include <assert.h>
#include <string.h>

/*
................................................................................
    nSector = sqlite3OsSectorSize(fd);
    nPage = pPager->pageSize;
  }

  assert(SQLITE_IOCAP_ATOMIC512==(512>>8));
  assert(SQLITE_IOCAP_ATOMIC64K==(65536>>8));

  if( !fd->pMethods || (dc&(SQLITE_IOCAP_ATOMIC|(nPage>>8))&&nSector<=nPage) ){
    return JOURNAL_HDR_SZ(pPager) + JOURNAL_PG_SZ(pPager);
  }
  return 0;
}
#endif

/*

Changes to src/test6.c.

15
16
17
18
19
20
21


22
23
24
25
26
27
28
...
172
173
174
175
176
177
178




179
180
181
182
183
184
185
...
214
215
216
217
218
219
220





221
222
223
224
225





226
227
228
229
230
231
232
233




234
235
236
237
238
239
240
** is used to test the ability of SQLite to recover from those situations.
*/
#if SQLITE_TEST          /* This file is used for the testing only */
#include "sqliteInt.h"
#include "tcl.h"

#ifndef SQLITE_OMIT_DISKIO  /* This file is a no-op if disk I/O is disabled */



typedef struct CrashFile CrashFile;
typedef struct CrashGlobal CrashGlobal;
typedef struct WriteBuffer WriteBuffer;

/*
** Method:
................................................................................
  if( !isCrash ){
    for(pWrite=g.pWriteList; pWrite; pWrite=pWrite->pNext){
      if( pWrite->pFile==pFile ){
        pFinal = pWrite;
      }
    }
  }





  sqlite3OsFileSize((sqlite3_file *)pFile, &iSize);

  ppPtr = &g.pWriteList;
  for(pWrite=*ppPtr; rc==SQLITE_OK && pWrite; pWrite=*ppPtr){
    sqlite3_file *pRealFile = pWrite->pFile->pRealFile;

................................................................................
          rc = sqlite3OsWrite(
              pRealFile, pWrite->zBuf, pWrite->nBuf, pWrite->iOffset
          );
        }else{
          rc = sqlite3OsTruncate(pRealFile, pWrite->iOffset);
        }
        *ppPtr = pWrite->pNext;





        sqlite3_free(pWrite);
        break;
      }
      case 2: {               /* Do nothing */
        ppPtr = &pWrite->pNext;





        break;
      }
      case 3: {               /* Trash sectors */
        u8 *zGarbage;
        int iFirst = (pWrite->iOffset/g.iSectorSize);
        int iLast = (pWrite->iOffset+pWrite->nBuf-1)/g.iSectorSize;

        assert(pWrite->zBuf);





        zGarbage = sqlite3_malloc(g.iSectorSize);
        if( zGarbage ){
          sqlite3_int64 i;
          for(i=iFirst; rc==SQLITE_OK && i<=iLast; i++){
            sqlite3Randomness(g.iSectorSize, zGarbage); 
            rc = sqlite3OsWrite(







>
>







 







>
>
>
>







 







>
>
>
>
>





>
>
>
>
>








>
>
>
>







15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
...
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
...
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
** is used to test the ability of SQLite to recover from those situations.
*/
#if SQLITE_TEST          /* This file is used for the testing only */
#include "sqliteInt.h"
#include "tcl.h"

#ifndef SQLITE_OMIT_DISKIO  /* This file is a no-op if disk I/O is disabled */

/* #define TRACE_CRASHTEST */

typedef struct CrashFile CrashFile;
typedef struct CrashGlobal CrashGlobal;
typedef struct WriteBuffer WriteBuffer;

/*
** Method:
................................................................................
  if( !isCrash ){
    for(pWrite=g.pWriteList; pWrite; pWrite=pWrite->pNext){
      if( pWrite->pFile==pFile ){
        pFinal = pWrite;
      }
    }
  }

#ifdef TRACE_CRASHTEST
  printf("Sync %s (is %s crash)\n", pFile->zName, (isCrash?"a":"not a"));
#endif

  sqlite3OsFileSize((sqlite3_file *)pFile, &iSize);

  ppPtr = &g.pWriteList;
  for(pWrite=*ppPtr; rc==SQLITE_OK && pWrite; pWrite=*ppPtr){
    sqlite3_file *pRealFile = pWrite->pFile->pRealFile;

................................................................................
          rc = sqlite3OsWrite(
              pRealFile, pWrite->zBuf, pWrite->nBuf, pWrite->iOffset
          );
        }else{
          rc = sqlite3OsTruncate(pRealFile, pWrite->iOffset);
        }
        *ppPtr = pWrite->pNext;
#ifdef TRACE_CRASHTEST
        if( isCrash ){
          printf("Writing %d bytes @ %d\n", pWrite->nBuf, (int)pWrite->iOffset);
        }
#endif
        sqlite3_free(pWrite);
        break;
      }
      case 2: {               /* Do nothing */
        ppPtr = &pWrite->pNext;
#ifdef TRACE_CRASHTEST
        if( isCrash ){
          printf("Omiting %d bytes @ %d\n", pWrite->nBuf, (int)pWrite->iOffset);
        }
#endif
        break;
      }
      case 3: {               /* Trash sectors */
        u8 *zGarbage;
        int iFirst = (pWrite->iOffset/g.iSectorSize);
        int iLast = (pWrite->iOffset+pWrite->nBuf-1)/g.iSectorSize;

        assert(pWrite->zBuf);

#ifdef TRACE_CRASHTEST
        printf("Trashing %d sectors @ sector %d\n", 1+iLast-iFirst, iFirst);
#endif

        zGarbage = sqlite3_malloc(g.iSectorSize);
        if( zGarbage ){
          sqlite3_int64 i;
          for(i=iFirst; rc==SQLITE_OK && i<=iLast; i++){
            sqlite3Randomness(g.iSectorSize, zGarbage); 
            rc = sqlite3OsWrite(

Added test/crash3.test.









































































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
# 2007 August 23
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#
# $Id: crash3.test,v 1.1 2007/08/23 11:07:10 danielk1977 Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

ifcapable !crashtest {
  finish_test
  return
}

proc do_test2 {name tcl res1 res2} {
  set script [subst -nocommands {
    do_test $name {
      set res1 {$res1}
      set res2 {$res2}
      set res [eval {$tcl}]
      if {[set res] eq [set res1] || [set res] eq [set res2]} {
        set res "{[set res1]} or {[set res2]}"
      }
      set res
    } {{$res1} or {$res2}}
  }]
  uplevel $script
}

# Each iteration of the following loop sets up the database to contain
# the following schema and data:
#
#    CREATE TABLE abc(a, b, c);
#    INSERT INTO abc VALUES(1, 2, 3);
#
# Then execute the SQL statement, scheduling a crash for part-way through
# the first sync() of either the database file or the journal file (often
# the journal file is not required - meaning no crash occurs).
#
# After the crash (or absence of a crash), open the database and 
# verify that:
#
#   * The integrity check passes, and
#   * The contents of table abc is either {1 2 3} or the value specified
#     to the right of the SQL statement below.
#
# The procedure is repeated 10 times for each SQL statement. Five times
# with the crash scheduled for midway through the first journal sync (if 
# any), and five times with the crash midway through the database sync.
#
set tn 1
foreach {sql res2} [list \
  {INSERT INTO abc VALUES(4, 5, 6)}                    {1 2 3 4 5 6} \
  {DELETE FROM abc}                                    {}    \
  {INSERT INTO abc SELECT * FROM abc}                  {1 2 3 1 2 3} \
  {UPDATE abc SET a = 2}                               {2 2 3}       \
  {INSERT INTO abc VALUES(4, 5, randstr(1000,1000))}   {n/a} \
  {CREATE TABLE def(d, e, f)}                          {n/a} \
] {
  for {set ii 0} {$ii < 10} {incr ii} {

    db close
    file delete -force test.db test.db-journal
    sqlite3 db test.db
    do_test crash3-1.$tn.1 {
      execsql {
        BEGIN;
        CREATE TABLE abc(a, b, c);
        INSERT INTO abc VALUES(1, 2, 3);
        COMMIT;
      }
    } {}
    db close
  
    set crashfile test.db
    if {($ii%2)==0} { append crashfile -journal }
    set rand "SELECT randstr($tn,$tn);"
    do_test crash3-1.$tn.2 [subst {
      crashsql -file $crashfile -char atomic {$rand $sql}
      sqlite3 db test.db
      execsql { PRAGMA integrity_check; }
    }] {ok}
  
    do_test2 crash3-1.$tn.3 {
      execsql { SELECT * FROM abc }
    } {1 2 3} $res2

    incr tn
  }
}

finish_test

Changes to test/io.test.

9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
...
260
261
262
263
264
265
266
267



















































268
269
270
271
#
#***********************************************************************
#
# The focus of this file is testing some specific characteristics of the 
# IO traffic generated by SQLite (making sure SQLite is not writing out
# more database pages than it has to, stuff like that).
#
# $Id: io.test,v 1.3 2007/08/23 08:06:45 danielk1977 Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# Test summary:
#
# io-1.* -  Test that quick-balance does not journal pages unnecessarily.
................................................................................
} {}
do_test io-2.8.3 {
  execsql {
    ROLLBACK;
    SELECT * FROM abc;
  }
} {1 2 3 4 5 6 7 8}




















































sqlite3_simulate_device -char {} -sectorsize 0

finish_test








|







 








>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>




9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
...
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
#
#***********************************************************************
#
# The focus of this file is testing some specific characteristics of the 
# IO traffic generated by SQLite (making sure SQLite is not writing out
# more database pages than it has to, stuff like that).
#
# $Id: io.test,v 1.4 2007/08/23 11:07:10 danielk1977 Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# Test summary:
#
# io-1.* -  Test that quick-balance does not journal pages unnecessarily.
................................................................................
} {}
do_test io-2.8.3 {
  execsql {
    ROLLBACK;
    SELECT * FROM abc;
  }
} {1 2 3 4 5 6 7 8}

# Test that the atomic write optimisation is not enabled if the sector
# size is larger than the page-size.
#
do_test io-2.9.1 {
  sqlite3_simulate_device -char atomic -sectorsize 2048
  execsql {
    BEGIN;
    INSERT INTO abc VALUES(9, 10);
  }
  file exists test.db-journal
} {1}
do_test io-2.9.2 {
  execsql { ROLLBACK; }
  db close
  file delete -force test.db test.db-journal
  sqlite3 db test.db
  execsql {
    PRAGMA page_size = 2048;
    CREATE TABLE abc(a, b);
  }
  execsql {
    BEGIN;
    INSERT INTO abc VALUES(9, 10);
  }
  file exists test.db-journal
} {0}
do_test io-2.9.3 {
  execsql { COMMIT }
} {}

# Test a couple of the more specific IOCAP_ATOMIC flags 
# (i.e IOCAP_ATOMIC2K etc.).
#
do_test io-2.10.1 {
  sqlite3_simulate_device -char atomic1k
  execsql {
    BEGIN;
    INSERT INTO abc VALUES(11, 12);
  }
  file exists test.db-journal
} {1}
do_test io-2.10.2 {
  execsql { ROLLBACK }
  sqlite3_simulate_device -char atomic2k
  execsql {
    BEGIN;
    INSERT INTO abc VALUES(11, 12);
  }
  file exists test.db-journal
} {0}

sqlite3_simulate_device -char {} -sectorsize 0

finish_test

Changes to test/tester.tcl.

7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
...
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362

363
364
365
366
367
368
369
370
371
372

373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file implements some common TCL routines used for regression
# testing the SQLite library
#
# $Id: tester.tcl,v 1.87 2007/08/23 02:47:54 drh Exp $

# Make sure tclsqlite3 was compiled correctly.  Abort now with an
# error message if not.
#
if {[sqlite3 -tcl-uses-utf]} {
  if {"\u1234"=="u1234"} {
    puts stderr "***** BUILD PROBLEM *****"
................................................................................
#
# The return value is a list of two elements. The first element is a
# boolean, indicating whether or not the process actually crashed or
# reported some other error. The second element in the returned list is the
# error message. This is "child process exited abnormally" if the crash
# occured.
#
#   crashsql -delay CRASHDELAY -file CRASHFILE ?-blocksize BLOCKSIZE $sql
#
proc crashsql {args} {
  if {$::tcl_platform(platform)!="unix"} {
    error "crashsql should only be used on unix"
  }

  set blocksize ""
  set crashdelay 1
  set crashfile ""

  set sql [lindex $args end]
  
  for {set ii 0} {$ii < [llength $args]-1} {incr ii 2} {
    set z [lindex $args $ii]
    set n [string length $z]
    set z2 [lindex $args [expr $ii+1]]

    if     {$n>1 && [string first $z -delay]==0}     {set crashdelay $z2} \
    elseif {$n>1 && [string first $z -file]==0}      {set crashfile $z2}  \
    elseif {$n>1 && [string first $z -blocksize]==0} {set blocksize "-s $z2" } \

    else   { error "Unrecognized option: $z" }
  }

  if {$crashfile eq ""} {
    error "Compulsory option -file missing"
  }

  set cfile [file join [pwd] $crashfile]

  set f [open crash.tcl w]
  puts $f "sqlite3_crashparams $blocksize $crashdelay $cfile"
  puts $f "set sqlite_pending_byte $::sqlite_pending_byte"
  puts $f "sqlite3 db test.db"

  # This block sets the cache size of the main database to 10
  # pages. This is done in case the build is configured to omit
  # "PRAGMA cache_size".
  puts $f {db eval {SELECT * FROM sqlite_master;}}







|







 







|









>










>










|







7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
...
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file implements some common TCL routines used for regression
# testing the SQLite library
#
# $Id: tester.tcl,v 1.88 2007/08/23 11:07:10 danielk1977 Exp $

# Make sure tclsqlite3 was compiled correctly.  Abort now with an
# error message if not.
#
if {[sqlite3 -tcl-uses-utf]} {
  if {"\u1234"=="u1234"} {
    puts stderr "***** BUILD PROBLEM *****"
................................................................................
#
# The return value is a list of two elements. The first element is a
# boolean, indicating whether or not the process actually crashed or
# reported some other error. The second element in the returned list is the
# error message. This is "child process exited abnormally" if the crash
# occured.
#
#   crashsql -delay CRASHDELAY -file CRASHFILE ?-blocksize BLOCKSIZE? $sql
#
proc crashsql {args} {
  if {$::tcl_platform(platform)!="unix"} {
    error "crashsql should only be used on unix"
  }

  set blocksize ""
  set crashdelay 1
  set crashfile ""
  set dc ""
  set sql [lindex $args end]
  
  for {set ii 0} {$ii < [llength $args]-1} {incr ii 2} {
    set z [lindex $args $ii]
    set n [string length $z]
    set z2 [lindex $args [expr $ii+1]]

    if     {$n>1 && [string first $z -delay]==0}     {set crashdelay $z2} \
    elseif {$n>1 && [string first $z -file]==0}      {set crashfile $z2}  \
    elseif {$n>1 && [string first $z -blocksize]==0} {set blocksize "-s $z2" } \
    elseif {$n>1 && [string first $z -characteristics]==0} {set dc "-c {$z2}" } \
    else   { error "Unrecognized option: $z" }
  }

  if {$crashfile eq ""} {
    error "Compulsory option -file missing"
  }

  set cfile [file join [pwd] $crashfile]

  set f [open crash.tcl w]
  puts $f "sqlite3_crashparams $blocksize $dc $crashdelay $cfile"
  puts $f "set sqlite_pending_byte $::sqlite_pending_byte"
  puts $f "sqlite3 db test.db"

  # This block sets the cache size of the main database to 10
  # pages. This is done in case the build is configured to omit
  # "PRAGMA cache_size".
  puts $f {db eval {SELECT * FROM sqlite_master;}}