SQLite

Check-in [b956ddca75]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix a problem with writing to databases larger than 2^32 bytes with WAL mode.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: b956ddca75d64ba662fa0b03b643822d836b6501
User & Date: dan 2010-07-07 09:48:45.000
Context
2010-07-07
11:05
Change the async-IO extension to return SQLITE_IOERR_SHORT_READ when appropriate. This prevents a valgrind warning in the test suite. (check-in: d9e3287900 user: dan tags: trunk)
09:48
Fix a problem with writing to databases larger than 2^32 bytes with WAL mode. (check-in: b956ddca75 user: dan tags: trunk)
01:52
Remote the SQLITE_OMIT_GLOBALRECOVER macro - it is subsumed by SQLITE_OMIT_DEPRECATED. Add support to sqlite3_compileoption_used() for SQLITE_OMIT_WAL. (check-in: 21a2db908b user: drh tags: trunk)
Changes
Unified Diff Ignore Whitespace Patch
Changes to src/wal.c.
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406

/*
** Return the offset of frame iFrame in the write-ahead log file, 
** assuming a database page size of szPage bytes. The offset returned
** is to the start of the write-ahead log frame-header.
*/
#define walFrameOffset(iFrame, szPage) (                               \
  WAL_HDRSIZE + ((iFrame)-1)*((szPage)+WAL_FRAME_HDRSIZE)        \
)

/*
** An open write-ahead log file is represented by an instance of the
** following object.
*/
struct Wal {







|







392
393
394
395
396
397
398
399
400
401
402
403
404
405
406

/*
** Return the offset of frame iFrame in the write-ahead log file, 
** assuming a database page size of szPage bytes. The offset returned
** is to the start of the write-ahead log frame-header.
*/
#define walFrameOffset(iFrame, szPage) (                               \
  WAL_HDRSIZE + ((iFrame)-1)*(i64)((szPage)+WAL_FRAME_HDRSIZE)         \
)

/*
** An open write-ahead log file is represented by an instance of the
** following object.
*/
struct Wal {
1573
1574
1575
1576
1577
1578
1579
1580


1581
1582

1583
1584
1585
1586
1587
1588
1589
    /* Iterate through the contents of the WAL, copying data to the db file. */
    while( rc==SQLITE_OK && 0==walIteratorNext(pIter, &iDbpage, &iFrame) ){
      assert( walFramePgno(pWal, iFrame)==iDbpage );
      if( iFrame<=nBackfill || iFrame>mxSafeFrame ) continue;
      rc = sqlite3OsRead(pWal->pWalFd, zBuf, szPage, 
          walFrameOffset(iFrame, szPage) + WAL_FRAME_HDRSIZE
      );
      if( rc!=SQLITE_OK ) break;


      rc = sqlite3OsWrite(pWal->pDbFd, zBuf, szPage, (iDbpage-1)*szPage);
      if( rc!=SQLITE_OK ) break;

    }

    /* If work was actually accomplished... */
    if( rc==SQLITE_OK ){
      if( mxSafeFrame==walIndexHdr(pWal)->mxFrame ){
        rc = sqlite3OsTruncate(pWal->pDbFd, ((i64)pWal->hdr.nPage*(i64)szPage));
        if( rc==SQLITE_OK && sync_flags ){







|
>
>
|
<
>







1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583

1584
1585
1586
1587
1588
1589
1590
1591
    /* Iterate through the contents of the WAL, copying data to the db file. */
    while( rc==SQLITE_OK && 0==walIteratorNext(pIter, &iDbpage, &iFrame) ){
      assert( walFramePgno(pWal, iFrame)==iDbpage );
      if( iFrame<=nBackfill || iFrame>mxSafeFrame ) continue;
      rc = sqlite3OsRead(pWal->pWalFd, zBuf, szPage, 
          walFrameOffset(iFrame, szPage) + WAL_FRAME_HDRSIZE
      );
      if( rc==SQLITE_OK ){
        i64 iOffset = (i64)(iDbpage-1)*szPage;
        testcase( iOffset > (((i64)1)<<32) );
        rc = sqlite3OsWrite(pWal->pDbFd, zBuf, szPage, iOffset);

      }
    }

    /* If work was actually accomplished... */
    if( rc==SQLITE_OK ){
      if( mxSafeFrame==walIndexHdr(pWal)->mxFrame ){
        rc = sqlite3OsTruncate(pWal->pDbFd, ((i64)pWal->hdr.nPage*(i64)szPage));
        if( rc==SQLITE_OK && sync_flags ){
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
  assert( pWal->szPage==szPage );

  /* Write the log file. */
  for(p=pList; p; p=p->pDirty){
    u32 nDbsize;                  /* Db-size field for frame header */
    i64 iOffset;                  /* Write offset in log file */
    void *pData;
   
   
    iOffset = walFrameOffset(++iFrame, szPage);
    
    /* Populate and write the frame header */
    nDbsize = (isCommit && p->pDirty==0) ? nTruncate : 0;
#if defined(SQLITE_HAS_CODEC)
    if( (pData = sqlite3PagerCodec(p))==0 ) return SQLITE_NOMEM;







<







2410
2411
2412
2413
2414
2415
2416

2417
2418
2419
2420
2421
2422
2423
  assert( pWal->szPage==szPage );

  /* Write the log file. */
  for(p=pList; p; p=p->pDirty){
    u32 nDbsize;                  /* Db-size field for frame header */
    i64 iOffset;                  /* Write offset in log file */
    void *pData;

   
    iOffset = walFrameOffset(++iFrame, szPage);
    
    /* Populate and write the frame header */
    nDbsize = (isCommit && p->pDirty==0) ? nTruncate : 0;
#if defined(SQLITE_HAS_CODEC)
    if( (pData = sqlite3PagerCodec(p))==0 ) return SQLITE_NOMEM;
Changes to test/permutations.test.
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
#   coverage-wal
#
test_suite "coverage-wal" -description {
  Coverage tests for file wal.c.
} -files {
  wal.test       wal2.test     wal3.test       walmode.test    
  walbak.test    walhook.test  walcrash2.test  walcksum.test
  walfault.test
} 

test_suite "coverage-pager" -description {
  Coverage tests for file pager.c.
} -files {
  pager1.test    pager2.test  pagerfault.test  pagerfault2.test
  walfault.test  walbak.test  journal2.test    tkt-9d68c883.test







|







161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
#   coverage-wal
#
test_suite "coverage-wal" -description {
  Coverage tests for file wal.c.
} -files {
  wal.test       wal2.test     wal3.test       walmode.test    
  walbak.test    walhook.test  walcrash2.test  walcksum.test
  walfault.test  walbig.test
} 

test_suite "coverage-pager" -description {
  Coverage tests for file pager.c.
} -files {
  pager1.test    pager2.test  pagerfault.test  pagerfault2.test
  walfault.test  walbak.test  journal2.test    tkt-9d68c883.test
Added test/walbig.test.


















































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
# 2010 July 07
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file implements regression tests for SQLite library.  The
# focus of this script testing the ability of SQLite to handle database
# files larger than 4GB in WAL mode.
#


set testdir [file dirname $argv0]
source $testdir/tester.tcl

# Do not use a codec for this file, as the database is manipulated using
# external methods (the [fake_big_file] and [hexio_write] commands).
#
do_not_use_codec

# If SQLITE_DISABLE_LFS is defined, omit this file.
ifcapable !lfs {
  finish_test
  return
}

set a_string_counter 1
proc a_string {n} {
  incr ::a_string_counter
  string range [string repeat "${::a_string_counter}." $n] 1 $n
}
db func a_string a_string

do_test walbig-1.0 {
  execsql {
    PRAGMA journal_mode = WAL;
    CREATE TABLE t1(a PRIMARY KEY, b UNIQUE);
    INSERT INTO t1 VALUES(a_string(300), a_string(500));
    INSERT INTO t1 SELECT a_string(300), a_string(500) FROM t1;
    INSERT INTO t1 SELECT a_string(300), a_string(500) FROM t1;
    INSERT INTO t1 SELECT a_string(300), a_string(500) FROM t1;
  }
} {wal}

db close
if {[catch {fake_big_file 5000 [pwd]/test.db}]} {
  puts "**** Unable to create a file larger than 5000 MB. *****"
  finish_test
  return
}
hexio_write test.db 28 00000000

sqlite3 db test.db
db func a_string a_string
do_test walbig-1.1 {
  execsql { INSERT INTO t1 SELECT a_string(300), a_string(500) FROM t1 }
} {}
db close

sqlite3 db test.db
do_test walbig-1.2 {
  execsql { SELECT a FROM t1 ORDER BY a }
} [lsort [execsql { SELECT a FROM t1 ORDER BY rowid }]]

do_test walbig-1.3 {
  execsql { SELECT b FROM t1 ORDER BY b }
} [lsort [execsql { SELECT b FROM t1 ORDER BY rowid }]]

finish_test