/ Check-in [b956ddca]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix a problem with writing to databases larger than 2^32 bytes with WAL mode.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1:b956ddca75d64ba662fa0b03b643822d836b6501
User & Date: dan 2010-07-07 09:48:45
Context
2010-07-07
11:05
Change the async-IO extension to return SQLITE_IOERR_SHORT_READ when appropriate. This prevents a valgrind warning in the test suite. check-in: d9e32879 user: dan tags: trunk
09:48
Fix a problem with writing to databases larger than 2^32 bytes with WAL mode. check-in: b956ddca user: dan tags: trunk
01:52
Remote the SQLITE_OMIT_GLOBALRECOVER macro - it is subsumed by SQLITE_OMIT_DEPRECATED. Add support to sqlite3_compileoption_used() for SQLITE_OMIT_WAL. check-in: 21a2db90 user: drh tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/wal.c.

392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
....
1573
1574
1575
1576
1577
1578
1579
1580


1581
1582

1583
1584
1585
1586
1587
1588
1589
....
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422

/*
** Return the offset of frame iFrame in the write-ahead log file, 
** assuming a database page size of szPage bytes. The offset returned
** is to the start of the write-ahead log frame-header.
*/
#define walFrameOffset(iFrame, szPage) (                               \
  WAL_HDRSIZE + ((iFrame)-1)*((szPage)+WAL_FRAME_HDRSIZE)        \
)

/*
** An open write-ahead log file is represented by an instance of the
** following object.
*/
struct Wal {
................................................................................
    /* Iterate through the contents of the WAL, copying data to the db file. */
    while( rc==SQLITE_OK && 0==walIteratorNext(pIter, &iDbpage, &iFrame) ){
      assert( walFramePgno(pWal, iFrame)==iDbpage );
      if( iFrame<=nBackfill || iFrame>mxSafeFrame ) continue;
      rc = sqlite3OsRead(pWal->pWalFd, zBuf, szPage, 
          walFrameOffset(iFrame, szPage) + WAL_FRAME_HDRSIZE
      );
      if( rc!=SQLITE_OK ) break;


      rc = sqlite3OsWrite(pWal->pDbFd, zBuf, szPage, (iDbpage-1)*szPage);
      if( rc!=SQLITE_OK ) break;

    }

    /* If work was actually accomplished... */
    if( rc==SQLITE_OK ){
      if( mxSafeFrame==walIndexHdr(pWal)->mxFrame ){
        rc = sqlite3OsTruncate(pWal->pDbFd, ((i64)pWal->hdr.nPage*(i64)szPage));
        if( rc==SQLITE_OK && sync_flags ){
................................................................................
  assert( pWal->szPage==szPage );

  /* Write the log file. */
  for(p=pList; p; p=p->pDirty){
    u32 nDbsize;                  /* Db-size field for frame header */
    i64 iOffset;                  /* Write offset in log file */
    void *pData;
   
   
    iOffset = walFrameOffset(++iFrame, szPage);
    
    /* Populate and write the frame header */
    nDbsize = (isCommit && p->pDirty==0) ? nTruncate : 0;
#if defined(SQLITE_HAS_CODEC)
    if( (pData = sqlite3PagerCodec(p))==0 ) return SQLITE_NOMEM;







|







 







|
>
>
|
<
>







 







<







392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
....
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583

1584
1585
1586
1587
1588
1589
1590
1591
....
2410
2411
2412
2413
2414
2415
2416

2417
2418
2419
2420
2421
2422
2423

/*
** Return the offset of frame iFrame in the write-ahead log file, 
** assuming a database page size of szPage bytes. The offset returned
** is to the start of the write-ahead log frame-header.
*/
#define walFrameOffset(iFrame, szPage) (                               \
  WAL_HDRSIZE + ((iFrame)-1)*(i64)((szPage)+WAL_FRAME_HDRSIZE)         \
)

/*
** An open write-ahead log file is represented by an instance of the
** following object.
*/
struct Wal {
................................................................................
    /* Iterate through the contents of the WAL, copying data to the db file. */
    while( rc==SQLITE_OK && 0==walIteratorNext(pIter, &iDbpage, &iFrame) ){
      assert( walFramePgno(pWal, iFrame)==iDbpage );
      if( iFrame<=nBackfill || iFrame>mxSafeFrame ) continue;
      rc = sqlite3OsRead(pWal->pWalFd, zBuf, szPage, 
          walFrameOffset(iFrame, szPage) + WAL_FRAME_HDRSIZE
      );
      if( rc==SQLITE_OK ){
        i64 iOffset = (i64)(iDbpage-1)*szPage;
        testcase( iOffset > (((i64)1)<<32) );
        rc = sqlite3OsWrite(pWal->pDbFd, zBuf, szPage, iOffset);

      }
    }

    /* If work was actually accomplished... */
    if( rc==SQLITE_OK ){
      if( mxSafeFrame==walIndexHdr(pWal)->mxFrame ){
        rc = sqlite3OsTruncate(pWal->pDbFd, ((i64)pWal->hdr.nPage*(i64)szPage));
        if( rc==SQLITE_OK && sync_flags ){
................................................................................
  assert( pWal->szPage==szPage );

  /* Write the log file. */
  for(p=pList; p; p=p->pDirty){
    u32 nDbsize;                  /* Db-size field for frame header */
    i64 iOffset;                  /* Write offset in log file */
    void *pData;

   
    iOffset = walFrameOffset(++iFrame, szPage);
    
    /* Populate and write the frame header */
    nDbsize = (isCommit && p->pDirty==0) ? nTruncate : 0;
#if defined(SQLITE_HAS_CODEC)
    if( (pData = sqlite3PagerCodec(p))==0 ) return SQLITE_NOMEM;

Changes to test/permutations.test.

161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
#   coverage-wal
#
test_suite "coverage-wal" -description {
  Coverage tests for file wal.c.
} -files {
  wal.test       wal2.test     wal3.test       walmode.test    
  walbak.test    walhook.test  walcrash2.test  walcksum.test
  walfault.test
} 

test_suite "coverage-pager" -description {
  Coverage tests for file pager.c.
} -files {
  pager1.test    pager2.test  pagerfault.test  pagerfault2.test
  walfault.test  walbak.test  journal2.test    tkt-9d68c883.test







|







161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
#   coverage-wal
#
test_suite "coverage-wal" -description {
  Coverage tests for file wal.c.
} -files {
  wal.test       wal2.test     wal3.test       walmode.test    
  walbak.test    walhook.test  walcrash2.test  walcksum.test
  walfault.test  walbig.test
} 

test_suite "coverage-pager" -description {
  Coverage tests for file pager.c.
} -files {
  pager1.test    pager2.test  pagerfault.test  pagerfault2.test
  walfault.test  walbak.test  journal2.test    tkt-9d68c883.test

Added test/walbig.test.



















































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
# 2010 July 07
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file implements regression tests for SQLite library.  The
# focus of this script testing the ability of SQLite to handle database
# files larger than 4GB in WAL mode.
#


set testdir [file dirname $argv0]
source $testdir/tester.tcl

# Do not use a codec for this file, as the database is manipulated using
# external methods (the [fake_big_file] and [hexio_write] commands).
#
do_not_use_codec

# If SQLITE_DISABLE_LFS is defined, omit this file.
ifcapable !lfs {
  finish_test
  return
}

set a_string_counter 1
proc a_string {n} {
  incr ::a_string_counter
  string range [string repeat "${::a_string_counter}." $n] 1 $n
}
db func a_string a_string

do_test walbig-1.0 {
  execsql {
    PRAGMA journal_mode = WAL;
    CREATE TABLE t1(a PRIMARY KEY, b UNIQUE);
    INSERT INTO t1 VALUES(a_string(300), a_string(500));
    INSERT INTO t1 SELECT a_string(300), a_string(500) FROM t1;
    INSERT INTO t1 SELECT a_string(300), a_string(500) FROM t1;
    INSERT INTO t1 SELECT a_string(300), a_string(500) FROM t1;
  }
} {wal}

db close
if {[catch {fake_big_file 5000 [pwd]/test.db}]} {
  puts "**** Unable to create a file larger than 5000 MB. *****"
  finish_test
  return
}
hexio_write test.db 28 00000000

sqlite3 db test.db
db func a_string a_string
do_test walbig-1.1 {
  execsql { INSERT INTO t1 SELECT a_string(300), a_string(500) FROM t1 }
} {}
db close

sqlite3 db test.db
do_test walbig-1.2 {
  execsql { SELECT a FROM t1 ORDER BY a }
} [lsort [execsql { SELECT a FROM t1 ORDER BY rowid }]]

do_test walbig-1.3 {
  execsql { SELECT b FROM t1 ORDER BY b }
} [lsort [execsql { SELECT b FROM t1 ORDER BY rowid }]]

finish_test