SQLite

Check-in [393741eba3]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add codec support to wal mode.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 393741eba353d5d242b8e3c96db3ea2b92228036
User & Date: dan 2010-06-22 15:18:44.000
Context
2010-06-22
21:15
Disable code used only by the codec when the codec is not deployed. (check-in: 2c90276e34 user: drh tags: trunk)
15:18
Add codec support to wal mode. (check-in: 393741eba3 user: dan tags: trunk)
14:49
When trying to transition from journal_mode MEMORY to WAL, use OFF as an intermediate journal mode. (check-in: 4775b8f9a9 user: drh tags: trunk)
Changes
Unified Diff Ignore Whitespace Patch
Changes to src/pager.c.
6097
6098
6099
6100
6101
6102
6103
















6104
6105
6106
      /* If we cannot get an EXCLUSIVE lock, downgrade the PENDING lock
      ** that we did get back to SHARED. */
      sqlite3OsUnlock(pPager->fd, SQLITE_LOCK_SHARED);
    }
  }
  return rc;
}
















#endif

#endif /* SQLITE_OMIT_DISKIO */







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>



6097
6098
6099
6100
6101
6102
6103
6104
6105
6106
6107
6108
6109
6110
6111
6112
6113
6114
6115
6116
6117
6118
6119
6120
6121
6122
      /* If we cannot get an EXCLUSIVE lock, downgrade the PENDING lock
      ** that we did get back to SHARED. */
      sqlite3OsUnlock(pPager->fd, SQLITE_LOCK_SHARED);
    }
  }
  return rc;
}

#ifdef SQLITE_HAS_CODEC
/*
** This function is called by the wal module when writing page content
** into the log file.
**
** This function returns a pointer to a buffer containing the encrypted
** page content. If a malloc fails, this function may return NULL.
*/
void *sqlite3PagerCodec(PgHdr *pPg){
  void *aData = 0;
  CODEC2(pPg->pPager, pPg->pData, pPg->pgno, 6, return 0, aData);
  return aData;
}
#endif

#endif

#endif /* SQLITE_OMIT_DISKIO */
Changes to src/pager.h.
152
153
154
155
156
157
158






159
160
161
162
163
164
165
const char *sqlite3PagerJournalname(Pager*);
int sqlite3PagerNosync(Pager*);
void *sqlite3PagerTempSpace(Pager*);
int sqlite3PagerIsMemdb(Pager*);

/* Functions used to truncate the database file. */
void sqlite3PagerTruncateImage(Pager*,Pgno);







/* Functions to support testing and debugging. */
#if !defined(NDEBUG) || defined(SQLITE_TEST)
  Pgno sqlite3PagerPagenumber(DbPage*);
  int sqlite3PagerIswriteable(DbPage*);
#endif
#ifdef SQLITE_TEST







>
>
>
>
>
>







152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
const char *sqlite3PagerJournalname(Pager*);
int sqlite3PagerNosync(Pager*);
void *sqlite3PagerTempSpace(Pager*);
int sqlite3PagerIsMemdb(Pager*);

/* Functions used to truncate the database file. */
void sqlite3PagerTruncateImage(Pager*,Pgno);

#if defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_WAL)
void *sqlite3PagerCodec(DbPage *);
#else
#define sqlite3PagerCodec(x) (x->pData)
#endif

/* Functions to support testing and debugging. */
#if !defined(NDEBUG) || defined(SQLITE_TEST)
  Pgno sqlite3PagerPagenumber(DbPage*);
  int sqlite3PagerIswriteable(DbPage*);
#endif
#ifdef SQLITE_TEST
Changes to src/wal.c.
2282
2283
2284
2285
2286
2287
2288

2289

2290
2291
2292
2293

2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317


2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
  }
  assert( pWal->szPage==szPage );

  /* Write the log file. */
  for(p=pList; p; p=p->pDirty){
    u32 nDbsize;                  /* Db-size field for frame header */
    i64 iOffset;                  /* Write offset in log file */



    iOffset = walFrameOffset(++iFrame, szPage);
    
    /* Populate and write the frame header */
    nDbsize = (isCommit && p->pDirty==0) ? nTruncate : 0;

    walEncodeFrame(pWal, p->pgno, nDbsize, p->pData, aFrame);
    rc = sqlite3OsWrite(pWal->pWalFd, aFrame, sizeof(aFrame), iOffset);
    if( rc!=SQLITE_OK ){
      return rc;
    }

    /* Write the page data */
    rc = sqlite3OsWrite(pWal->pWalFd, p->pData, szPage, iOffset+sizeof(aFrame));
    if( rc!=SQLITE_OK ){
      return rc;
    }
    pLast = p;
  }

  /* Sync the log file if the 'isSync' flag was specified. */
  if( sync_flags ){
    i64 iSegment = sqlite3OsSectorSize(pWal->pWalFd);
    i64 iOffset = walFrameOffset(iFrame+1, szPage);

    assert( isCommit );
    assert( iSegment>0 );

    iSegment = (((iOffset+iSegment-1)/iSegment) * iSegment);
    while( iOffset<iSegment ){


      walEncodeFrame(pWal, pLast->pgno, nTruncate, pLast->pData, aFrame);
      rc = sqlite3OsWrite(pWal->pWalFd, aFrame, sizeof(aFrame), iOffset);
      if( rc!=SQLITE_OK ){
        return rc;
      }

      iOffset += WAL_FRAME_HDRSIZE;
      rc = sqlite3OsWrite(pWal->pWalFd, pLast->pData, szPage, iOffset); 
      if( rc!=SQLITE_OK ){
        return rc;
      }
      nLast++;
      iOffset += szPage;
    }








>
|
>




>
|






|
















>
>
|




<

|







2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327

2328
2329
2330
2331
2332
2333
2334
2335
2336
  }
  assert( pWal->szPage==szPage );

  /* Write the log file. */
  for(p=pList; p; p=p->pDirty){
    u32 nDbsize;                  /* Db-size field for frame header */
    i64 iOffset;                  /* Write offset in log file */
    void *pData;
   
   
    iOffset = walFrameOffset(++iFrame, szPage);
    
    /* Populate and write the frame header */
    nDbsize = (isCommit && p->pDirty==0) ? nTruncate : 0;
    if( (pData = sqlite3PagerCodec(p))==0 ) return SQLITE_NOMEM;
    walEncodeFrame(pWal, p->pgno, nDbsize, pData, aFrame);
    rc = sqlite3OsWrite(pWal->pWalFd, aFrame, sizeof(aFrame), iOffset);
    if( rc!=SQLITE_OK ){
      return rc;
    }

    /* Write the page data */
    rc = sqlite3OsWrite(pWal->pWalFd, pData, szPage, iOffset+sizeof(aFrame));
    if( rc!=SQLITE_OK ){
      return rc;
    }
    pLast = p;
  }

  /* Sync the log file if the 'isSync' flag was specified. */
  if( sync_flags ){
    i64 iSegment = sqlite3OsSectorSize(pWal->pWalFd);
    i64 iOffset = walFrameOffset(iFrame+1, szPage);

    assert( isCommit );
    assert( iSegment>0 );

    iSegment = (((iOffset+iSegment-1)/iSegment) * iSegment);
    while( iOffset<iSegment ){
      void *pData;
      if( (pData = sqlite3PagerCodec(pLast))==0 ) return SQLITE_NOMEM;
      walEncodeFrame(pWal, pLast->pgno, nTruncate, pData, aFrame);
      rc = sqlite3OsWrite(pWal->pWalFd, aFrame, sizeof(aFrame), iOffset);
      if( rc!=SQLITE_OK ){
        return rc;
      }

      iOffset += WAL_FRAME_HDRSIZE;
      rc = sqlite3OsWrite(pWal->pWalFd, pData, szPage, iOffset); 
      if( rc!=SQLITE_OK ){
        return rc;
      }
      nLast++;
      iOffset += szPage;
    }

Changes to test/lock_common.tcl.
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
#
#***********************************************************************
# This file contains code used by several different test scripts. The
# code in this file allows testfixture to control another process (or
# processes) to test locking.
#

do_not_use_codec

proc do_multiclient_test {varname script} {

  foreach code [list {
    set ::code2_chan [launch_testfixture]
    set ::code3_chan [launch_testfixture]
    proc code2 {tcl} { testfixture $::code2_chan $tcl }
    proc code3 {tcl} { testfixture $::code3_chan $tcl }







<
<







9
10
11
12
13
14
15


16
17
18
19
20
21
22
#
#***********************************************************************
# This file contains code used by several different test scripts. The
# code in this file allows testfixture to control another process (or
# processes) to test locking.
#



proc do_multiclient_test {varname script} {

  foreach code [list {
    set ::code2_chan [launch_testfixture]
    set ::code3_chan [launch_testfixture]
    proc code2 {tcl} { testfixture $::code2_chan $tcl }
    proc code3 {tcl} { testfixture $::code3_chan $tcl }
62
63
64
65
66
67
68

69
70
71
72
73
74
75
}

# Launch another testfixture process to be controlled by this one. A
# channel name is returned that may be passed as the first argument to proc
# 'testfixture' to execute a command. The child testfixture process is shut
# down by closing the channel.
proc launch_testfixture {} {

  set prg [info nameofexec]
  if {$prg eq ""} {
    set prg [file join . testfixture]
  }
  set chan [open "|$prg tf_main.tcl" r+]
  fconfigure $chan -buffering line
  testfixture $chan "sqlite3_test_control_pending_byte $::sqlite_pending_byte"







>







60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
}

# Launch another testfixture process to be controlled by this one. A
# channel name is returned that may be passed as the first argument to proc
# 'testfixture' to execute a command. The child testfixture process is shut
# down by closing the channel.
proc launch_testfixture {} {
  write_main_loop
  set prg [info nameofexec]
  if {$prg eq ""} {
    set prg [file join . testfixture]
  }
  set chan [open "|$prg tf_main.tcl" r+]
  fconfigure $chan -buffering line
  testfixture $chan "sqlite3_test_control_pending_byte $::sqlite_pending_byte"
126
127
128
129
130
131
132












133
134
135

136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153

154

155
156

# Write the main loop for the child testfixture processes into file
# tf_main.tcl. The parent (this script) interacts with the child processes
# via a two way pipe. The parent writes a script to the stdin of the child
# process, followed by the word "OVER" on a line of its own. The child
# process evaluates the script and writes the results to stdout, followed
# by an "OVER" of its own.












set f [open tf_main.tcl w]
puts $f {
  set l [open log w]

  set script ""
  while {![eof stdin]} {
    flush stdout
    set line [gets stdin]
    puts $l "READ $line"
    if { $line == "OVER" } {
      set rc [catch {eval $script} result]
      puts [list $rc $result]
      puts $l "WRITE [list $rc $result]"
      puts OVER
      puts $l "WRITE OVER"
      flush stdout
      set script ""
    } else {
      append script $line
      append script "\n"
    }
  }

  close $l

}
close $f







>
>
>
>
>
>
>
>
>
>
>
>
|
|
<
>
|
|
|
|
<
|
|
|
<
|
<
|
|
|
|
|
|
|
>
|
>

|
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145

146
147
148
149
150

151
152
153

154

155
156
157
158
159
160
161
162
163
164
165
166

# Write the main loop for the child testfixture processes into file
# tf_main.tcl. The parent (this script) interacts with the child processes
# via a two way pipe. The parent writes a script to the stdin of the child
# process, followed by the word "OVER" on a line of its own. The child
# process evaluates the script and writes the results to stdout, followed
# by an "OVER" of its own.
#
set main_loop_written 0
proc write_main_loop {} {
  if {$::main_loop_written} return
  set wrapper ""
  if {[sqlite3 -has-codec] && [info exists ::do_not_use_codec]==0} {
    set wrapper "
      rename sqlite3 sqlite_orig
      proc sqlite3 {args} {[info body sqlite3]}
    "
  }

  set fd [open tf_main.tcl w]
  puts $fd [string map [list %WRAPPER% $wrapper] {

    %WRAPPER%
    set script ""
    while {![eof stdin]} {
      flush stdout
      set line [gets stdin]

      if { $line == "OVER" } {
        set rc [catch {eval $script} result]
        puts [list $rc $result]

        puts OVER

        flush stdout
        set script ""
      } else {
        append script $line
        append script "\n"
      }
    }
  }]
  close $fd
  set main_loop_written 1
}

Changes to test/wal.test.
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
source $testdir/lock_common.tcl
source $testdir/malloc_common.tcl

# Do not use a codec for tests in this file, as the database file is
# manipulated directly using tcl scripts (using the [hexio_write] command).
#
do_not_use_codec

ifcapable !wal {finish_test ; return }

proc reopen_db {} {
  catch { db close }
  file delete -force test.db test.db-wal test.db-wal-summary
  sqlite3_wal db test.db
}







<
<
<
<
<







14
15
16
17
18
19
20





21
22
23
24
25
26
27
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
source $testdir/lock_common.tcl
source $testdir/malloc_common.tcl






ifcapable !wal {finish_test ; return }

proc reopen_db {} {
  catch { db close }
  file delete -force test.db test.db-wal test.db-wal-summary
  sqlite3_wal db test.db
}
Changes to test/wal2.test.
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
source $testdir/lock_common.tcl
ifcapable !wal {finish_test ; return }

# Do not use a codec for tests in this file, as the database file is
# manipulated directly using tcl scripts (using the [hexio_write] command).
#
do_not_use_codec

proc set_tvfs_hdr {file args} {

  # Set $nHdr to the number of bytes in the wal-index header:
  set nHdr 40
  set nInt [expr {$nHdr/4}]

  if {[llength $args]>2} {







<
<
<
<
<







14
15
16
17
18
19
20





21
22
23
24
25
26
27
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
source $testdir/lock_common.tcl
ifcapable !wal {finish_test ; return }






proc set_tvfs_hdr {file args} {

  # Set $nHdr to the number of bytes in the wal-index header:
  set nHdr 40
  set nInt [expr {$nHdr/4}]

  if {[llength $args]>2} {
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
    PRAGMA journal_mode = WAL;
    CREATE TABLE t1(x);
    INSERT INTO t1 VALUES(zeroblob(8188*1020));
    CREATE TABLE t2(y);
  }
  execsql {
    PRAGMA wal_checkpoint;
    SELECT rootpage FROM sqlite_master WHERE tbl_name = 't2';
  }
} {8192}
do_test wal2-8.1.3 {
  execsql {
    PRAGMA cache_size = 10;
    CREATE TABLE t3(z);
    BEGIN;
      INSERT INTO t3 VALUES(randomblob(900));
      INSERT INTO t3 SELECT randomblob(900) FROM t3;







|

|







798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
    PRAGMA journal_mode = WAL;
    CREATE TABLE t1(x);
    INSERT INTO t1 VALUES(zeroblob(8188*1020));
    CREATE TABLE t2(y);
  }
  execsql {
    PRAGMA wal_checkpoint;
    SELECT rootpage>=8192 FROM sqlite_master WHERE tbl_name = 't2';
  }
} {1}
do_test wal2-8.1.3 {
  execsql {
    PRAGMA cache_size = 10;
    CREATE TABLE t3(z);
    BEGIN;
      INSERT INTO t3 VALUES(randomblob(900));
      INSERT INTO t3 SELECT randomblob(900) FROM t3;