/ Check-in [f664f940]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Increase coverage provided by permutation "coverage-wal" on this branch.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | begin-concurrent-wal2
Files: files | file ages | folders
SHA3-256: f664f940a7eb6938b1ee171143a691e2f497aec921f979be63ef844d38053f56
User & Date: dan 2018-12-29 16:34:22
Wiki:begin-concurrent-wal2
Context
2018-12-29
20:47
Merge latest begin-concurrent changes with this branch. check-in: 1625887c user: dan tags: begin-concurrent-wal2
16:34
Increase coverage provided by permutation "coverage-wal" on this branch. check-in: f664f940 user: dan tags: begin-concurrent-wal2
2018-12-27
17:11
Merge latest wal2 changes with this branch. check-in: ea96001e user: dan tags: begin-concurrent-wal2
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/wal.c.

1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
....
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
....
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
....
4045
4046
4047
4048
4049
4050
4051

















4052




4053
4054
4055
4056
4057
4058
4059
4060
  testcase( mxFrame==HASHTABLE_NPAGE_ONE-1 );
  testcase( mxFrame==HASHTABLE_NPAGE_ONE );
  testcase( mxFrame==HASHTABLE_NPAGE_ONE+1 );

  if( mxFrame==0 ) return;

  /* Obtain pointers to the hash-table and page-number array containing 
  ** the entry that corresponds to frame pWal->hdr.mxFrame. It is guaranteed
  ** that the page said hash-table and array reside on is already mapped.
  */
  assert( pWal->nWiData>walFramePage(iExternal) );
  assert( pWal->apWiData[walFramePage(iExternal)] );
  walHashGet(pWal, walFramePage(iExternal), &sLoc);

  /* Zero all hash-table entries that correspond to frame numbers greater
  ** than pWal->hdr.mxFrame.
  */
................................................................................
      ** occurs if some other writer has crashed while committing a 
      ** transaction to this database since the current concurrent transaction
      ** was opened.  */
      rc = SQLITE_BUSY_SNAPSHOT;
    }else if( memcmp(&pWal->hdr, (void*)&head, sizeof(WalIndexHdr))!=0 ){
      int bWal2 = isWalMode2(pWal);
      int iHash;
      int iLastHash = walFramePage(head.mxFrame);
      int nLoop = 1+(bWal2 && walidxGetFile(&head)!=walidxGetFile(&pWal->hdr));
      int iLoop;
      

      assert( nLoop==1 || nLoop==2 );
      for(iLoop=0; iLoop<nLoop && rc==SQLITE_OK; iLoop++){
        u32 iFirst;               /* First (external) wal frame to check */
        u32 iLastHash;            /* Last hash to check this loop */
        u32 mxFrame;              /* Last (external) wal frame to check */

        if( bWal2==0 ){
          assert( iLoop==0 );
          /* Special case for wal mode. If this concurrent transaction was
................................................................................
              if( sLoc.aPgno[i]==1 ){
                /* Check that the schema cookie has not been modified. If
                ** it has not, the commit can proceed. */
                u8 aNew[4];
                u8 *aOld = &((u8*)pPage1->pData)[40];
                int sz;
                i64 iOff;
                int iFrame = sLoc.iZero + i;
                int iWal = 0;
                if( bWal2 ){
                  iWal = walExternalDecode(iFrame, &iFrame);
                }
                sz = pWal->hdr.szPage;
                sz = (sz&0xfe00) + ((sz&0x0001)<<16);
                iOff = walFrameOffset(iFrame, sz) + WAL_FRAME_HDRSIZE + 40;
................................................................................

    assert( isWalMode2(pWal) || iWal==0 );

    /* Restore the clients cache of the wal-index header to the state it
    ** was in before the client began writing to the database. 
    */
    memcpy(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr));

















    assert( walidxGetFile(&pWal->hdr)==iWal );




    iNew = walidxGetMxFrame(&pWal->hdr, walidxGetFile(&pWal->hdr));

    for(iFrame=iNew+1; ALWAYS(rc==SQLITE_OK) && iFrame<=iMax; iFrame++){
      /* This call cannot fail. Unless the page for which the page number
      ** is passed as the second argument is (a) in the cache and 
      ** (b) has an outstanding reference, then xUndo is either a no-op
      ** (if (a) is false) or simply expels the page from the cache (if (b)
      ** is false).







|
<
<







 







<





|







 







|







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|
>
>
>
>
|







1316
1317
1318
1319
1320
1321
1322
1323


1324
1325
1326
1327
1328
1329
1330
....
3864
3865
3866
3867
3868
3869
3870

3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
....
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
....
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
  testcase( mxFrame==HASHTABLE_NPAGE_ONE-1 );
  testcase( mxFrame==HASHTABLE_NPAGE_ONE );
  testcase( mxFrame==HASHTABLE_NPAGE_ONE+1 );

  if( mxFrame==0 ) return;

  /* Obtain pointers to the hash-table and page-number array containing 
  ** the entry that corresponds to frame pWal->hdr.mxFrame.  */


  assert( pWal->nWiData>walFramePage(iExternal) );
  assert( pWal->apWiData[walFramePage(iExternal)] );
  walHashGet(pWal, walFramePage(iExternal), &sLoc);

  /* Zero all hash-table entries that correspond to frame numbers greater
  ** than pWal->hdr.mxFrame.
  */
................................................................................
      ** occurs if some other writer has crashed while committing a 
      ** transaction to this database since the current concurrent transaction
      ** was opened.  */
      rc = SQLITE_BUSY_SNAPSHOT;
    }else if( memcmp(&pWal->hdr, (void*)&head, sizeof(WalIndexHdr))!=0 ){
      int bWal2 = isWalMode2(pWal);
      int iHash;

      int nLoop = 1+(bWal2 && walidxGetFile(&head)!=walidxGetFile(&pWal->hdr));
      int iLoop;
      

      assert( nLoop==1 || nLoop==2 );
      for(iLoop=0; rc==SQLITE_OK && iLoop<nLoop; iLoop++){
        u32 iFirst;               /* First (external) wal frame to check */
        u32 iLastHash;            /* Last hash to check this loop */
        u32 mxFrame;              /* Last (external) wal frame to check */

        if( bWal2==0 ){
          assert( iLoop==0 );
          /* Special case for wal mode. If this concurrent transaction was
................................................................................
              if( sLoc.aPgno[i]==1 ){
                /* Check that the schema cookie has not been modified. If
                ** it has not, the commit can proceed. */
                u8 aNew[4];
                u8 *aOld = &((u8*)pPage1->pData)[40];
                int sz;
                i64 iOff;
                u32 iFrame = sLoc.iZero + i;
                int iWal = 0;
                if( bWal2 ){
                  iWal = walExternalDecode(iFrame, &iFrame);
                }
                sz = pWal->hdr.szPage;
                sz = (sz&0xfe00) + ((sz&0x0001)<<16);
                iOff = walFrameOffset(iFrame, sz) + WAL_FRAME_HDRSIZE + 40;
................................................................................

    assert( isWalMode2(pWal) || iWal==0 );

    /* Restore the clients cache of the wal-index header to the state it
    ** was in before the client began writing to the database. 
    */
    memcpy(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr));
    iNew = walidxGetMxFrame(&pWal->hdr, walidxGetFile(&pWal->hdr));

    /* BEGIN CONCURRENT transactions are different, as the header just
    ** memcpy()d into pWal->hdr may not be the same as the current header 
    ** when the transaction was started. Instead, pWal->hdr now contains
    ** the header written by the most recent successful COMMIT. Because
    ** Wal.writeLock is set, if this is a BEGIN CONCURRENT transaction,
    ** the rollback must be taking place because an error occurred during
    ** a COMMIT.
    **
    ** The code below is still valid. All frames between (iNew+1) and iMax 
    ** must have been written by this transaction before the error occurred.
    ** The exception is in wal2 mode - if the current wal file at the time
    ** of the last COMMIT is not wal file iWal, then the error must have
    ** occurred in WalLockForCommit(), before any pages were written
    ** to the database file. In this case return early.  */
#ifndef SQLITE_OMIT_CONCURRENT
    if( walidxGetFile(&pWal->hdr)!=iWal ){
      assert( isWalMode2(pWal) );
      return SQLITE_OK;
    }
#endif
    assert( walidxGetFile(&pWal->hdr)==iWal );

    for(iFrame=iNew+1; ALWAYS(rc==SQLITE_OK) && iFrame<=iMax; iFrame++){
      /* This call cannot fail. Unless the page for which the page number
      ** is passed as the second argument is (a) in the cache and 
      ** (b) has an outstanding reference, then xUndo is either a no-op
      ** (if (a) is false) or simply expels the page from the cache (if (b)
      ** is false).

Added test/concfault2.test.



































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
# 2018 Dec 28
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#
# This file contains fault injection tests designed to test the concurrent
# transactions feature.
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
source $testdir/malloc_common.tcl
set testprefix concfault2

ifcapable !concurrent {
  finish_test
  return
}

do_execsql_test 1.0 {
  PRAGMA auto_vacuum = 0;
  PRAGMA journal_mode = wal2;
  CREATE TABLE t1(a PRIMARY KEY, b);
  CREATE TABLE t2(a PRIMARY KEY, b);
  INSERT INTO t1 VALUES(randomblob(1000), randomblob(100));
  INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1;
  INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1;
  INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1;
  INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1;
  DELETE FROM t1 WHERE rowid%2;
} {wal2}

do_test 1.1 {
  list [expr [file size test.db-wal]>75000] [file size test.db-shm]
} {1 32768}

faultsim_save_and_close

do_faultsim_test 1 -prep {
  faultsim_restore_and_reopen
  execsql {
    BEGIN CONCURRENT;
      INSERT INTO t2 VALUES(1, 2);
  }
  sqlite3 db2 test.db
  execsql {
    PRAGMA journal_size_limit = 10000;
    INSERT INTO t1 VALUES(randomblob(1000), randomblob(1000));
  } db2
  db2 close
} -body {
  execsql { COMMIT }
} -test {
  faultsim_test_result {0 {}} 
  catchsql { ROLLBACK }
  faultsim_integrity_check
}
finish_test

Changes to test/concurrent2.test.

18
19
20
21
22
23
24




25
26
27
28
29
30
31
source $testdir/wal_common.tcl
set ::testprefix concurrent2

ifcapable !concurrent {
  finish_test
  return
}





do_multiclient_test tn {

  do_test 1.$tn.1 {
    sql1 {
      PRAGMA journal_mode = wal;
      CREATE TABLE t1(x);







>
>
>
>







18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
source $testdir/wal_common.tcl
set ::testprefix concurrent2

ifcapable !concurrent {
  finish_test
  return
}

do_test 0.1 {
  llength [sqlite3_wal_info db main]
} {2}

do_multiclient_test tn {

  do_test 1.$tn.1 {
    sql1 {
      PRAGMA journal_mode = wal;
      CREATE TABLE t1(x);

Changes to test/permutations.test.

441
442
443
444
445
446
447

448
449
450
451
452
453
454
walshared.test walslow.test wal.test
wal2savepoint.test wal2lock.test wal2recover2.test

  wal2concurrent.test
  concurrent.test concurrent2.test concurrent3.test
  concurrent4.test concurrent5.test concurrent6.test
  concurrent7.test


  walvfs.test walfault2.test nockpt.test
  snapshot2.test snapshot3.test snapshot4.test
  snapshot_fault.test snapshot.test snapshot_up.test
  walcrash2.test walcrash3.test walcrash4.test walcrash.test
  wal2fault.test
} 







>







441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
walshared.test walslow.test wal.test
wal2savepoint.test wal2lock.test wal2recover2.test

  wal2concurrent.test
  concurrent.test concurrent2.test concurrent3.test
  concurrent4.test concurrent5.test concurrent6.test
  concurrent7.test
  concfault.test concfault2.test

  walvfs.test walfault2.test nockpt.test
  snapshot2.test snapshot3.test snapshot4.test
  snapshot_fault.test snapshot.test snapshot_up.test
  walcrash2.test walcrash3.test walcrash4.test walcrash.test
  wal2fault.test
} 

Changes to test/wal2recover2.test.

216
217
218
219
220
221
222











































223
224
225
226
227
228
229
      SELECT sum(x) FROM t1;
      SELECT sum(x) FROM t2;
    } db2
  } $res
  db2 close
}












































#-------------------------------------------------------------------------
reset_db
do_execsql_test 1.0 {
  CREATE TABLE t1(a, b, c);
  CREATE INDEX t1a ON t1(a);
  CREATE INDEX t1b ON t1(b);
  CREATE INDEX t1c ON t1(c);







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
      SELECT sum(x) FROM t1;
      SELECT sum(x) FROM t2;
    } db2
  } $res
  db2 close
}


#-------------------------------------------------------------------------
reset_db
do_execsql_test 1.8.1 {
  PRAGMA autovacuum = 0;
  PRAGMA page_size = 4096;
  CREATE TABLE t1(x);
  CREATE TABLE t2(x);
  WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<1500 )
    INSERT INTO t1 SELECT i FROM s;
  WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<1500 )
    INSERT INTO t2 SELECT i FROM s;

  PRAGMA journal_mode = wal2;
  PRAGMA journal_size_limit = 10000;

  WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<1500 )
    INSERT INTO t2 SELECT i FROM s;
} {wal2 10000}

do_test 1.8.2 {
  list [file size test.db-wal] [file size test.db-wal2]
} {24752 0}

do_execsql_test 1.8.3 { PRAGMA user_version = 123 }
do_test 1.8.4 {
  list [file size test.db-wal] [file size test.db-wal2]
} {24752 4152}

do_test 1.8.5 {
  hexio_write test.db-wal2 [expr 56+16] 0400
  fix_wal_cksums test.db-wal2
} {}

do_test 1.8.6 {
  forcecopy test.db test.db2
  forcecopy test.db-wal test.db2-wal
  forcecopy test.db-wal2 test.db2-wal2
  sqlite3 db2 test.db2
  catchsql { SELECT * FROM sqlite_master } db2
} {1 {database disk image is malformed}}
db2 close

#-------------------------------------------------------------------------
reset_db
do_execsql_test 1.0 {
  CREATE TABLE t1(a, b, c);
  CREATE INDEX t1a ON t1(a);
  CREATE INDEX t1b ON t1(b);
  CREATE INDEX t1c ON t1(c);

Changes to test/wal2savepoint.test.

50
51
52
53
54
55
56













57
58
59
60
    ROLLBACK TO abc;
    WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s where i < 10)
    INSERT INTO t1 SELECT random(), random(), random() FROM s;
  COMMIT;
  SELECT count(*) FROM t1;
  PRAGMA integrity_check;
} {210 ok}















finish_test








>
>
>
>
>
>
>
>
>
>
>
>
>




50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
    ROLLBACK TO abc;
    WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s where i < 10)
    INSERT INTO t1 SELECT random(), random(), random() FROM s;
  COMMIT;
  SELECT count(*) FROM t1;
  PRAGMA integrity_check;
} {210 ok}

do_execsql_test 1.4 {
  BEGIN;
    SAVEPOINT abc;
      WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s where i < 100)
      INSERT INTO t1 SELECT random(), random(), random() FROM s;
    ROLLBACK TO abc;
    WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s where i < 10)
    INSERT INTO t1 SELECT random(), random(), random() FROM s;
  COMMIT;
  SELECT count(*) FROM t1;
  PRAGMA integrity_check;
} {220 ok}


finish_test

Changes to test/wal2snapshot.test.

63
64
65
66
67
68
69

70
71









72
73
74
75
76
77
78
79
80
81
82
83
84
    do_test 1.6 {
      execsql BEGIN
      set SNAPSHOT [sqlite3_snapshot_get_blob db main]
      sqlite3_snapshot_open_blob db main $SNAPSHOT
      execsql COMMIT
    } {}
  } else {

    do_test 2.6 {
      execsql BEGIN









      set res [
        list [catch { sqlite3_snapshot_open_blob db main $SNAPSHOT } msg] $msg
      ]
      execsql COMMIT
      set res
    } {1 SQLITE_ERROR}
  }
}


finish_test









>
|

>
>
>
>
>
>
>
>
>













63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
    do_test 1.6 {
      execsql BEGIN
      set SNAPSHOT [sqlite3_snapshot_get_blob db main]
      sqlite3_snapshot_open_blob db main $SNAPSHOT
      execsql COMMIT
    } {}
  } else {

    do_test 2.6.1 {
      execsql BEGIN
      set res [
        list [catch { sqlite3_snapshot_open_blob db main $SNAPSHOT } msg] $msg
      ]
      execsql COMMIT
      set res
    } {1 SQLITE_ERROR}
    do_test 2.6.2 {
      execsql BEGIN
      execsql {SELECT * FROM sqlite_master}
      set res [
        list [catch { sqlite3_snapshot_open_blob db main $SNAPSHOT } msg] $msg
      ]
      execsql COMMIT
      set res
    } {1 SQLITE_ERROR}
  }
}


finish_test