/ Check-in [a56506b9]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Cherrypick a couple of fixes from begin-concurrent-pnu into this branch. The differences between the two branches are now that this one does not have "PRAGMA noop_update" or the mutex-free PRNG.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | begin-concurrent
Files: files | file ages | folders
SHA3-256:a56506b9387a067ef259504d127694ad20223f4b08781d1676ff7f5fdd9443d8
User & Date: dan 2018-12-03 19:29:37
Wiki:begin-concurrent
Context
2018-12-03
20:38
Minor change to wal.c on this branch to make it more similar to trunk. check-in: 6a7af3ea user: dan tags: begin-concurrent
19:29
Cherrypick a couple of fixes from begin-concurrent-pnu into this branch. The differences between the two branches are now that this one does not have "PRAGMA noop_update" or the mutex-free PRNG. check-in: a56506b9 user: dan tags: begin-concurrent
18:15
Bring up to date with version 3.26.0. check-in: f0ddb358 user: drh tags: begin-concurrent
2018-02-20
21:00
Add extra code to log details when corruption is detected in the pointer-map structure maintained by the b-tree layer in begin-concurrent transactions. check-in: 57023371 user: dan tags: begin-concurrent-pnu
2018-01-04
18:36
Fix problem causing free-list corruption when merging free-lists for two concurrent transactions that have both used page X as an in-memory free-list trunk page, where X lies past the end of the initial database images. check-in: dc0fc2aa user: dan tags: begin-concurrent-pnu
2018-01-02
19:57
Fix a spurious SQLITE_CORRUPT error that could occur within a COMMIT of a concurrent transaction. check-in: 50c8952c user: dan tags: begin-concurrent-pnu
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/btree.c.

652
653
654
655
656
657
658




































659
660
661
662
663

664
665
666
667
668
669
670
....
4148
4149
4150
4151
4152
4153
4154
4155



4156
4157
4158
4159
4160
4161
4162
....
4205
4206
4207
4208
4209
4210
4211


4212
4213
4214
4215
4216
4217
4218
....
4232
4233
4234
4235
4236
4237
4238
4239
4240
4241

4242

4243
4244
4245
4246
4247
4248
4249
....
6225
6226
6227
6228
6229
6230
6231
6232
6233
6234
6235
6236
6237
6238
6239
    sqlite3_free(pMap->aRollback);
    sqlite3_free(pMap->aPtr);
    sqlite3_free(pMap->aSvpt);
    sqlite3_free(pMap);
    pBt->pMap = 0;
  }
}




































#else  /* SQLITE_OMIT_CONCURRENT */
# define btreePtrmapAllocate(x) SQLITE_OK
# define btreePtrmapDelete(x) 
# define btreePtrmapBegin(x,y)  SQLITE_OK
# define btreePtrmapEnd(x,y,z) 

#endif /* SQLITE_OMIT_CONCURRENT */

static void releasePage(MemPage *pPage);  /* Forward reference */
static void releasePageOne(MemPage *pPage);      /* Forward reference */
static void releasePageNotNull(MemPage *pPage);  /* Forward reference */

/*
................................................................................

    if( iPg==PENDING_BYTE_PAGE(pBt) ) continue;
    pEntry = &pMap->aPtr[iPg - pMap->iFirst];

    if( pEntry->eType==PTRMAP_FREEPAGE ){
      Pgno dummy;
      rc = allocateBtreePage(pBt, &pFree, &dummy, iPg, BTALLOC_EXACT);
      releasePage(pFree);



      assert( rc!=SQLITE_OK || dummy==iPg );
    }else if( pnCurrent ){
      btreeGetPage(pBt, iPg, &pPg, 0);
      assert( sqlite3PagerIswriteable(pPg->pDbPage) );
      assert( sqlite3PagerPageRefcount(pPg->pDbPage)==1 );
      iNew = ++(*pnCurrent);
      if( iNew==PENDING_BYTE_PAGE(pBt) ) iNew = ++(*pnCurrent);
................................................................................
  if( rc==SQLITE_OK ){
    Pgno nHPage = get4byte(&p1[28]);
    Pgno nFin = nHPage;         /* Size of db after transaction merge */

    if( sqlite3PagerIswriteable(pPage1->pDbPage) ){
      Pgno iHTrunk = get4byte(&p1[32]);
      u32 nHFree = get4byte(&p1[36]);



      /* Attach the head database free list to the end of the current
      ** transactions free-list (if any).  */
      if( iTrunk!=0 ){
        put4byte(&p1[36], nHFree + nFree);
        put4byte(&p1[32], iTrunk);
        while( iTrunk ){
................................................................................
        ** not have shrunk since the transaction was opened. Therefore nHPage
        ** should be set to (pMap->iFirst-1) or greater. */
        rc = SQLITE_CORRUPT_BKPT;
      }else{
        /* The current transaction allocated pages pMap->iFirst through
        ** nPage (inclusive) at the end of the database file. Meanwhile,
        ** other transactions have allocated (iFirst..nHPage). So move
        ** pages (iFirst..MIN(nPage,nHPage)) to (MAX(nPage,nHPage)+1).  */
        Pgno iLast = MIN(nPage, nHPage);    /* Last page to move */
        Pgno nCurrent;                      /* Current size of db */

        nCurrent = MAX(nPage, nHPage);

        rc = btreeRelocateRange(pBt, pMap->iFirst, iLast, &nCurrent);

        /* There are now no collisions with the snapshot at the head of the
        ** database file. So at this point it would be possible to write
        ** the transaction out to disk. Before doing so though, attempt to
        ** relocate some of the new pages to free locations within the body
        ** of the database file (i.e. free-list entries). */
................................................................................
  assert( eMode==BTALLOC_ANY || (nearby>0 && REQUIRE_PTRMAP ) );
  pPage1 = pBt->pPage1;
  mxPage = btreePagecount(pBt);
  /* EVIDENCE-OF: R-05119-02637 The 4-byte big-endian integer at offset 36
  ** stores stores the total number of pages on the freelist. */
  n = get4byte(&pPage1->aData[36]);
  testcase( n==mxPage-1 );
  if( ISCONCURRENT==0 && n>=mxPage ){
    return SQLITE_CORRUPT_BKPT;
  }

  /* Ensure page 1 is writable. This function will either change the number
  ** of pages in the free-list or the size of the database file. Since both
  ** of these operations involve modifying page 1 header fields, page 1
  ** will definitely be written by this transaction. If this is an CONCURRENT







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>





>







 







|
>
>
>







 







>
>







 







|


>

>







 







|







652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
....
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
....
4245
4246
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
4257
4258
4259
4260
....
4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284
4285
4286
4287
4288
4289
4290
4291
4292
4293
....
6269
6270
6271
6272
6273
6274
6275
6276
6277
6278
6279
6280
6281
6282
6283
    sqlite3_free(pMap->aRollback);
    sqlite3_free(pMap->aPtr);
    sqlite3_free(pMap->aSvpt);
    sqlite3_free(pMap);
    pBt->pMap = 0;
  }
}

/*
** Check that the pointer-map does not contain any entries with a parent
** page of 0. Call sqlite3_log() multiple times to output the entire
** data structure if it does.
*/
static void btreePtrmapCheck(BtShared *pBt, Pgno nPage){
  Pgno i;
  int bProblem = 0;
  BtreePtrmap *p = pBt->pMap;

  for(i=p->iFirst; i<=nPage; i++){
    PtrmapEntry *pEntry = &p->aPtr[i-p->iFirst];
    if( pEntry->eType==PTRMAP_OVERFLOW1
     || pEntry->eType==PTRMAP_OVERFLOW2
     || pEntry->eType==PTRMAP_BTREE
    ){
      if( pEntry->parent==0 ){
        bProblem = 1;
        break;
      }
    }
  }

  if( bProblem ){
    for(i=p->iFirst; i<=nPage; i++){
      PtrmapEntry *pEntry = &p->aPtr[i-p->iFirst];
      sqlite3_log(SQLITE_CORRUPT, 
          "btreePtrmapCheck: pgno=%d eType=%d parent=%d", 
          (int)i, (int)pEntry->eType, (int)pEntry->parent
      );
    }
    abort();
  }
}

#else  /* SQLITE_OMIT_CONCURRENT */
# define btreePtrmapAllocate(x) SQLITE_OK
# define btreePtrmapDelete(x) 
# define btreePtrmapBegin(x,y)  SQLITE_OK
# define btreePtrmapEnd(x,y,z) 
# define btreePtrmapCheck(y,z) 
#endif /* SQLITE_OMIT_CONCURRENT */

static void releasePage(MemPage *pPage);  /* Forward reference */
static void releasePageOne(MemPage *pPage);      /* Forward reference */
static void releasePageNotNull(MemPage *pPage);  /* Forward reference */

/*
................................................................................

    if( iPg==PENDING_BYTE_PAGE(pBt) ) continue;
    pEntry = &pMap->aPtr[iPg - pMap->iFirst];

    if( pEntry->eType==PTRMAP_FREEPAGE ){
      Pgno dummy;
      rc = allocateBtreePage(pBt, &pFree, &dummy, iPg, BTALLOC_EXACT);
      if( pFree ){
        assert( sqlite3PagerPageRefcount(pFree->pDbPage)==1 );
        sqlite3PcacheDrop(pFree->pDbPage);
      }
      assert( rc!=SQLITE_OK || dummy==iPg );
    }else if( pnCurrent ){
      btreeGetPage(pBt, iPg, &pPg, 0);
      assert( sqlite3PagerIswriteable(pPg->pDbPage) );
      assert( sqlite3PagerPageRefcount(pPg->pDbPage)==1 );
      iNew = ++(*pnCurrent);
      if( iNew==PENDING_BYTE_PAGE(pBt) ) iNew = ++(*pnCurrent);
................................................................................
  if( rc==SQLITE_OK ){
    Pgno nHPage = get4byte(&p1[28]);
    Pgno nFin = nHPage;         /* Size of db after transaction merge */

    if( sqlite3PagerIswriteable(pPage1->pDbPage) ){
      Pgno iHTrunk = get4byte(&p1[32]);
      u32 nHFree = get4byte(&p1[36]);

      btreePtrmapCheck(pBt, nPage);

      /* Attach the head database free list to the end of the current
      ** transactions free-list (if any).  */
      if( iTrunk!=0 ){
        put4byte(&p1[36], nHFree + nFree);
        put4byte(&p1[32], iTrunk);
        while( iTrunk ){
................................................................................
        ** not have shrunk since the transaction was opened. Therefore nHPage
        ** should be set to (pMap->iFirst-1) or greater. */
        rc = SQLITE_CORRUPT_BKPT;
      }else{
        /* The current transaction allocated pages pMap->iFirst through
        ** nPage (inclusive) at the end of the database file. Meanwhile,
        ** other transactions have allocated (iFirst..nHPage). So move
        ** pages (iFirst..MIN(nPage,nHPage)) to (MAX(nPage,nHPage)+1). */
        Pgno iLast = MIN(nPage, nHPage);    /* Last page to move */
        Pgno nCurrent;                      /* Current size of db */

        nCurrent = MAX(nPage, nHPage);
        pBt->nPage = nCurrent;
        rc = btreeRelocateRange(pBt, pMap->iFirst, iLast, &nCurrent);

        /* There are now no collisions with the snapshot at the head of the
        ** database file. So at this point it would be possible to write
        ** the transaction out to disk. Before doing so though, attempt to
        ** relocate some of the new pages to free locations within the body
        ** of the database file (i.e. free-list entries). */
................................................................................
  assert( eMode==BTALLOC_ANY || (nearby>0 && REQUIRE_PTRMAP ) );
  pPage1 = pBt->pPage1;
  mxPage = btreePagecount(pBt);
  /* EVIDENCE-OF: R-05119-02637 The 4-byte big-endian integer at offset 36
  ** stores stores the total number of pages on the freelist. */
  n = get4byte(&pPage1->aData[36]);
  testcase( n==mxPage-1 );
  if( n>=mxPage ){
    return SQLITE_CORRUPT_BKPT;
  }

  /* Ensure page 1 is writable. This function will either change the number
  ** of pages in the free-list or the size of the database file. Since both
  ** of these operations involve modifying page 1 header fields, page 1
  ** will definitely be written by this transaction. If this is an CONCURRENT

Added test/concurrent6.test.

























































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
# 2017 May 26
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
source $testdir/lock_common.tcl
source $testdir/wal_common.tcl
set ::testprefix concurrent6

ifcapable !concurrent {
  finish_test
  return
}

sqlite3 db2 test.db

do_execsql_test 1.0 {
  PRAGMA page_size = 1024;
  PRAGMA journal_mode = wal;
  CREATE TABLE t1(x);
  CREATE TABLE t2(x);
  CREATE TABLE t3(x);
  CREATE TABLE t4(x);

  INSERT INTO t1 VALUES(zeroblob(1500));
} {wal}

do_execsql_test -db db2 1.1 {
  BEGIN CONCURRENT;
    INSERT INTO t3 VALUES(zeroblob(4000));
    DELETE FROM t1;
}

do_execsql_test 1.2 {
  WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100) 
  INSERT INTO t2 SELECT zeroblob(1000) FROM s;

  WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100) 
  INSERT INTO t4  SELECT zeroblob(1000) FROM s;

  DELETE FROM t4;
}

do_execsql_test -db db2 1.3 {
  COMMIT;
}


finish_test

Added test/concurrent7.test.









































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
# 2018 Jan 5
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#


set testdir [file dirname $argv0]
source $testdir/tester.tcl
set ::testprefix concurrent7

sqlite3 db2 test.db

do_execsql_test 1 {
  PRAGMA journal_mode = wal;
  CREATE TABLE t1(x);
  CREATE TABLE t2(x);
} {wal}

do_execsql_test -db db2 2 {
  SELECT * FROM t1;
}

do_execsql_test 3 {
  BEGIN CONCURRENT;
    INSERT INTO t1 VALUES(randomblob(1500));
    INSERT INTO t1 VALUES(randomblob(1500));
    DELETE FROM t1 WHERE rowid = 1;
}

do_execsql_test -db db2 4 {
  INSERT INTO t2 VALUES(randomblob(1500));
  INSERT INTO t2 VALUES(randomblob(1500));
  INSERT INTO t2 VALUES(randomblob(1500));
  INSERT INTO t2 VALUES(randomblob(1500));
  DELETE FROM t2 WHERE rowid IN (1, 2);
}

do_execsql_test 5 {
  COMMIT;
  PRAGMA integrity_check;
} {ok}

finish_test