/ Check-in [06967916]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:When committing an UNLOCKED transaction, try to move pages allocated at the end of the file to free slots within the file (like an incremental-vacuum operation does).
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | begin-concurrent
Files: files | file ages | folders
SHA1: 069679162d8d50e9731831e658aa58f280dbb3e7
User & Date: dan 2015-08-21 18:55:22
Wiki:begin-concurrent
Context
2015-08-21
20:11
Fix many minor issues in the unlocked transaction code. check-in: 53aaeea6 user: dan tags: begin-concurrent
18:55
When committing an UNLOCKED transaction, try to move pages allocated at the end of the file to free slots within the file (like an incremental-vacuum operation does). check-in: 06967916 user: dan tags: begin-concurrent
17:57
Fix a problem with UNLOCKED transactions that free pages allocated within the same transaction. check-in: 227bb8a1 user: dan tags: begin-concurrent
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/btree.c.

3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
....
3887
3888
3889
3890
3891
3892
3893
3894


3895
3896
3897

3898
3899
3900
3901
3902
3903
3904
3905
3906

3907
3908
3909
3910
3911
3912
3913
3914
3915
....
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
























3929
3930
3931
3932
3933
3934
3935
....
5780
5781
5782
5783
5784
5785
5786
5787
5788
5789
5790
5791
5792
5793
5794
5795
5796
....
5827
5828
5829
5830
5831
5832
5833
5834
5835
5836
5837
5838
5839
5840
5841
5842
5843
5844
5845
5846

  /* If page 1 of the database is not writable, then no pages were allocated
  ** or freed by this transaction. In this case no special handling is 
  ** required. Otherwise, if page 1 is dirty, proceed.  */
  BtreePtrmap *pMap = pBt->pMap;
  Pgno iTrunk = get4byte(&p1[32]);
  Pgno nPage = btreePagecount(pBt);
  Pgno nOrig = pMap->iFirst-1;
  u32 nFree = get4byte(&p1[36]);

  assert( sqlite3PagerIsUnlocked(pPager) );
  assert( pBt->pMap );
  rc = sqlite3PagerUpgradeSnapshot(pPager, pPage1->pDbPage);
  assert( p1==pPage1->aData );

................................................................................
          if( iTrunk==0 ){
            put4byte((u8*)pTrunk->pData, iHTrunk);
          }
          sqlite3PagerUnref(pTrunk);
        };
      }

      if( nHPage<nOrig ){


        /* An unlocked transaction may not be executed on an auto-vacuum 
        ** database. Therefore the db should not have shrunk since the
        ** transaction was opened.  */

        rc = SQLITE_CORRUPT_BKPT;
      }else{
        /* The current transaction allocated pages pMap->iFirst through
        ** nPage (inclusive) at the end of the database file. Meanwhile,
        ** other transactions have allocated (iFirst..nHPage). So move
        ** pages (iFirst..MIN(nPage,nHPage)) to (MAX(nPage,nHPage)+1).
        */
        Pgno iLast = MIN(nPage, nHPage);    /* Last page to move */
        Pgno iPg;


        nFinal = MAX(nPage, nHPage);
        for(iPg=pMap->iFirst; iPg<=iLast && rc==SQLITE_OK; iPg++){
          MemPage *pPg = 0;
          Pgno iNew;              /* New page number for pPg */
          PtrmapEntry *pEntry;    /* Pointer map entry for page iPg */

          pEntry = &pMap->aPtr[iPg - pMap->iFirst];
          if( pEntry->eType==PTRMAP_FREEPAGE ){
................................................................................
            rc = allocateBtreePage(pBt, &pFree, &dummy, iPg, BTALLOC_EXACT);
            releasePage(pFree);
            assert( rc!=SQLITE_OK || dummy==iPg );
          }else{
            btreeGetPage(pBt, iPg, &pPg, 0);
            assert( sqlite3PagerIswriteable(pPg->pDbPage) );
            assert( sqlite3PagerPageRefcount(pPg->pDbPage)==1 );
            iNew = ++nFinal;
            rc = relocatePage(pBt, pPg, pEntry->eType, pEntry->parent, iNew, 1);
            releasePageNotNull(pPg);
          }
























        }
        put4byte(&p1[28], nFinal);
      }
    }
    sqlite3PagerSetDbsize(pPager, nFinal);
  }

................................................................................
  u32 n;     /* Number of pages on the freelist */
  u32 k;     /* Number of leaves on the trunk of the freelist */
  MemPage *pTrunk = 0;
  MemPage *pPrevTrunk = 0;
  Pgno mxPage;     /* Total size of the database file */

  assert( sqlite3_mutex_held(pBt->mutex) );
  assert( eMode==BTALLOC_ANY || (nearby>0 && ISAUTOVACUUM) 
      || (eMode==BTALLOC_EXACT && sqlite3PagerIsUnlocked(pBt->pPager))
  );
  pPage1 = pBt->pPage1;
  mxPage = btreePagecount(pBt);
  /* EVIDENCE-OF: R-05119-02637 The 4-byte big-endian integer at offset 36
  ** stores stores the total number of pages on the freelist. */
  n = get4byte(&pPage1->aData[36]);
  testcase( n==mxPage-1 );
  if( sqlite3PagerIsUnlocked(pBt->pPager)==0 && n>=mxPage ){
................................................................................
          if( eType==PTRMAP_FREEPAGE ){
            searchList = 1;
          }
        }
      }else{
        searchList = 1;
      }
    }
#ifndef SQLITE_OMIT_AUTOVACUUM
    else if( eMode==BTALLOC_LE ){
      searchList = 1;
    }
#endif

    /* Decrement the free-list count by 1. Set iTrunk to the index of the
    ** first free-list trunk page. iPrevTrunk is initially 1.
    */
    put4byte(&pPage1->aData[36], n-1);

    /* The code within this loop is run only once if the 'searchList' variable







<







 







|
>
>
|
|
<
>









>

|







 







|



>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







|
<
<







 







<
<
|


<







3856
3857
3858
3859
3860
3861
3862

3863
3864
3865
3866
3867
3868
3869
....
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897

3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
....
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
....
5806
5807
5808
5809
5810
5811
5812
5813


5814
5815
5816
5817
5818
5819
5820
....
5851
5852
5853
5854
5855
5856
5857


5858
5859
5860

5861
5862
5863
5864
5865
5866
5867

  /* If page 1 of the database is not writable, then no pages were allocated
  ** or freed by this transaction. In this case no special handling is 
  ** required. Otherwise, if page 1 is dirty, proceed.  */
  BtreePtrmap *pMap = pBt->pMap;
  Pgno iTrunk = get4byte(&p1[32]);
  Pgno nPage = btreePagecount(pBt);

  u32 nFree = get4byte(&p1[36]);

  assert( sqlite3PagerIsUnlocked(pPager) );
  assert( pBt->pMap );
  rc = sqlite3PagerUpgradeSnapshot(pPager, pPage1->pDbPage);
  assert( p1==pPage1->aData );

................................................................................
          if( iTrunk==0 ){
            put4byte((u8*)pTrunk->pData, iHTrunk);
          }
          sqlite3PagerUnref(pTrunk);
        };
      }

      if( nHPage<(pMap->iFirst-1) ){
        /* The database consisted of (pMap->iFirst-1) pages when the current
        ** unlocked transaction was opened. And an unlocked transaction may
        ** not be executed on an auto-vacuum database - so the db should 
        ** not have shrunk since the transaction was opened. Therefore nHPage

        ** should be set to (pMap->iFirst-1) or greater. */
        rc = SQLITE_CORRUPT_BKPT;
      }else{
        /* The current transaction allocated pages pMap->iFirst through
        ** nPage (inclusive) at the end of the database file. Meanwhile,
        ** other transactions have allocated (iFirst..nHPage). So move
        ** pages (iFirst..MIN(nPage,nHPage)) to (MAX(nPage,nHPage)+1).
        */
        Pgno iLast = MIN(nPage, nHPage);    /* Last page to move */
        Pgno iPg;
        Pgno nCurrent;                      /* Current size of db */

        nCurrent = MAX(nPage, nHPage);
        for(iPg=pMap->iFirst; iPg<=iLast && rc==SQLITE_OK; iPg++){
          MemPage *pPg = 0;
          Pgno iNew;              /* New page number for pPg */
          PtrmapEntry *pEntry;    /* Pointer map entry for page iPg */

          pEntry = &pMap->aPtr[iPg - pMap->iFirst];
          if( pEntry->eType==PTRMAP_FREEPAGE ){
................................................................................
            rc = allocateBtreePage(pBt, &pFree, &dummy, iPg, BTALLOC_EXACT);
            releasePage(pFree);
            assert( rc!=SQLITE_OK || dummy==iPg );
          }else{
            btreeGetPage(pBt, iPg, &pPg, 0);
            assert( sqlite3PagerIswriteable(pPg->pDbPage) );
            assert( sqlite3PagerPageRefcount(pPg->pDbPage)==1 );
            iNew = ++nCurrent;
            rc = relocatePage(pBt, pPg, pEntry->eType, pEntry->parent, iNew, 1);
            releasePageNotNull(pPg);
          }
        }
        sqlite3PagerSetDbsize(pPager, nCurrent);

        nFree = get4byte(&p1[36]);
        nFinal = MAX(nCurrent-nFree, nHPage);

        for(iPg=nFinal+1; rc==SQLITE_OK && iPg<nCurrent; iPg++){
          Pgno iNew;              /* New page number for pPg */
          MemPage *pFree;
          PtrmapEntry *pEntry;    /* Pointer map entry for page iPg */

          pEntry = &pMap->aPtr[iPg - pMap->iFirst];
          if( pEntry->eType==PTRMAP_FREEPAGE ){
            rc = allocateBtreePage(pBt, &pFree, &iNew, iPg, BTALLOC_EXACT);
            releasePage(pFree);
          }else{
            rc = allocateBtreePage(pBt, &pFree, &iNew, nFinal, BTALLOC_LE);
            releasePage(pFree);
            if( rc==SQLITE_OK ){
              MemPage *pPg = 0;
              btreeGetPage(pBt, iPg, &pPg, 0);
              rc = relocatePage(pBt, pPg, pEntry->eType, pEntry->parent,iNew,1);
            }
          }
        }
        put4byte(&p1[28], nFinal);
      }
    }
    sqlite3PagerSetDbsize(pPager, nFinal);
  }

................................................................................
  u32 n;     /* Number of pages on the freelist */
  u32 k;     /* Number of leaves on the trunk of the freelist */
  MemPage *pTrunk = 0;
  MemPage *pPrevTrunk = 0;
  Pgno mxPage;     /* Total size of the database file */

  assert( sqlite3_mutex_held(pBt->mutex) );
  assert( eMode==BTALLOC_ANY || (nearby>0 && REQUIRE_PTRMAP ) );


  pPage1 = pBt->pPage1;
  mxPage = btreePagecount(pBt);
  /* EVIDENCE-OF: R-05119-02637 The 4-byte big-endian integer at offset 36
  ** stores stores the total number of pages on the freelist. */
  n = get4byte(&pPage1->aData[36]);
  testcase( n==mxPage-1 );
  if( sqlite3PagerIsUnlocked(pBt->pPager)==0 && n>=mxPage ){
................................................................................
          if( eType==PTRMAP_FREEPAGE ){
            searchList = 1;
          }
        }
      }else{
        searchList = 1;
      }


    }else if( eMode==BTALLOC_LE ){
      searchList = 1;
    }


    /* Decrement the free-list count by 1. Set iTrunk to the index of the
    ** first free-list trunk page. iPrevTrunk is initially 1.
    */
    put4byte(&pPage1->aData[36], n-1);

    /* The code within this loop is run only once if the 'searchList' variable

Changes to test/unlocked2.test.

21
22
23
24
25
26
27

28
29
30
31
32
33
34
..
36
37
38
39
40
41
42

43
44
45
46
47
48
49
50
51
52
53
54
55

56
57
58
59
60
61
62
63
64
65
66
67

68
69
70
71
72
73
74
  do_test 1.$tn.1 {
    sql1 {
      PRAGMA journal_mode = wal;
      CREATE TABLE t1(x);
      CREATE TABLE t2(y);
    }
  } {wal}


  # Test that an UNLOCKED transaction that allocates/frees no pages does
  # not conflict with a transaction that does allocate pages.
  do_test 1.$tn.2  {
    sql1 { 
      BEGIN UNLOCKED;
        INSERT INTO t1 VALUES(4);
................................................................................
    sql2 {
      INSERT INTO t2 VALUES(randomblob(1500));
    }
    sql1 {
      COMMIT;
    }
  } {}

  
  # But that an UNLOCKED transaction does conflict with a transaction
  # that modifies the db schema.
  do_test 1.$tn.3  {
    sql1 {
      BEGIN UNLOCKED;
        INSERT INTO t1 VALUES(5);
    }
    sql2 {
      CREATE TABLE t3(z);
    }
    list [catch { sql1 COMMIT } msg] $msg
  } {1 {database is locked}}

  
  # Test that an UNLOCKED transaction that allocates at least one page 
  # does not conflict with a transaction that allocates no pages.
  do_test 1.$tn.4  {
    sql1 { 
      ROLLBACK;
      BEGIN UNLOCKED;
        INSERT INTO t1 VALUES(randomblob(1500));
    }
    sql2 {
      INSERT INTO t2 VALUES(8);
    }

    sql1 {
      COMMIT;
    }
  } {}

  do_test 1.$tn.5 { sql3 { PRAGMA integrity_check } } {ok}
}







>







 







>













>












>







21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
..
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
  do_test 1.$tn.1 {
    sql1 {
      PRAGMA journal_mode = wal;
      CREATE TABLE t1(x);
      CREATE TABLE t2(y);
    }
  } {wal}
  do_test 1.$tn.5 { sql3 { PRAGMA integrity_check } } {ok}

  # Test that an UNLOCKED transaction that allocates/frees no pages does
  # not conflict with a transaction that does allocate pages.
  do_test 1.$tn.2  {
    sql1 { 
      BEGIN UNLOCKED;
        INSERT INTO t1 VALUES(4);
................................................................................
    sql2 {
      INSERT INTO t2 VALUES(randomblob(1500));
    }
    sql1 {
      COMMIT;
    }
  } {}
  do_test 1.$tn.5 { sql3 { PRAGMA integrity_check } } {ok}
  
  # But that an UNLOCKED transaction does conflict with a transaction
  # that modifies the db schema.
  do_test 1.$tn.3  {
    sql1 {
      BEGIN UNLOCKED;
        INSERT INTO t1 VALUES(5);
    }
    sql2 {
      CREATE TABLE t3(z);
    }
    list [catch { sql1 COMMIT } msg] $msg
  } {1 {database is locked}}
  do_test 1.$tn.5 { sql3 { PRAGMA integrity_check } } {ok}
  
  # Test that an UNLOCKED transaction that allocates at least one page 
  # does not conflict with a transaction that allocates no pages.
  do_test 1.$tn.4  {
    sql1 { 
      ROLLBACK;
      BEGIN UNLOCKED;
        INSERT INTO t1 VALUES(randomblob(1500));
    }
    sql2 {
      INSERT INTO t2 VALUES(8);
    }
    breakpoint
    sql1 {
      COMMIT;
    }
  } {}

  do_test 1.$tn.5 { sql3 { PRAGMA integrity_check } } {ok}
}