/ Check-in [0af764a0]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:For filesystem databases, do not store a list of pages in the statement journal in main memory. (CVS 3825)
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1:0af764a02695281b0a7d70ef4e7f8229fd3d6a30
User & Date: danielk1977 2007-04-07 15:03:17
Context
2007-04-07
17:44
Make sure autovacuum is turned off for malloc5.test. (CVS 3826) check-in: 105a7efb user: drh tags: trunk
15:03
For filesystem databases, do not store a list of pages in the statement journal in main memory. (CVS 3825) check-in: 0af764a0 user: danielk1977 tags: trunk
2007-04-06
21:42
Changes to test scripts to support alternative configurations. (CVS 3824) check-in: 3471a226 user: drh tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/btree.c.

5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
....
5526
5527
5528
5529
5530
5531
5532






5533
5534
5535
5536


5537
5538
5539
5540
5541
5542
5543
....
5548
5549
5550
5551
5552
5553
5554


5555
5556
5557
5558
5559
5560
5561
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
** $Id: btree.c,v 1.352 2007/04/06 15:02:14 drh Exp $
**
** This file implements a external (disk-based) database using BTrees.
** For a detailed discussion of BTrees, refer to
**
**     Donald E. Knuth, THE ART OF COMPUTER PROGRAMMING, Volume 3:
**     "Sorting And Searching", pages 473-480. Addison-Wesley
**     Publishing Company, Reading, Massachusetts.
................................................................................
    */
    rc = allocateBtreePage(pBt, &pPageMove, &pgnoMove, pgnoRoot, 1);
    if( rc!=SQLITE_OK ){
      return rc;
    }

    if( pgnoMove!=pgnoRoot ){






      u8 eType;
      Pgno iPtrPage;

      releasePage(pPageMove);


      rc = getPage(pBt, pgnoRoot, &pRoot, 0);
      if( rc!=SQLITE_OK ){
        return rc;
      }
      rc = ptrmapGet(pBt, pgnoRoot, &eType, &iPtrPage);
      if( rc!=SQLITE_OK || eType==PTRMAP_ROOTPAGE || eType==PTRMAP_FREEPAGE ){
        releasePage(pRoot);
................................................................................
      rc = sqlite3PagerWrite(pRoot->pDbPage);
      if( rc!=SQLITE_OK ){
        releasePage(pRoot);
        return rc;
      }
      rc = relocatePage(pBt, pRoot, eType, iPtrPage, pgnoMove);
      releasePage(pRoot);


      if( rc!=SQLITE_OK ){
        return rc;
      }
      rc = getPage(pBt, pgnoRoot, &pRoot, 0);
      if( rc!=SQLITE_OK ){
        return rc;
      }







|







 







>
>
>
>
>
>




>
>







 







>
>







5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
....
5526
5527
5528
5529
5530
5531
5532
5533
5534
5535
5536
5537
5538
5539
5540
5541
5542
5543
5544
5545
5546
5547
5548
5549
5550
5551
....
5556
5557
5558
5559
5560
5561
5562
5563
5564
5565
5566
5567
5568
5569
5570
5571
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
** $Id: btree.c,v 1.353 2007/04/07 15:03:17 danielk1977 Exp $
**
** This file implements a external (disk-based) database using BTrees.
** For a detailed discussion of BTrees, refer to
**
**     Donald E. Knuth, THE ART OF COMPUTER PROGRAMMING, Volume 3:
**     "Sorting And Searching", pages 473-480. Addison-Wesley
**     Publishing Company, Reading, Massachusetts.
................................................................................
    */
    rc = allocateBtreePage(pBt, &pPageMove, &pgnoMove, pgnoRoot, 1);
    if( rc!=SQLITE_OK ){
      return rc;
    }

    if( pgnoMove!=pgnoRoot ){
      /* pgnoRoot is the page that will be used for the root-page of
      ** the new table (assuming an error did not occur). But we were
      ** allocated pgnoMove. If required (i.e. if it was not allocated
      ** by extending the file), the current page at position pgnoMove
      ** is already journaled.
      */
      u8 eType;
      Pgno iPtrPage;

      releasePage(pPageMove);

      /* Move the page currently at pgnoRoot to pgnoMove. */
      rc = getPage(pBt, pgnoRoot, &pRoot, 0);
      if( rc!=SQLITE_OK ){
        return rc;
      }
      rc = ptrmapGet(pBt, pgnoRoot, &eType, &iPtrPage);
      if( rc!=SQLITE_OK || eType==PTRMAP_ROOTPAGE || eType==PTRMAP_FREEPAGE ){
        releasePage(pRoot);
................................................................................
      rc = sqlite3PagerWrite(pRoot->pDbPage);
      if( rc!=SQLITE_OK ){
        releasePage(pRoot);
        return rc;
      }
      rc = relocatePage(pBt, pRoot, eType, iPtrPage, pgnoMove);
      releasePage(pRoot);

      /* Obtain the page at pgnoRoot */
      if( rc!=SQLITE_OK ){
        return rc;
      }
      rc = getPage(pBt, pgnoRoot, &pRoot, 0);
      if( rc!=SQLITE_OK ){
        return rc;
      }

Changes to src/pager.c.

14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
...
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
...
185
186
187
188
189
190
191


192
193
194
195
196
197
198
...
388
389
390
391
392
393
394















395
396
397
398
399
400
401
...
810
811
812
813
814
815
816


817
818
819
820
821

822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
....
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
....
3004
3005
3006
3007
3008
3009
3010

3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
....
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
....
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
....
3464
3465
3466
3467
3468
3469
3470


3471

3472
3473
3474
3475
3476
3477
3478
3479
3480

3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
....
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688


3689

3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
....
3836
3837
3838
3839
3840
3841
3842
3843

3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
....
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
....
4047
4048
4049
4050
4051
4052
4053
4054
4055

4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
....
4076
4077
4078
4079
4080
4081
4082

4083
4084
4085
4086
4087
4088
4089
4090
4091
** The pager is used to access a database disk file.  It implements
** atomic commit and rollback through the use of a journal file that
** is separate from the database file.  The pager also implements file
** locking to prevent two processes from writing the same database
** file simultaneously, or one process from reading the database while
** another is writing.
**
** @(#) $Id: pager.c,v 1.324 2007/04/06 18:23:18 drh Exp $
*/
#ifndef SQLITE_OMIT_DISKIO
#include "sqliteInt.h"
#include "os.h"
#include "pager.h"
#include <assert.h>
#include <string.h>
................................................................................
typedef struct PgHdr PgHdr;
struct PgHdr {
  Pager *pPager;                 /* The pager to which this page belongs */
  Pgno pgno;                     /* The page number for this page */
  PgHdr *pNextHash, *pPrevHash;  /* Hash collision chain for PgHdr.pgno */
  PgHdr *pNextFree, *pPrevFree;  /* Freelist of pages where nRef==0 */
  PgHdr *pNextAll;               /* A list of all pages */
  PgHdr *pNextStmt, *pPrevStmt;  /* List of pages in the statement journal */
  u8 inJournal;                  /* TRUE if has been written to journal */
  u8 inStmt;                     /* TRUE if in the statement subjournal */
  u8 dirty;                      /* TRUE if we need to write back changes */
  u8 needSync;                   /* Sync journal before writing this page */
  u8 alwaysRollback;             /* Disable DontRollback() for this page */
  short int nRef;                /* Number of users of this page */
  PgHdr *pDirty, *pPrevDirty;    /* Dirty pages */
  u32 notUsed;                   /* Buffer space */
#ifdef SQLITE_CHECK_PAGES
................................................................................
** even for disk-based databases.  Splitting it out saves 8 bytes.  This
** is only a savings of 0.8% but those percentages add up.
*/
typedef struct PgHistory PgHistory;
struct PgHistory {
  u8 *pOrig;     /* Original page text.  Restore to this on a full rollback */
  u8 *pStmt;     /* Text as it was at the beginning of the current statement */


};

/*
** A macro used for invoking the codec if there is one
*/
#ifdef SQLITE_HAS_CODEC
# define CODEC1(P,D,N,X) if( P->xCodec!=0 ){ P->xCodec(P->pCodecArg,D,N,X); }
................................................................................
    cnt++;   /* Something to set a breakpoint on */
  }
# define REFINFO(X)  pager_refinfo(X)
#else
# define REFINFO(X)
#endif

















/*
** Change the size of the pager hash table to N.  N must be a power
** of two.
*/
static void pager_resize_hash_table(Pager *pPager, int N){
  PgHdr **aHash, *pPg;
................................................................................
** The Pager keeps a separate list of pages that are currently in
** the statement journal.  This helps the sqlite3PagerStmtCommit()
** routine run MUCH faster for the common case where there are many
** pages in memory but only a few are in the statement journal.
*/
static void page_add_to_stmt_list(PgHdr *pPg){
  Pager *pPager = pPg->pPager;


  if( pPg->inStmt ) return;
  assert( pPg->pPrevStmt==0 && pPg->pNextStmt==0 );
  pPg->pPrevStmt = 0;
  if( pPager->pStmt ){
    pPager->pStmt->pPrevStmt = pPg;

  }
  pPg->pNextStmt = pPager->pStmt;
  pPager->pStmt = pPg;
  pPg->inStmt = 1;
}
static void page_remove_from_stmt_list(PgHdr *pPg){
  if( !pPg->inStmt ) return;
  if( pPg->pPrevStmt ){
    assert( pPg->pPrevStmt->pNextStmt==pPg );
    pPg->pPrevStmt->pNextStmt = pPg->pNextStmt;
  }else{
    assert( pPg->pPager->pStmt==pPg );
    pPg->pPager->pStmt = pPg->pNextStmt;
  }
  if( pPg->pNextStmt ){
    assert( pPg->pNextStmt->pPrevStmt==pPg );
    pPg->pNextStmt->pPrevStmt = pPg->pPrevStmt;
  }
  pPg->pNextStmt = 0;
  pPg->pPrevStmt = 0;
  pPg->inStmt = 0;
}

/*
** Find a page in the hash table given its page number.  Return
** a pointer to the page or NULL if not found.
*/
static PgHdr *pager_lookup(Pager *pPager, Pgno pgno){
................................................................................
        ** Remove it from this list before freeing.
        **
        ** Todo: Check the Pager.pStmt list to make sure this is Ok. It 
        ** probably is though.
        */
        PgHdr *pTmp;
        assert( pPg );
        page_remove_from_stmt_list(pPg);
        if( pPg==pPager->pAll ){
           pPager->pAll = pPg->pNextAll;
        }else{
          for( pTmp=pPager->pAll; pTmp->pNextAll!=pPg; pTmp=pTmp->pNextAll ){}
          pTmp->pNextAll = pPg->pNextAll;
        }
        nReleased += sqliteAllocSize(pPg);
................................................................................
    TEST_INCR(pPager->nMiss);
    rc = pagerAllocatePage(pPager, &pPg);
    if( rc!=SQLITE_OK ){
      return rc;
    }

    pPg->pgno = pgno;

    if( pPager->aInJournal && (int)pgno<=pPager->origDbSize ){
      sqlite3CheckMemory(pPager->aInJournal, pgno/8);
      assert( pPager->journalOpen );
      pPg->inJournal = (pPager->aInJournal[pgno/8] & (1<<(pgno&7)))!=0;
      pPg->needSync = 0;
    }else{
      pPg->inJournal = 0;
      pPg->needSync = 0;
    }
    if( pPager->aInStmt && (int)pgno<=pPager->stmtSize
             && (pPager->aInStmt[pgno/8] & (1<<(pgno&7)))!=0 ){
      page_add_to_stmt_list(pPg);
    }else{
      page_remove_from_stmt_list(pPg);
    }
    makeClean(pPg);
    pPg->nRef = 1;
    REFINFO(pPg);

    pPager->nRef++;
    if( pPager->nExtra>0 ){
      memset(PGHDR_TO_EXTRA(pPg, pPager), 0, pPager->nExtra);
................................................................................

  CHECK_PAGE(pPg);

  /* Mark the page as dirty.  If the page has already been written
  ** to the journal then we can return right away.
  */
  makeDirty(pPg);
  if( pPg->inJournal && (pPg->inStmt || pPager->stmtInUse==0) ){
    pPager->dirtyCache = 1;
  }else{

    /* If we get this far, it means that the page needs to be
    ** written to the transaction journal or the ckeckpoint journal
    ** or both.
    **
................................................................................

          pPager->nRec++;
          assert( pPager->aInJournal!=0 );
          pPager->aInJournal[pPg->pgno/8] |= 1<<(pPg->pgno&7);
          pPg->needSync = !pPager->noSync;
          if( pPager->stmtInUse ){
            pPager->aInStmt[pPg->pgno/8] |= 1<<(pPg->pgno&7);
            page_add_to_stmt_list(pPg);
          }
        }
      }else{
        pPg->needSync = !pPager->journalStarted && !pPager->noSync;
        PAGERTRACE4("APPEND %d page %d needSync=%d\n",
                PAGERID(pPager), pPg->pgno, pPg->needSync);
      }
................................................................................
    }
  
    /* If the statement journal is open and the page is not in it,
    ** then write the current page to the statement journal.  Note that
    ** the statement journal format differs from the standard journal format
    ** in that it omits the checksums and the header.
    */


    if( pPager->stmtInUse && !pPg->inStmt && (int)pPg->pgno<=pPager->stmtSize ){

      assert( pPg->inJournal || (int)pPg->pgno>pPager->origDbSize );
      if( MEMDB ){
        PgHistory *pHist = PGHDR_TO_HIST(pPg, pPager);
        assert( pHist->pStmt==0 );
        pHist->pStmt = sqliteMallocRaw( pPager->pageSize );
        if( pHist->pStmt ){
          memcpy(pHist->pStmt, PGHDR_TO_DATA(pPg), pPager->pageSize);
        }
        PAGERTRACE3("STMT-JOURNAL %d page %d\n", PAGERID(pPager), pPg->pgno);

      }else{
        char *pData2 = CODEC2(pPager, pData, pPg->pgno, 7)-4;
        put32bits(pData2, pPg->pgno);
        rc = sqlite3OsWrite(pPager->stfd, pData2, pPager->pageSize+4);
        PAGERTRACE3("STMT-JOURNAL %d page %d\n", PAGERID(pPager), pPg->pgno);
        if( rc!=SQLITE_OK ){
          return rc;
        }
        pPager->stmtNRec++;
        assert( pPager->aInStmt!=0 );
        pPager->aInStmt[pPg->pgno/8] |= 1<<(pPg->pgno&7);
      }
      page_add_to_stmt_list(pPg);
    }
  }

  /* Update the database size and return.
  */
  assert( pPager->state>=PAGER_SHARED );
  if( pPager->dbSize<(int)pPg->pgno ){
................................................................................
  if( pPg->alwaysRollback || pPager->alwaysRollback || MEMDB ) return;
  if( !pPg->inJournal && (int)pPg->pgno <= pPager->origDbSize ){
    assert( pPager->aInJournal!=0 );
    pPager->aInJournal[pPg->pgno/8] |= 1<<(pPg->pgno&7);
    pPg->inJournal = 1;
    if( pPager->stmtInUse ){
      pPager->aInStmt[pPg->pgno/8] |= 1<<(pPg->pgno&7);
      page_add_to_stmt_list(pPg);
    }
    PAGERTRACE3("DONT_ROLLBACK page %d of %d\n", pPg->pgno, PAGERID(pPager));
    IOTRACE(("GARBAGE %p %d\n", pPager, pPg->pgno))
  }


  if( pPager->stmtInUse && !pPg->inStmt && (int)pPg->pgno<=pPager->stmtSize ){

    assert( pPg->inJournal || (int)pPg->pgno>pPager->origDbSize );
    assert( pPager->aInStmt!=0 );
    pPager->aInStmt[pPg->pgno/8] |= 1<<(pPg->pgno&7);
    page_add_to_stmt_list(pPg);
  }
}


/*
** This routine is called to increment the database file change-counter,
** stored at byte 24 of the pager file.
................................................................................
  if( pPager->state<PAGER_RESERVED ){
    return SQLITE_ERROR;
  }
  PAGERTRACE2("COMMIT %d\n", PAGERID(pPager));
  if( MEMDB ){
    pPg = pager_get_all_dirty_pages(pPager);
    while( pPg ){
      clearHistory(PGHDR_TO_HIST(pPg, pPager));

      pPg->dirty = 0;
      pPg->inJournal = 0;
      pPg->inStmt = 0;
      pPg->needSync = 0;
      pPg->pPrevStmt = pPg->pNextStmt = 0;
      pPg = pPg->pDirty;
    }
    pPager->pDirty = 0;
#ifndef NDEBUG
    for(pPg=pPager->pAll; pPg; pPg=pPg->pNextAll){
      PgHistory *pHist = PGHDR_TO_HIST(pPg, pPager);
      assert( !pPg->alwaysRollback );
................................................................................
        PAGERTRACE3("ROLLBACK-PAGE %d of %d\n", p->pgno, PAGERID(pPager));
      }else{
        PAGERTRACE3("PAGE %d is clean on %d\n", p->pgno, PAGERID(pPager));
      }
      clearHistory(pHist);
      p->dirty = 0;
      p->inJournal = 0;
      p->inStmt = 0;
      p->pPrevStmt = p->pNextStmt = 0;
      if( pPager->xReiniter ){
        pPager->xReiniter(p, pPager->pageSize);
      }
    }
    pPager->pDirty = 0;
    pPager->pStmt = 0;
    pPager->dbSize = pPager->origDbSize;
................................................................................
    PgHdr *pPg, *pNext;
    PAGERTRACE2("STMT-COMMIT %d\n", PAGERID(pPager));
    if( !MEMDB ){
      sqlite3OsSeek(pPager->stfd, 0);
      /* sqlite3OsTruncate(pPager->stfd, 0); */
      sqliteFree( pPager->aInStmt );
      pPager->aInStmt = 0;
    }
    for(pPg=pPager->pStmt; pPg; pPg=pNext){

      pNext = pPg->pNextStmt;
      assert( pPg->inStmt );
      pPg->inStmt = 0;
      pPg->pPrevStmt = pPg->pNextStmt = 0;
      if( MEMDB ){
        PgHistory *pHist = PGHDR_TO_HIST(pPg, pPager);
        sqliteFree(pHist->pStmt);
        pHist->pStmt = 0;
      }
    }
    pPager->stmtNRec = 0;
    pPager->stmtInUse = 0;
    pPager->pStmt = 0;
................................................................................
*/
int sqlite3PagerStmtRollback(Pager *pPager){
  int rc;
  if( pPager->stmtInUse ){
    PAGERTRACE2("STMT-ROLLBACK %d\n", PAGERID(pPager));
    if( MEMDB ){
      PgHdr *pPg;

      for(pPg=pPager->pStmt; pPg; pPg=pPg->pNextStmt){
        PgHistory *pHist = PGHDR_TO_HIST(pPg, pPager);
        if( pHist->pStmt ){
          memcpy(PGHDR_TO_DATA(pPg), pHist->pStmt, pPager->pageSize);
          sqliteFree(pHist->pStmt);
          pHist->pStmt = 0;
        }
      }
      pPager->dbSize = pPager->stmtSize;







|







 







<

<







 







>
>







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







>
>
|
|
<
|
<
>
|
|
|
|
|
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<







 







<







 







>









<
<
<
<
<
|







 







|







 







<







 







>
>
|
>









>












<







 







<




>
>
|
>



<







 







|
>


|

|







 







|
|







 







|
|
>
|
|
|
|
<
<







 







>
|
|







14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
...
154
155
156
157
158
159
160

161

162
163
164
165
166
167
168
...
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
...
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
...
825
826
827
828
829
830
831
832
833
834
835

836

837
838
839
840
841
842
















843
844
845
846
847
848
849
....
2663
2664
2665
2666
2667
2668
2669

2670
2671
2672
2673
2674
2675
2676
....
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019





3020
3021
3022
3023
3024
3025
3026
3027
....
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
....
3440
3441
3442
3443
3444
3445
3446

3447
3448
3449
3450
3451
3452
3453
....
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490

3491
3492
3493
3494
3495
3496
3497
....
3674
3675
3676
3677
3678
3679
3680

3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691

3692
3693
3694
3695
3696
3697
3698
....
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
....
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
....
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059


4060
4061
4062
4063
4064
4065
4066
....
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086
4087
4088
4089
4090
** The pager is used to access a database disk file.  It implements
** atomic commit and rollback through the use of a journal file that
** is separate from the database file.  The pager also implements file
** locking to prevent two processes from writing the same database
** file simultaneously, or one process from reading the database while
** another is writing.
**
** @(#) $Id: pager.c,v 1.325 2007/04/07 15:03:17 danielk1977 Exp $
*/
#ifndef SQLITE_OMIT_DISKIO
#include "sqliteInt.h"
#include "os.h"
#include "pager.h"
#include <assert.h>
#include <string.h>
................................................................................
typedef struct PgHdr PgHdr;
struct PgHdr {
  Pager *pPager;                 /* The pager to which this page belongs */
  Pgno pgno;                     /* The page number for this page */
  PgHdr *pNextHash, *pPrevHash;  /* Hash collision chain for PgHdr.pgno */
  PgHdr *pNextFree, *pPrevFree;  /* Freelist of pages where nRef==0 */
  PgHdr *pNextAll;               /* A list of all pages */

  u8 inJournal;                  /* TRUE if has been written to journal */

  u8 dirty;                      /* TRUE if we need to write back changes */
  u8 needSync;                   /* Sync journal before writing this page */
  u8 alwaysRollback;             /* Disable DontRollback() for this page */
  short int nRef;                /* Number of users of this page */
  PgHdr *pDirty, *pPrevDirty;    /* Dirty pages */
  u32 notUsed;                   /* Buffer space */
#ifdef SQLITE_CHECK_PAGES
................................................................................
** even for disk-based databases.  Splitting it out saves 8 bytes.  This
** is only a savings of 0.8% but those percentages add up.
*/
typedef struct PgHistory PgHistory;
struct PgHistory {
  u8 *pOrig;     /* Original page text.  Restore to this on a full rollback */
  u8 *pStmt;     /* Text as it was at the beginning of the current statement */
  PgHdr *pNextStmt, *pPrevStmt;  /* List of pages in the statement journal */
  u8 inStmt;                     /* TRUE if in the statement subjournal */
};

/*
** A macro used for invoking the codec if there is one
*/
#ifdef SQLITE_HAS_CODEC
# define CODEC1(P,D,N,X) if( P->xCodec!=0 ){ P->xCodec(P->pCodecArg,D,N,X); }
................................................................................
    cnt++;   /* Something to set a breakpoint on */
  }
# define REFINFO(X)  pager_refinfo(X)
#else
# define REFINFO(X)
#endif

/*
** Return true if page *pPg has already been written to the statement
** journal (or statement snapshot has been created, if *pPg is part
** of an in-memory database).
*/
static int pageInStatement(PgHdr *pPg){
  Pager *pPager = pPg->pPager;
  if( MEMDB ){
    return PGHDR_TO_HIST(pPg, pPager)->inStmt;
  }else{
    Pgno pgno = pPg->pgno;
    u8 *a = pPager->aInStmt;
    return (a && (int)pgno<=pPager->stmtSize && (a[pgno/8] & (1<<(pgno&7))));
  }
}

/*
** Change the size of the pager hash table to N.  N must be a power
** of two.
*/
static void pager_resize_hash_table(Pager *pPager, int N){
  PgHdr **aHash, *pPg;
................................................................................
** The Pager keeps a separate list of pages that are currently in
** the statement journal.  This helps the sqlite3PagerStmtCommit()
** routine run MUCH faster for the common case where there are many
** pages in memory but only a few are in the statement journal.
*/
static void page_add_to_stmt_list(PgHdr *pPg){
  Pager *pPager = pPg->pPager;
  PgHistory *pHist = PGHDR_TO_HIST(pPg, pPager);
  assert( MEMDB );
  if( !pHist->inStmt ){
    assert( pHist->pPrevStmt==0 && pHist->pNextStmt==0 );

    if( pPager->pStmt ){

      PGHDR_TO_HIST(pPager->pStmt, pPager)->pPrevStmt = pPg;
    }
    pHist->pNextStmt = pPager->pStmt;
    pPager->pStmt = pPg;
    pHist->inStmt = 1;
  }
















}

/*
** Find a page in the hash table given its page number.  Return
** a pointer to the page or NULL if not found.
*/
static PgHdr *pager_lookup(Pager *pPager, Pgno pgno){
................................................................................
        ** Remove it from this list before freeing.
        **
        ** Todo: Check the Pager.pStmt list to make sure this is Ok. It 
        ** probably is though.
        */
        PgHdr *pTmp;
        assert( pPg );

        if( pPg==pPager->pAll ){
           pPager->pAll = pPg->pNextAll;
        }else{
          for( pTmp=pPager->pAll; pTmp->pNextAll!=pPg; pTmp=pTmp->pNextAll ){}
          pTmp->pNextAll = pPg->pNextAll;
        }
        nReleased += sqliteAllocSize(pPg);
................................................................................
    TEST_INCR(pPager->nMiss);
    rc = pagerAllocatePage(pPager, &pPg);
    if( rc!=SQLITE_OK ){
      return rc;
    }

    pPg->pgno = pgno;
    assert( !MEMDB || pgno>pPager->stmtSize );
    if( pPager->aInJournal && (int)pgno<=pPager->origDbSize ){
      sqlite3CheckMemory(pPager->aInJournal, pgno/8);
      assert( pPager->journalOpen );
      pPg->inJournal = (pPager->aInJournal[pgno/8] & (1<<(pgno&7)))!=0;
      pPg->needSync = 0;
    }else{
      pPg->inJournal = 0;
      pPg->needSync = 0;
    }






    makeClean(pPg);
    pPg->nRef = 1;
    REFINFO(pPg);

    pPager->nRef++;
    if( pPager->nExtra>0 ){
      memset(PGHDR_TO_EXTRA(pPg, pPager), 0, pPager->nExtra);
................................................................................

  CHECK_PAGE(pPg);

  /* Mark the page as dirty.  If the page has already been written
  ** to the journal then we can return right away.
  */
  makeDirty(pPg);
  if( pPg->inJournal && (pageInStatement(pPg) || pPager->stmtInUse==0) ){
    pPager->dirtyCache = 1;
  }else{

    /* If we get this far, it means that the page needs to be
    ** written to the transaction journal or the ckeckpoint journal
    ** or both.
    **
................................................................................

          pPager->nRec++;
          assert( pPager->aInJournal!=0 );
          pPager->aInJournal[pPg->pgno/8] |= 1<<(pPg->pgno&7);
          pPg->needSync = !pPager->noSync;
          if( pPager->stmtInUse ){
            pPager->aInStmt[pPg->pgno/8] |= 1<<(pPg->pgno&7);

          }
        }
      }else{
        pPg->needSync = !pPager->journalStarted && !pPager->noSync;
        PAGERTRACE4("APPEND %d page %d needSync=%d\n",
                PAGERID(pPager), pPg->pgno, pPg->needSync);
      }
................................................................................
    }
  
    /* If the statement journal is open and the page is not in it,
    ** then write the current page to the statement journal.  Note that
    ** the statement journal format differs from the standard journal format
    ** in that it omits the checksums and the header.
    */
    if( pPager->stmtInUse 
     && !pageInStatement(pPg) 
     && (int)pPg->pgno<=pPager->stmtSize 
    ){
      assert( pPg->inJournal || (int)pPg->pgno>pPager->origDbSize );
      if( MEMDB ){
        PgHistory *pHist = PGHDR_TO_HIST(pPg, pPager);
        assert( pHist->pStmt==0 );
        pHist->pStmt = sqliteMallocRaw( pPager->pageSize );
        if( pHist->pStmt ){
          memcpy(pHist->pStmt, PGHDR_TO_DATA(pPg), pPager->pageSize);
        }
        PAGERTRACE3("STMT-JOURNAL %d page %d\n", PAGERID(pPager), pPg->pgno);
        page_add_to_stmt_list(pPg);
      }else{
        char *pData2 = CODEC2(pPager, pData, pPg->pgno, 7)-4;
        put32bits(pData2, pPg->pgno);
        rc = sqlite3OsWrite(pPager->stfd, pData2, pPager->pageSize+4);
        PAGERTRACE3("STMT-JOURNAL %d page %d\n", PAGERID(pPager), pPg->pgno);
        if( rc!=SQLITE_OK ){
          return rc;
        }
        pPager->stmtNRec++;
        assert( pPager->aInStmt!=0 );
        pPager->aInStmt[pPg->pgno/8] |= 1<<(pPg->pgno&7);
      }

    }
  }

  /* Update the database size and return.
  */
  assert( pPager->state>=PAGER_SHARED );
  if( pPager->dbSize<(int)pPg->pgno ){
................................................................................
  if( pPg->alwaysRollback || pPager->alwaysRollback || MEMDB ) return;
  if( !pPg->inJournal && (int)pPg->pgno <= pPager->origDbSize ){
    assert( pPager->aInJournal!=0 );
    pPager->aInJournal[pPg->pgno/8] |= 1<<(pPg->pgno&7);
    pPg->inJournal = 1;
    if( pPager->stmtInUse ){
      pPager->aInStmt[pPg->pgno/8] |= 1<<(pPg->pgno&7);

    }
    PAGERTRACE3("DONT_ROLLBACK page %d of %d\n", pPg->pgno, PAGERID(pPager));
    IOTRACE(("GARBAGE %p %d\n", pPager, pPg->pgno))
  }
  if( pPager->stmtInUse 
   && !pageInStatement(pPg) 
   && (int)pPg->pgno<=pPager->stmtSize 
  ){
    assert( pPg->inJournal || (int)pPg->pgno>pPager->origDbSize );
    assert( pPager->aInStmt!=0 );
    pPager->aInStmt[pPg->pgno/8] |= 1<<(pPg->pgno&7);

  }
}


/*
** This routine is called to increment the database file change-counter,
** stored at byte 24 of the pager file.
................................................................................
  if( pPager->state<PAGER_RESERVED ){
    return SQLITE_ERROR;
  }
  PAGERTRACE2("COMMIT %d\n", PAGERID(pPager));
  if( MEMDB ){
    pPg = pager_get_all_dirty_pages(pPager);
    while( pPg ){
      PgHistory *pHist = PGHDR_TO_HIST(pPg, pPager);
      clearHistory(pHist);
      pPg->dirty = 0;
      pPg->inJournal = 0;
      pHist->inStmt = 0;
      pPg->needSync = 0;
      pHist->pPrevStmt = pHist->pNextStmt = 0;
      pPg = pPg->pDirty;
    }
    pPager->pDirty = 0;
#ifndef NDEBUG
    for(pPg=pPager->pAll; pPg; pPg=pPg->pNextAll){
      PgHistory *pHist = PGHDR_TO_HIST(pPg, pPager);
      assert( !pPg->alwaysRollback );
................................................................................
        PAGERTRACE3("ROLLBACK-PAGE %d of %d\n", p->pgno, PAGERID(pPager));
      }else{
        PAGERTRACE3("PAGE %d is clean on %d\n", p->pgno, PAGERID(pPager));
      }
      clearHistory(pHist);
      p->dirty = 0;
      p->inJournal = 0;
      pHist->inStmt = 0;
      pHist->pPrevStmt = pHist->pNextStmt = 0;
      if( pPager->xReiniter ){
        pPager->xReiniter(p, pPager->pageSize);
      }
    }
    pPager->pDirty = 0;
    pPager->pStmt = 0;
    pPager->dbSize = pPager->origDbSize;
................................................................................
    PgHdr *pPg, *pNext;
    PAGERTRACE2("STMT-COMMIT %d\n", PAGERID(pPager));
    if( !MEMDB ){
      sqlite3OsSeek(pPager->stfd, 0);
      /* sqlite3OsTruncate(pPager->stfd, 0); */
      sqliteFree( pPager->aInStmt );
      pPager->aInStmt = 0;
    }else{
      for(pPg=pPager->pStmt; pPg; pPg=pNext){
        PgHistory *pHist = PGHDR_TO_HIST(pPg, pPager);
        pNext = pHist->pNextStmt;
        assert( pHist->inStmt );
        pHist->inStmt = 0;
        pHist->pPrevStmt = pHist->pNextStmt = 0;


        sqliteFree(pHist->pStmt);
        pHist->pStmt = 0;
      }
    }
    pPager->stmtNRec = 0;
    pPager->stmtInUse = 0;
    pPager->pStmt = 0;
................................................................................
*/
int sqlite3PagerStmtRollback(Pager *pPager){
  int rc;
  if( pPager->stmtInUse ){
    PAGERTRACE2("STMT-ROLLBACK %d\n", PAGERID(pPager));
    if( MEMDB ){
      PgHdr *pPg;
      PgHistory *pHist;
      for(pPg=pPager->pStmt; pPg; pPg=pHist->pNextStmt){
        pHist = PGHDR_TO_HIST(pPg, pPager);
        if( pHist->pStmt ){
          memcpy(PGHDR_TO_DATA(pPg), pHist->pStmt, pPager->pageSize);
          sqliteFree(pHist->pStmt);
          pHist->pStmt = 0;
        }
      }
      pPager->dbSize = pPager->stmtSize;

Changes to test/autovacuum.test.

7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
...
487
488
489
490
491
492
493

494
495
496
497
498












499
500
501
502
503
504
505
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file implements regression tests for SQLite library.  The
# focus of this file is testing the SELECT statement.
#
# $Id: autovacuum.test,v 1.25 2007/04/02 14:19:16 danielk1977 Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# If this build of the library does not support auto-vacuum, omit this
# whole file.
ifcapable {!autovacuum || !pragma} {
................................................................................
} $AUTOVACUUM
do_test autovacuum-3.7 {
  execsql {
    DROP TABLE av1;
  }
  file_pages
} [expr $AUTOVACUUM?1:2]


#-----------------------------------------------------------------------
# Test that if a statement transaction around a CREATE INDEX statement is
# rolled back no corruption occurs.
#












do_test autovacuum-4.1 {
  execsql {
    CREATE TABLE av1(a, b);
    BEGIN;
  }
  for {set i 0} {$i<100} {incr i} {
    execsql "INSERT INTO av1 VALUES($i, '[string repeat X 200]');"







|







 







>





>
>
>
>
>
>
>
>
>
>
>
>







7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
...
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file implements regression tests for SQLite library.  The
# focus of this file is testing the SELECT statement.
#
# $Id: autovacuum.test,v 1.26 2007/04/07 15:03:17 danielk1977 Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# If this build of the library does not support auto-vacuum, omit this
# whole file.
ifcapable {!autovacuum || !pragma} {
................................................................................
} $AUTOVACUUM
do_test autovacuum-3.7 {
  execsql {
    DROP TABLE av1;
  }
  file_pages
} [expr $AUTOVACUUM?1:2]


#-----------------------------------------------------------------------
# Test that if a statement transaction around a CREATE INDEX statement is
# rolled back no corruption occurs.
#
do_test autovacuum-4.0 {
  # The last round of tests may have left the db in non-autovacuum mode.
  # Reset everything just in case.
  #
  db close
  file delete -force test.db test.db-journal
  sqlite3 db test.db
  execsql {
    PRAGMA auto_vacuum = 1;
    PRAGMA auto_vacuum;
  }
} {1}
do_test autovacuum-4.1 {
  execsql {
    CREATE TABLE av1(a, b);
    BEGIN;
  }
  for {set i 0} {$i<100} {incr i} {
    execsql "INSERT INTO av1 VALUES($i, '[string repeat X 200]');"