SQLite

Check-in [ab7aeeead3]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Simplifications and performance improvement in pager_write().
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: ab7aeeead395a05b91a921ef9ebe9252fffad667
User & Date: drh 2015-06-29 14:11:50.755
Context
2015-06-29
15:41
Fix minor problems in the ota demo application. (check-in: 6aaaec6e63 user: dan tags: trunk)
14:11
Simplifications and performance improvement in pager_write(). (check-in: ab7aeeead3 user: drh tags: trunk)
04:21
Add the new PGHDR_CLEAN bit to PgHdr.flags in pcache.c. This bit is always the opposite of PGHDR_DIRTY. Use the extra bit to avoid a comparison for a small performance boost. (check-in: 8619fc346d user: drh tags: trunk)
Changes
Unified Diff Ignore Whitespace Patch
Changes to src/bitvec.c.
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
}

/*
** Check to see if the i-th bit is set.  Return true or false.
** If p is NULL (if the bitmap has not been created) or if
** i is out of range, then return false.
*/
int sqlite3BitvecTest(Bitvec *p, u32 i){
  if( p==0 ) return 0;
  i--;
  if( i>=p->iSize ) return 0;
  while( p->iDivisor ){
    u32 bin = i/p->iDivisor;
    i = i%p->iDivisor;
    p = p->u.apSub[bin];
    if (!p) {







|
|







122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
}

/*
** Check to see if the i-th bit is set.  Return true or false.
** If p is NULL (if the bitmap has not been created) or if
** i is out of range, then return false.
*/
int sqlite3BitvecTestNotNull(Bitvec *p, u32 i){
  assert( p!=0 );
  i--;
  if( i>=p->iSize ) return 0;
  while( p->iDivisor ){
    u32 bin = i/p->iDivisor;
    i = i%p->iDivisor;
    p = p->u.apSub[bin];
    if (!p) {
145
146
147
148
149
150
151



152
153
154
155
156
157
158
    while( p->u.aHash[h] ){
      if( p->u.aHash[h]==i ) return 1;
      h = (h+1) % BITVEC_NINT;
    }
    return 0;
  }
}




/*
** Set the i-th bit.  Return 0 on success and an error code if
** anything goes wrong.
**
** This routine might cause sub-bitmaps to be allocated.  Failing
** to get the memory needed to hold the sub-bitmap is the only







>
>
>







145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
    while( p->u.aHash[h] ){
      if( p->u.aHash[h]==i ) return 1;
      h = (h+1) % BITVEC_NINT;
    }
    return 0;
  }
}
int sqlite3BitvecTest(Bitvec *p, u32 i){
  return p!=0 && sqlite3BitvecTestNotNull(p,i);
}

/*
** Set the i-th bit.  Return 0 on success and an error code if
** anything goes wrong.
**
** This routine might cause sub-bitmaps to be allocated.  Failing
** to get the memory needed to hold the sub-bitmap is the only
Changes to src/pager.c.
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
**
**   if( isOpen(pPager->jfd) ){ ...
**
** instead of
**
**   if( pPager->jfd->pMethods ){ ...
*/
#define isOpen(pFd) ((pFd)->pMethods)

/*
** Return true if this pager uses a write-ahead log instead of the usual
** rollback journal. Otherwise false.
*/
#ifndef SQLITE_OMIT_WAL
static int pagerUseWal(Pager *pPager){







|







804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
**
**   if( isOpen(pPager->jfd) ){ ...
**
** instead of
**
**   if( pPager->jfd->pMethods ){ ...
*/
#define isOpen(pFd) ((pFd)->pMethods!=0)

/*
** Return true if this pager uses a write-ahead log instead of the usual
** rollback journal. Otherwise false.
*/
#ifndef SQLITE_OMIT_WAL
static int pagerUseWal(Pager *pPager){
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040

1041
1042
1043
1044
1045
1046

1047
1048
1049
1050
1051
1052
1053
static int subjRequiresPage(PgHdr *pPg){
  Pager *pPager = pPg->pPager;
  PagerSavepoint *p;
  Pgno pgno = pPg->pgno;
  int i;
  for(i=0; i<pPager->nSavepoint; i++){
    p = &pPager->aSavepoint[i];
    if( p->nOrig>=pgno && 0==sqlite3BitvecTest(p->pInSavepoint, pgno) ){
      return 1;
    }
  }
  return 0;
}


/*
** Return true if the page is already in the journal file.
*/
static int pageInJournal(Pager *pPager, PgHdr *pPg){
  return sqlite3BitvecTest(pPager->pInJournal, pPg->pgno);
}


/*
** Read a 32-bit integer from the given file descriptor.  Store the integer
** that is read in *pRes.  Return SQLITE_OK if everything worked, or an
** error code is something goes wrong.
**
** All values are stored on disk as big-endian.







|






>






>







1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
static int subjRequiresPage(PgHdr *pPg){
  Pager *pPager = pPg->pPager;
  PagerSavepoint *p;
  Pgno pgno = pPg->pgno;
  int i;
  for(i=0; i<pPager->nSavepoint; i++){
    p = &pPager->aSavepoint[i];
    if( p->nOrig>=pgno && 0==sqlite3BitvecTestNotNull(p->pInSavepoint, pgno) ){
      return 1;
    }
  }
  return 0;
}

#ifdef SQLITE_DEBUG
/*
** Return true if the page is already in the journal file.
*/
static int pageInJournal(Pager *pPager, PgHdr *pPg){
  return sqlite3BitvecTest(pPager->pInJournal, pPg->pgno);
}
#endif

/*
** Read a 32-bit integer from the given file descriptor.  Store the integer
** that is read in *pRes.  Return SQLITE_OK if everything worked, or an
** error code is something goes wrong.
**
** All values are stored on disk as big-endian.
5643
5644
5645
5646
5647
5648
5649





















































5650
5651
5652
5653
5654
5655
5656
5657
5658
5659
5660
5661
5662
5663
5664
5665
5666
5667
5668
5669
5670
5671
5672
5673
5674
5675
5676
5677
5678
5679
5680
5681
5682
5683
5684
5685
5686
5687
5688
5689
5690
5691
5692
5693
5694
5695
5696
5697
5698
5699
5700
5701
5702
5703
5704
5705
5706
5707
5708
5709
5710
5711
5712
5713
5714
5715
5716
5717
5718
5719
5720
5721
5722
5723
5724
5725
5726
5727
5728
5729
5730
5731
5732
5733
5734
5735
5736
5737
5738
5739
5740
5741
5742
5743
5744
5745
5746
5747
5748
5749
5750
5751
5752
5753
5754
5755
5756
5757
5758
5759
5760
5761
5762
5763
5764
5765
5766
5767
5768
5769
5770
5771
5772
5773
5774
5775
5776
5777
5778
5779
5780
5781
5782
5783
5784
    assert( rc!=SQLITE_OK || pPager->eState==PAGER_WRITER_LOCKED );
    assert( assert_pager_state(pPager) );
  }

  PAGERTRACE(("TRANSACTION %d\n", PAGERID(pPager)));
  return rc;
}






















































/*
** Mark a single data page as writeable. The page is written into the 
** main journal or sub-journal as required. If the page is written into
** one of the journals, the corresponding bit is set in the 
** Pager.pInJournal bitvec and the PagerSavepoint.pInSavepoint bitvecs
** of any open savepoints as appropriate.
*/
static int pager_write(PgHdr *pPg){
  Pager *pPager = pPg->pPager;
  int rc = SQLITE_OK;
  int inJournal;

  /* This routine is not called unless a write-transaction has already 
  ** been started. The journal file may or may not be open at this point.
  ** It is never called in the ERROR state.
  */
  assert( pPager->eState==PAGER_WRITER_LOCKED
       || pPager->eState==PAGER_WRITER_CACHEMOD
       || pPager->eState==PAGER_WRITER_DBMOD
  );
  assert( assert_pager_state(pPager) );
  assert( pPager->errCode==0 );
  assert( pPager->readOnly==0 );

  CHECK_PAGE(pPg);

  /* The journal file needs to be opened. Higher level routines have already
  ** obtained the necessary locks to begin the write-transaction, but the
  ** rollback journal might not yet be open. Open it now if this is the case.
  **
  ** This is done before calling sqlite3PcacheMakeDirty() on the page. 
  ** Otherwise, if it were done after calling sqlite3PcacheMakeDirty(), then
  ** an error might occur and the pager would end up in WRITER_LOCKED state
  ** with pages marked as dirty in the cache.
  */
  if( pPager->eState==PAGER_WRITER_LOCKED ){
    rc = pager_open_journal(pPager);
    if( rc!=SQLITE_OK ) return rc;
  }
  assert( pPager->eState>=PAGER_WRITER_CACHEMOD );
  assert( assert_pager_state(pPager) );

  /* Mark the page as dirty.  If the page has already been written
  ** to the journal then we can return right away.
  */
  sqlite3PcacheMakeDirty(pPg);
  inJournal = pageInJournal(pPager, pPg);
  if( inJournal && (pPager->nSavepoint==0 || !subjRequiresPage(pPg)) ){
    assert( !pagerUseWal(pPager) );
  }else{
  
    /* The transaction journal now exists and we have a RESERVED or an
    ** EXCLUSIVE lock on the main database file.  Write the current page to
    ** the transaction journal if it is not there already.
    */
    if( !inJournal && !pagerUseWal(pPager) ){
      assert( pagerUseWal(pPager)==0 );
      if( pPg->pgno<=pPager->dbOrigSize && isOpen(pPager->jfd) ){
        u32 cksum;
        char *pData2;
        i64 iOff = pPager->journalOff;

        /* We should never write to the journal file the page that
        ** contains the database locks.  The following assert verifies
        ** that we do not. */
        assert( pPg->pgno!=PAGER_MJ_PGNO(pPager) );

        assert( pPager->journalHdr<=pPager->journalOff );
        CODEC2(pPager, pPg->pData, pPg->pgno, 7, return SQLITE_NOMEM, pData2);
        cksum = pager_cksum(pPager, (u8*)pData2);

        /* Even if an IO or diskfull error occurs while journalling the
        ** page in the block above, set the need-sync flag for the page.
        ** Otherwise, when the transaction is rolled back, the logic in
        ** playback_one_page() will think that the page needs to be restored
        ** in the database file. And if an IO error occurs while doing so,
        ** then corruption may follow.
        */
        pPg->flags |= PGHDR_NEED_SYNC;

        rc = write32bits(pPager->jfd, iOff, pPg->pgno);
        if( rc!=SQLITE_OK ) return rc;
        rc = sqlite3OsWrite(pPager->jfd, pData2, pPager->pageSize, iOff+4);
        if( rc!=SQLITE_OK ) return rc;
        rc = write32bits(pPager->jfd, iOff+pPager->pageSize+4, cksum);
        if( rc!=SQLITE_OK ) return rc;

        IOTRACE(("JOUT %p %d %lld %d\n", pPager, pPg->pgno, 
                 pPager->journalOff, pPager->pageSize));
        PAGER_INCR(sqlite3_pager_writej_count);
        PAGERTRACE(("JOURNAL %d page %d needSync=%d hash(%08x)\n",
             PAGERID(pPager), pPg->pgno, 
             ((pPg->flags&PGHDR_NEED_SYNC)?1:0), pager_pagehash(pPg)));

        pPager->journalOff += 8 + pPager->pageSize;
        pPager->nRec++;
        assert( pPager->pInJournal!=0 );
        rc = sqlite3BitvecSet(pPager->pInJournal, pPg->pgno);
        testcase( rc==SQLITE_NOMEM );
        assert( rc==SQLITE_OK || rc==SQLITE_NOMEM );
        rc |= addToSavepointBitvecs(pPager, pPg->pgno);
        if( rc!=SQLITE_OK ){
          assert( rc==SQLITE_NOMEM );
          return rc;
        }
      }else{
        if( pPager->eState!=PAGER_WRITER_DBMOD ){
          pPg->flags |= PGHDR_NEED_SYNC;
        }
        PAGERTRACE(("APPEND %d page %d needSync=%d\n",
                PAGERID(pPager), pPg->pgno,
               ((pPg->flags&PGHDR_NEED_SYNC)?1:0)));
      }
    }
  
    /* If the statement journal is open and the page is not in it,
    ** then write the current page to the statement journal.  Note that
    ** the statement journal format differs from the standard journal format
    ** in that it omits the checksums and the header.
    */
    if( pPager->nSavepoint>0 && subjRequiresPage(pPg) ){
      rc = subjournalPage(pPg);
    }
  }

  /* Update the database size and return.
  */
  if( pPager->dbSize<pPg->pgno ){
    pPager->dbSize = pPg->pgno;
  }
  return rc;
}

/*







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>











<












<


















|
<
<

<
<
<
<
|
<
<
<
<
<
<
<
<
<
<
|
<
<
<
<
|
<
<
<
|
<
<
<
<
<
<
|
<
|
<
<
<
<
<
<
|
<
<
<
<
|
<
|
|
|
|
<
<
<
<
|
<
|
|
|
|
|
|
|
|
|
|
|

|
|
<
<
|
|
|
|
|
<
|
<







5645
5646
5647
5648
5649
5650
5651
5652
5653
5654
5655
5656
5657
5658
5659
5660
5661
5662
5663
5664
5665
5666
5667
5668
5669
5670
5671
5672
5673
5674
5675
5676
5677
5678
5679
5680
5681
5682
5683
5684
5685
5686
5687
5688
5689
5690
5691
5692
5693
5694
5695
5696
5697
5698
5699
5700
5701
5702
5703
5704
5705
5706
5707
5708
5709
5710
5711
5712
5713
5714
5715

5716
5717
5718
5719
5720
5721
5722
5723
5724
5725
5726
5727

5728
5729
5730
5731
5732
5733
5734
5735
5736
5737
5738
5739
5740
5741
5742
5743
5744
5745
5746


5747




5748










5749




5750



5751






5752

5753






5754




5755

5756
5757
5758
5759




5760

5761
5762
5763
5764
5765
5766
5767
5768
5769
5770
5771
5772
5773
5774


5775
5776
5777
5778
5779

5780

5781
5782
5783
5784
5785
5786
5787
    assert( rc!=SQLITE_OK || pPager->eState==PAGER_WRITER_LOCKED );
    assert( assert_pager_state(pPager) );
  }

  PAGERTRACE(("TRANSACTION %d\n", PAGERID(pPager)));
  return rc;
}

/*
** Write page pPg onto the end of the rollback journal.
*/
static SQLITE_NOINLINE int pagerAddPageToRollbackJournal(PgHdr *pPg){
  Pager *pPager = pPg->pPager;
  int rc;
  u32 cksum;
  char *pData2;
  i64 iOff = pPager->journalOff;

  /* We should never write to the journal file the page that
  ** contains the database locks.  The following assert verifies
  ** that we do not. */
  assert( pPg->pgno!=PAGER_MJ_PGNO(pPager) );

  assert( pPager->journalHdr<=pPager->journalOff );
  CODEC2(pPager, pPg->pData, pPg->pgno, 7, return SQLITE_NOMEM, pData2);
  cksum = pager_cksum(pPager, (u8*)pData2);

  /* Even if an IO or diskfull error occurs while journalling the
  ** page in the block above, set the need-sync flag for the page.
  ** Otherwise, when the transaction is rolled back, the logic in
  ** playback_one_page() will think that the page needs to be restored
  ** in the database file. And if an IO error occurs while doing so,
  ** then corruption may follow.
  */
  pPg->flags |= PGHDR_NEED_SYNC;

  rc = write32bits(pPager->jfd, iOff, pPg->pgno);
  if( rc!=SQLITE_OK ) return rc;
  rc = sqlite3OsWrite(pPager->jfd, pData2, pPager->pageSize, iOff+4);
  if( rc!=SQLITE_OK ) return rc;
  rc = write32bits(pPager->jfd, iOff+pPager->pageSize+4, cksum);
  if( rc!=SQLITE_OK ) return rc;

  IOTRACE(("JOUT %p %d %lld %d\n", pPager, pPg->pgno, 
           pPager->journalOff, pPager->pageSize));
  PAGER_INCR(sqlite3_pager_writej_count);
  PAGERTRACE(("JOURNAL %d page %d needSync=%d hash(%08x)\n",
       PAGERID(pPager), pPg->pgno, 
       ((pPg->flags&PGHDR_NEED_SYNC)?1:0), pager_pagehash(pPg)));

  pPager->journalOff += 8 + pPager->pageSize;
  pPager->nRec++;
  assert( pPager->pInJournal!=0 );
  rc = sqlite3BitvecSet(pPager->pInJournal, pPg->pgno);
  testcase( rc==SQLITE_NOMEM );
  assert( rc==SQLITE_OK || rc==SQLITE_NOMEM );
  rc |= addToSavepointBitvecs(pPager, pPg->pgno);
  assert( rc==SQLITE_OK || rc==SQLITE_NOMEM );
  return rc;
}

/*
** Mark a single data page as writeable. The page is written into the 
** main journal or sub-journal as required. If the page is written into
** one of the journals, the corresponding bit is set in the 
** Pager.pInJournal bitvec and the PagerSavepoint.pInSavepoint bitvecs
** of any open savepoints as appropriate.
*/
static int pager_write(PgHdr *pPg){
  Pager *pPager = pPg->pPager;
  int rc = SQLITE_OK;


  /* This routine is not called unless a write-transaction has already 
  ** been started. The journal file may or may not be open at this point.
  ** It is never called in the ERROR state.
  */
  assert( pPager->eState==PAGER_WRITER_LOCKED
       || pPager->eState==PAGER_WRITER_CACHEMOD
       || pPager->eState==PAGER_WRITER_DBMOD
  );
  assert( assert_pager_state(pPager) );
  assert( pPager->errCode==0 );
  assert( pPager->readOnly==0 );

  CHECK_PAGE(pPg);

  /* The journal file needs to be opened. Higher level routines have already
  ** obtained the necessary locks to begin the write-transaction, but the
  ** rollback journal might not yet be open. Open it now if this is the case.
  **
  ** This is done before calling sqlite3PcacheMakeDirty() on the page. 
  ** Otherwise, if it were done after calling sqlite3PcacheMakeDirty(), then
  ** an error might occur and the pager would end up in WRITER_LOCKED state
  ** with pages marked as dirty in the cache.
  */
  if( pPager->eState==PAGER_WRITER_LOCKED ){
    rc = pager_open_journal(pPager);
    if( rc!=SQLITE_OK ) return rc;
  }
  assert( pPager->eState>=PAGER_WRITER_CACHEMOD );
  assert( assert_pager_state(pPager) );

  /* Mark the page that is about to be modified as dirty. */


  sqlite3PcacheMakeDirty(pPg);















  /* If a rollback journal is in use, them make sure the page that is about




  ** to change is in the rollback journal, or if the page is a new page off



  ** then end of the file, make sure it is marked as PGHDR_NEED_SYNC.






  */

  assert( (pPager->pInJournal!=0) == isOpen(pPager->jfd) );






  if( pPager->pInJournal!=0                                      /* Journal open */




   && sqlite3BitvecTestNotNull(pPager->pInJournal, pPg->pgno)==0 /* pPg not in jrnl */

  ){
    assert( pagerUseWal(pPager)==0 );
    if( pPg->pgno<=pPager->dbOrigSize ){
      rc = pagerAddPageToRollbackJournal(pPg);




      if( rc!=SQLITE_OK ){

        return rc;
      }
    }else{
      if( pPager->eState!=PAGER_WRITER_DBMOD ){
        pPg->flags |= PGHDR_NEED_SYNC;
      }
      PAGERTRACE(("APPEND %d page %d needSync=%d\n",
              PAGERID(pPager), pPg->pgno,
             ((pPg->flags&PGHDR_NEED_SYNC)?1:0)));
    }
  }
  
  /* If the statement journal is open and the page is not in it,
  ** then write the page into the statement journal.


  */
  if( pPager->nSavepoint>0 && subjRequiresPage(pPg) ){
    rc = subjournalPage(pPg);
  }


  /* Update the database size and return. */

  if( pPager->dbSize<pPg->pgno ){
    pPager->dbSize = pPg->pgno;
  }
  return rc;
}

/*
Changes to src/sqliteInt.h.
3234
3235
3236
3237
3238
3239
3240

3241
3242
3243
3244
3245
3246
3247
# define sqlite3FaultSim(X) SQLITE_OK
#else
  int sqlite3FaultSim(int);
#endif

Bitvec *sqlite3BitvecCreate(u32);
int sqlite3BitvecTest(Bitvec*, u32);

int sqlite3BitvecSet(Bitvec*, u32);
void sqlite3BitvecClear(Bitvec*, u32, void*);
void sqlite3BitvecDestroy(Bitvec*);
u32 sqlite3BitvecSize(Bitvec*);
#ifndef SQLITE_OMIT_BUILTIN_TEST
int sqlite3BitvecBuiltinTest(int,int*);
#endif







>







3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
# define sqlite3FaultSim(X) SQLITE_OK
#else
  int sqlite3FaultSim(int);
#endif

Bitvec *sqlite3BitvecCreate(u32);
int sqlite3BitvecTest(Bitvec*, u32);
int sqlite3BitvecTestNotNull(Bitvec*, u32);
int sqlite3BitvecSet(Bitvec*, u32);
void sqlite3BitvecClear(Bitvec*, u32, void*);
void sqlite3BitvecDestroy(Bitvec*);
u32 sqlite3BitvecSize(Bitvec*);
#ifndef SQLITE_OMIT_BUILTIN_TEST
int sqlite3BitvecBuiltinTest(int,int*);
#endif