SQLite4: Check-in [4b36a0245f]

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview

Comment:	Fix some problems with circular logs.
Downloads:	Tarball \| ZIP archive \| SQL archive
Timelines:	family \| ancestors \| descendants \| both \| trunk
Files:	files \| file ages \| folders
SHA1:	4b36a0245fda855f497b4ba471c110882ca04304
User & Date:	dan 2013-10-28 10:47:26

Context

2013-10-28
10:56		Fix a problem causing builds without LSM_NO_SYNC to fail. check-in: f8ce90c4a6 user: dan tags: trunk
10:47		Fix some problems with circular logs. check-in: 4b36a0245f user: dan tags: trunk
2013-10-26
19:58		Begin adding code to make the log file circular. check-in: ec6dda0ade user: dan tags: trunk

Changes

Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/btInt.h.


int sqlite4BtLogSnapshotOpen(BtLog*);
int sqlite4BtLogSnapshotClose(BtLog*);
int sqlite4BtLogSnapshotWritable(BtLog*);

int sqlite4BtLogSize(BtLog*);
int sqlite4BtLogCheckpoint(BtLog*);







/*
** End of bt_log.c interface.
*************************************************************************/

/*************************************************************************
** Interface to bt_lock.c functionality.








>
>
>
>
>
>


int sqlite4BtLogSnapshotOpen(BtLog*);
int sqlite4BtLogSnapshotClose(BtLog*);
int sqlite4BtLogSnapshotWritable(BtLog*);

int sqlite4BtLogSize(BtLog*);
int sqlite4BtLogCheckpoint(BtLog*);

#ifndef NDEBUG
void sqlite4BtDebugReadPage(u32 pgno, u8 *aData, int pgsz);
#else
# define sqlite4BtDebugReadPage(a,b,c)
#endif

/*
** End of bt_log.c interface.
*************************************************************************/

/*************************************************************************
** Interface to bt_lock.c functionality.

Changes to src/bt_log.c.

** of the input value as a little-endian integer.
*/
#define BYTESWAP32(x) ( \
  (((x)&0x000000FF)<<24) + (((x)&0x0000FF00)<<8)  \
  + (((x)&0x00FF0000)>>8)  + (((x)&0xFF000000)>>24) \
)

static void btLogDebugTopology(char *zStr, u32 *aLog){
  fprintf(stderr, "%s: %d..%d  %d..%d  %d..%d\n", zStr,
      (int)aLog[0], (int)aLog[1], (int)aLog[2], 
      (int)aLog[3], (int)aLog[4], (int)aLog[5]
  );
  fflush(stderr);
}

/*
** Generate or extend an 8 byte checksum based on the data in
** array aByte[] and the initial values of aIn[0] and aIn[1] (or
** initial values of 0 and 0 if aIn==NULL).
**
** The checksum is written back into aOut[] before returning.
**
................................................................................
    }while( aData<aEnd );
  }

  aOut[0] = s1;
  aOut[1] = s2;
}














































/*
** Ensure that shared-memory chunk iChunk is mapped and available in
** the BtLog.apShm[] array. If an error occurs, return an SQLite4 error
** code. Otherwise, SQLITE4_OK.
*/
static int btLogMapShm(BtLog *pLog, int iChunk){
  int rc = SQLITE4_OK;
................................................................................
  return rc;
}

static int btLogHashSearch(
  BtLog *pLog,                    /* Log module handle */
  int iSide,                      /* 0 or 1 - the side of hash table to read */
  int iHash,                      /* Index of hash to query */

  u32 pgno,                       /* query for this page number */
  u32 *piFrame                    /* OUT: Frame number for matching entry */
){
  ht_slot *aHash;
  u32 *aPgno;
  u32 iZero;
  int rc;
................................................................................
    int nCollide = HASHTABLE_NSLOT*2;
    int iSlot;
    u32 iFrame = 0;
    
    iSlot=btLogHashKey(pLog, pgno); 
    for( ; aHash[iSlot]; iSlot=btLogHashNext(pLog, iSlot)){
      if( aPgno[aHash[iSlot]-1]==pgno ){
        iFrame = iZero + aHash[iSlot] - 1;

      }
      if( (nCollide--)==0 ) return btErrorBkpt(SQLITE4_CORRUPT);
    }

    *piFrame = iFrame;
    if( iFrame==0 ){
      rc = SQLITE4_NOTFOUND;
................................................................................
int sqlite4BtLogRead(BtLog *pLog, u32 pgno, u8 *aData){
  const int pgsz = sqlite4BtPagerPagesize((BtPager*)(pLog->pLock));
  int rc = SQLITE4_NOTFOUND;
  u32 iFrame = 0;
  int i;

  /* Loop through regions (c), (b) and (a) of the log file. In that order. */
  for(i=2; i>=0; i--){
    u32 iLo = pLog->snapshot.aLog[i*2+0];
    u32 iHi = pLog->snapshot.aLog[i*2+1];
    int iSide;
    int iHash;
    int iHashLast;

    iHash = btLogFrameHash(pLog, iHi);
    iHashLast = btLogFrameHash(pLog, iLo);
    iSide = (pLog->snapshot.iHashSide + (i==0)) % 2;

    for( ; rc==SQLITE4_NOTFOUND && iHash>=iHashLast && iFrame==0; iHash--){
      rc = btLogHashSearch(pLog, iSide, iHash, pgno, &iFrame);
      if( rc==SQLITE4_OK && (iFrame<iLo || iFrame>iHi) ){
        rc = SQLITE4_NOTFOUND;
      }
    }
  }

  if( rc==SQLITE4_OK && iFrame!=0 ){
    bt_env *pVfs = pLog->pLock->pVfs;
    i64 iOff;
    assert( rc==SQLITE4_OK );
    iOff = btLogFrameOffset(pLog, pgsz, iFrame);
    rc = pVfs->xRead(pLog->pFd, iOff + sizeof(BtFrameHdr), aData, pgsz);

#if 0
................................................................................
    memset(&frame, 0, sizeof(frame));
    frame.pgno = pgno;
    frame.ctrl = (bCommit ? BT_FRAME_COMMIT : 0) + iNextFrame;
    a = pLog->snapshot.aFrameCksum;
    btLogChecksum(1, (u8*)&frame, offsetof(BtFrameHdr,aCksum), a, frame.aCksum);
    btLogChecksum(1, aData, pgsz, frame.aCksum, frame.aCksum);

#if 0
    fprintf(stderr, "writing page %d at log offset %d (frame %d)\n", (int)pgno, (int)iOff, (int)iFrame);
    fflush(stderr);
#endif

    /* Write the frame header to the log file. */
    rc = btLogWriteData(pLog, iOff, (u8*)&frame, sizeof(frame));
  }
  pLog->snapshot.iNextFrame = iNextFrame;

  /* Write the frame contents to the log file. */
................................................................................
      pLog->snapshot.iHashSide = (pLog->snapshot.iHashSide+1) %2;
    }
    rc = btLogHashInsert(pLog, pgno, iFrame);
  }

  /* Update the private copy of the shm-header */
  if( rc==SQLITE4_OK ){
btLogDebugTopology("log1", aLog);
    if( btLogIsEmpty(pLog) ){
      assert( iFrame==1 );
      aLog[4] = iFrame;
    }else if( iFrame==1 ){
      assert( aLog[0]==0 && aLog[1]==0 && aLog[2]==0 && aLog[3]==0 );
      aLog[0] = aLog[4];
      aLog[1] = aLog[5];
................................................................................
      aLog[2] = aLog[4];
      aLog[3] = aLog[5];
      aLog[4] = iFrame;
    }

    aLog[5] = iFrame;
    memcpy(pLog->snapshot.aFrameCksum, frame.aCksum, sizeof(frame.aCksum));
btLogDebugTopology("log2", aLog);
  }

  /* If this is a COMMIT, also update the shared shm-header. */
  if( bCommit ){
    rc = btLogUpdateSharedHdr(pLog);
  }

................................................................................

  if( rc==SQLITE4_OK ){
    int iRegion;
    u32 *aLog = pLog->snapshot.aLog;
    BtShm *pShm = btLogShm(pLog);
    u32 iCkpt = pShm->ckpt.iFirstRecover;

fprintf(stderr, "iCkpt=%d\n", (int)iCkpt);
btLogDebugTopology("log3", aLog);

    for(iRegion=0; iRegion<3; iRegion++){
      if( aLog[iRegion*2] ){
        if( iCkpt>=aLog[iRegion*2] && iCkpt<=aLog[iRegion*2+1] ){
          aLog[iRegion*2] = iCkpt;
          break;
        }else{
          aLog[iRegion*2] = 0;
          aLog[iRegion*2+1] = 0;
        }
      }
    }

btLogDebugTopology("log4", aLog);
  }

  return rc;
}

int sqlite4BtLogSnapshotClose(BtLog *pLog){
  sqlite4BtLockReaderUnlock(pLog->pLock);
................................................................................
  u32 **paPgno,                   /* OUT: s4_malloc'd array of sorted pgno */
  int *pnPgno,                    /* OUT: Number of entries in *paPgno */
  u32 *piLastFrame
){
  u32 *aLog = pLog->snapshot.aLog;/* Log file topology */
  u32 i;
  u32 *aPgno;                     /* Returned array */

  u32 *aSpace;                    /* Temporary space used by merge-sort */
  int nMax;
  int rc = SQLITE4_OK;
  int iRegion;

  /* Determine an upper limit on the number of distinct page numbers. This
  ** limit is used to allocate space for the returned array.  */
................................................................................
    }
  }

  /* Allocate space to collect all page numbers. */
  aPgno = (u32*)sqlite4_malloc(pLog->pLock->pEnv, sizeof(u32)*nMax*2);
  if( aPgno==0 ) rc = btErrorBkpt(SQLITE4_NOMEM);
  aSpace = &aPgno[nMax];


  /* Copy the required page numbers into the allocated array */
  for(iRegion=0; iRegion<3; iRegion++){
    u32 iFirst = aLog[iRegion*2];
    u32 iLast = aLog[iRegion*2+1];
    if( iFirst ){
      for(i=iFirst; rc==SQLITE4_OK && i<=iLast; i++){
................................................................................

        /* It doesn't matter which 'side' of the hash table is requested here,
        ** as only the page-number array, not the aHash[] table, will be used.
        ** And it is the same for both sides. Hence the constant 0 passed as
        ** the second argument to btLogFindHash().  */
        rc = btLogFindHash(pLog, 0, iHash, &aHash, &aPage, &iZero);
        if( rc==SQLITE4_OK ){
          aPgno[i-iFirst] = aPage[i-iZero];
        }
      }
      *piLastFrame = iLast;
    }
  }

  /* Sort the contents of the array in ascending order. This step also 
  ** eliminates any  duplicate page numbers. */
  if( rc==SQLITE4_OK ){
    btLogMergeSort(aPgno, &nMax, aSpace);
    *pnPgno = nMax;
    *paPgno = aPgno;
  }else{
    sqlite4_free(pLog->pLock->pEnv, aPgno);
    *paPgno = 0;
    *pnPgno = 0;
  }

................................................................................
    }

    /* Copy data from the log file to the database file. */
    for(i=0; rc==SQLITE4_OK && i<nPgno; i++){
      u32 pgno = aPgno[i];
      rc = sqlite4BtLogRead(pLog, pgno, aBuf);
      if( rc==SQLITE4_OK ){

        i64 iOff = (i64)pgsz * (pgno-1);
        rc = pVfs->xWrite(pFd, iOff, aBuf, pgsz);
      }
    }

    /* Update the first field of the checkpoint-header. This tells readers
    ** that they need not consider anything that in the log before this

** of the input value as a little-endian integer.
*/
#define BYTESWAP32(x) ( \
  (((x)&0x000000FF)<<24) + (((x)&0x0000FF00)<<8)  \
  + (((x)&0x00FF0000)>>8)  + (((x)&0xFF000000)>>24) \
)









/*
** Generate or extend an 8 byte checksum based on the data in
** array aByte[] and the initial values of aIn[0] and aIn[1] (or
** initial values of 0 and 0 if aIn==NULL).
**
** The checksum is written back into aOut[] before returning.
**
................................................................................
    }while( aData<aEnd );
  }

  aOut[0] = s1;
  aOut[1] = s2;
}


static void btDebugTopology(char *zStr, u32 *aLog){
  fprintf(stderr, "%s: %d..%d  %d..%d  %d..%d\n", zStr,
      (int)aLog[0], (int)aLog[1], (int)aLog[2], 
      (int)aLog[3], (int)aLog[4], (int)aLog[5]
  );
  fflush(stderr);
}

static void btDebugCkptPage(u32 pgno, u8 *aData, int pgsz){
#if 0
  static nCall = 0;
  u32 aCksum[2];
  btLogChecksum(1, aData, pgsz, 0, aCksum);
  fprintf(stderr, "%d: Ckpt page %d (cksum=%08x%08x)\n", nCall++,
      (int)pgno, aCksum[0], aCksum[1]
  );
  fflush(stderr);
#endif
}

static void btDebugLogPage(u32 pgno, u32 iFrame, u8 *aData, int pgsz){
#if 0
  static nCall = 0;
  u32 aCksum[2];
  btLogChecksum(1, aData, pgsz, 0, aCksum);
  fprintf(stderr, "%d: Log page %d to frame %d (cksum=%08x%08x)\n", nCall++,
      (int)pgno, (int)iFrame, aCksum[0], aCksum[1]
  );
  fflush(stderr);
#endif
}

void sqlite4BtDebugReadPage(u32 pgno, u8 *aData, int pgsz){
#if 0
  static nCall = 0;
  u32 aCksum[2];
  btLogChecksum(1, aData, pgsz, 0, aCksum);
  fprintf(stderr, "%d: Read page %d (cksum=%08x%08x)\n", nCall++,
      (int)pgno, aCksum[0], aCksum[1]
  );
  fflush(stderr);
#endif
}

/*
** Ensure that shared-memory chunk iChunk is mapped and available in
** the BtLog.apShm[] array. If an error occurs, return an SQLite4 error
** code. Otherwise, SQLITE4_OK.
*/
static int btLogMapShm(BtLog *pLog, int iChunk){
  int rc = SQLITE4_OK;
................................................................................
  return rc;
}

static int btLogHashSearch(
  BtLog *pLog,                    /* Log module handle */
  int iSide,                      /* 0 or 1 - the side of hash table to read */
  int iHash,                      /* Index of hash to query */
  u32 iHi,                        /* Consider no frames after this one */
  u32 pgno,                       /* query for this page number */
  u32 *piFrame                    /* OUT: Frame number for matching entry */
){
  ht_slot *aHash;
  u32 *aPgno;
  u32 iZero;
  int rc;
................................................................................
    int nCollide = HASHTABLE_NSLOT*2;
    int iSlot;
    u32 iFrame = 0;
    
    iSlot=btLogHashKey(pLog, pgno); 
    for( ; aHash[iSlot]; iSlot=btLogHashNext(pLog, iSlot)){
      if( aPgno[aHash[iSlot]-1]==pgno ){
        u32 iCandidate = iZero + aHash[iSlot] - 1;
        if( iCandidate<=iHi ) iFrame = iCandidate;
      }
      if( (nCollide--)==0 ) return btErrorBkpt(SQLITE4_CORRUPT);
    }

    *piFrame = iFrame;
    if( iFrame==0 ){
      rc = SQLITE4_NOTFOUND;
................................................................................
int sqlite4BtLogRead(BtLog *pLog, u32 pgno, u8 *aData){
  const int pgsz = sqlite4BtPagerPagesize((BtPager*)(pLog->pLock));
  int rc = SQLITE4_NOTFOUND;
  u32 iFrame = 0;
  int i;

  /* Loop through regions (c), (b) and (a) of the log file. In that order. */
  for(i=2; i>=0 && rc==SQLITE4_NOTFOUND; i--){
    u32 iLo = pLog->snapshot.aLog[i*2+0];
    u32 iHi = pLog->snapshot.aLog[i*2+1];
    int iSide;
    int iHash;
    int iHashLast;

    iHash = btLogFrameHash(pLog, iHi);
    iHashLast = btLogFrameHash(pLog, iLo);
    iSide = (pLog->snapshot.iHashSide + (i==0)) % 2;

    for( ; rc==SQLITE4_NOTFOUND && iHash>=iHashLast; iHash--){
      rc = btLogHashSearch(pLog, iSide, iHash, iHi, pgno, &iFrame);
      if( rc==SQLITE4_OK && (iFrame<iLo || iFrame>iHi) ){
        rc = SQLITE4_NOTFOUND;
      }
    }
  }

  if( rc==SQLITE4_OK ){
    bt_env *pVfs = pLog->pLock->pVfs;
    i64 iOff;
    assert( rc==SQLITE4_OK );
    iOff = btLogFrameOffset(pLog, pgsz, iFrame);
    rc = pVfs->xRead(pLog->pFd, iOff + sizeof(BtFrameHdr), aData, pgsz);

#if 0
................................................................................
    memset(&frame, 0, sizeof(frame));
    frame.pgno = pgno;
    frame.ctrl = (bCommit ? BT_FRAME_COMMIT : 0) + iNextFrame;
    a = pLog->snapshot.aFrameCksum;
    btLogChecksum(1, (u8*)&frame, offsetof(BtFrameHdr,aCksum), a, frame.aCksum);
    btLogChecksum(1, aData, pgsz, frame.aCksum, frame.aCksum);

    btDebugLogPage(pgno, iFrame, aData, pgsz);




    /* Write the frame header to the log file. */
    rc = btLogWriteData(pLog, iOff, (u8*)&frame, sizeof(frame));
  }
  pLog->snapshot.iNextFrame = iNextFrame;

  /* Write the frame contents to the log file. */
................................................................................
      pLog->snapshot.iHashSide = (pLog->snapshot.iHashSide+1) %2;
    }
    rc = btLogHashInsert(pLog, pgno, iFrame);
  }

  /* Update the private copy of the shm-header */
  if( rc==SQLITE4_OK ){

    if( btLogIsEmpty(pLog) ){
      assert( iFrame==1 );
      aLog[4] = iFrame;
    }else if( iFrame==1 ){
      assert( aLog[0]==0 && aLog[1]==0 && aLog[2]==0 && aLog[3]==0 );
      aLog[0] = aLog[4];
      aLog[1] = aLog[5];
................................................................................
      aLog[2] = aLog[4];
      aLog[3] = aLog[5];
      aLog[4] = iFrame;
    }

    aLog[5] = iFrame;
    memcpy(pLog->snapshot.aFrameCksum, frame.aCksum, sizeof(frame.aCksum));

  }

  /* If this is a COMMIT, also update the shared shm-header. */
  if( bCommit ){
    rc = btLogUpdateSharedHdr(pLog);
  }

................................................................................

  if( rc==SQLITE4_OK ){
    int iRegion;
    u32 *aLog = pLog->snapshot.aLog;
    BtShm *pShm = btLogShm(pLog);
    u32 iCkpt = pShm->ckpt.iFirstRecover;




    for(iRegion=0; iRegion<3; iRegion++){
      if( aLog[iRegion*2] ){
        if( iCkpt>=aLog[iRegion*2] && iCkpt<=aLog[iRegion*2+1] ){
          aLog[iRegion*2] = iCkpt;
          break;
        }else{
          aLog[iRegion*2] = 0;
          aLog[iRegion*2+1] = 0;
        }
      }
    }


  }

  return rc;
}

int sqlite4BtLogSnapshotClose(BtLog *pLog){
  sqlite4BtLockReaderUnlock(pLog->pLock);
................................................................................
  u32 **paPgno,                   /* OUT: s4_malloc'd array of sorted pgno */
  int *pnPgno,                    /* OUT: Number of entries in *paPgno */
  u32 *piLastFrame
){
  u32 *aLog = pLog->snapshot.aLog;/* Log file topology */
  u32 i;
  u32 *aPgno;                     /* Returned array */
  int nPgno;                      /* Elements in aPgno[] */
  u32 *aSpace;                    /* Temporary space used by merge-sort */
  int nMax;
  int rc = SQLITE4_OK;
  int iRegion;

  /* Determine an upper limit on the number of distinct page numbers. This
  ** limit is used to allocate space for the returned array.  */
................................................................................
    }
  }

  /* Allocate space to collect all page numbers. */
  aPgno = (u32*)sqlite4_malloc(pLog->pLock->pEnv, sizeof(u32)*nMax*2);
  if( aPgno==0 ) rc = btErrorBkpt(SQLITE4_NOMEM);
  aSpace = &aPgno[nMax];
  nPgno = 0;

  /* Copy the required page numbers into the allocated array */
  for(iRegion=0; iRegion<3; iRegion++){
    u32 iFirst = aLog[iRegion*2];
    u32 iLast = aLog[iRegion*2+1];
    if( iFirst ){
      for(i=iFirst; rc==SQLITE4_OK && i<=iLast; i++){
................................................................................

        /* It doesn't matter which 'side' of the hash table is requested here,
        ** as only the page-number array, not the aHash[] table, will be used.
        ** And it is the same for both sides. Hence the constant 0 passed as
        ** the second argument to btLogFindHash().  */
        rc = btLogFindHash(pLog, 0, iHash, &aHash, &aPage, &iZero);
        if( rc==SQLITE4_OK ){
          aPgno[nPgno++] = aPage[i-iZero];
        }
      }
      *piLastFrame = iLast;
    }
  }

  /* Sort the contents of the array in ascending order. This step also 
  ** eliminates any  duplicate page numbers. */
  if( rc==SQLITE4_OK ){
    btLogMergeSort(aPgno, &nPgno, aSpace);
    *pnPgno = nPgno;
    *paPgno = aPgno;
  }else{
    sqlite4_free(pLog->pLock->pEnv, aPgno);
    *paPgno = 0;
    *pnPgno = 0;
  }

................................................................................
    }

    /* Copy data from the log file to the database file. */
    for(i=0; rc==SQLITE4_OK && i<nPgno; i++){
      u32 pgno = aPgno[i];
      rc = sqlite4BtLogRead(pLog, pgno, aBuf);
      if( rc==SQLITE4_OK ){
        btDebugCkptPage(pgno, aBuf, pgsz);
        i64 iOff = (i64)pgsz * (pgno-1);
        rc = pVfs->xWrite(pFd, iOff, aBuf, pgsz);
      }
    }

    /* Update the first field of the checkpoint-header. This tells readers
    ** that they need not consider anything that in the log before this

Changes to src/bt_pager.c.

/*
** 2013 September 14
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
................................................................................
/*
** Close a pager database handle.
*/
int sqlite4BtPagerClose(BtPager *p){
  int i;
  int rc;

  rc = sqlite4BtLockDisconnect(p, btCheckpoint, btCleanup);

  if( p->btl.pFd ){
    p->btl.pVfs->xClose(p->btl.pFd);
  }

  btPurgeCache(p);
  sqlite4BtLogClose(p->pLog, 0);
................................................................................
        rc = btHashAdd(p, pRet);
      }
      if( rc!=SQLITE4_OK ){
        btFreePage(p, pRet);
        pRet = 0;
      }
    }

  }

  assert( (pRet!=0)==(rc==SQLITE4_OK) );
  if( rc==SQLITE4_OK ){
    p->nTotalRef++;
    pRet->nRef++;
  }

/*;
** 2013 September 14
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
................................................................................
/*
** Close a pager database handle.
*/
int sqlite4BtPagerClose(BtPager *p){
  int i;
  int rc;

  rc = sqlite4BtLockDisconnect((BtLock*)p, btCheckpoint, btCleanup);

  if( p->btl.pFd ){
    p->btl.pVfs->xClose(p->btl.pFd);
  }

  btPurgeCache(p);
  sqlite4BtLogClose(p->pLog, 0);
................................................................................
        rc = btHashAdd(p, pRet);
      }
      if( rc!=SQLITE4_OK ){
        btFreePage(p, pRet);
        pRet = 0;
      }
    }
    sqlite4BtDebugReadPage(pgno, pRet->aData, p->dbhdr.pgsz);
  }

  assert( (pRet!=0)==(rc==SQLITE4_OK) );
  if( rc==SQLITE4_OK ){
    p->nTotalRef++;
    pRet->nRef++;
  }