SQLite: Check-in [181ceb32]

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview

Comment:	Fix some asserts and other things in the new WAL branch.
Downloads:	Tarball \| ZIP archive \| SQL archive
Timelines:	family \| ancestors \| descendants \| both \| wal-incr-ckpt
Files:	files \| file ages \| folders
SHA1:	181ceb32ead7f540a7c6437f53a5b0f3e78162db
User & Date:	dan 2010-05-31 11:16:31

References

2010-05-31
11:39		Fix a bug in checkpoint introduced by [181ceb32ea]. check-in: b499dbc8 user: dan tags: wal-incr-ckpt

Context

2010-05-31
11:39		Fix a bug in checkpoint introduced by [181ceb32ea]. check-in: b499dbc8 user: dan tags: wal-incr-ckpt
11:16		Fix some asserts and other things in the new WAL branch. check-in: 181ceb32 user: dan tags: wal-incr-ckpt
01:41		WAL runs but quickly deadlocks. check-in: ace58acb user: drh tags: wal-incr-ckpt

Changes

Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/wal.c.

  }

  /* This routine only runs while holding SQLITE_SHM_CHECKPOINT.  No other
  ** thread is able to write to shared memory while this routine is
  ** running (or, indeed, while the WalIterator object exists).  Hence,
  ** we can cast off the volatile qualifacation from shared memory
  */
  assert( pWal->ckptLock );
  aData = (u32*)pWal->pWiData;

  /* Allocate space for the WalIterator object */
  iLast = pWal->hdr.mxFrame;
  nSegment = (iLast >> 8) + 1;
  nFinal = (iLast & 0x000000FF);
  nByte = sizeof(WalIterator) + (nSegment+1)*(sizeof(struct WalSegment)+256);
................................................................................
  }

  /* Compute in mxSafeFrame the index of the last frame of the WAL that is
  ** safe to write into the database.  Frames beyond mxSafeFrame might
  ** overwrite database pages that are in use by active readers and thus
  ** cannot be backfilled from the WAL.
  */
  mxSafeFrame = 0;
  pHdr = (volatile WalIndexHdr*)pWal->pWiData;
  pInfo = (volatile WalCkptInfo*)&pHdr[2];
  assert( pInfo==walCkptInfo(pWal) );
  for(i=1; i<WAL_NREADER; i++){
    u32 y = pInfo->aReadMark[i];
    if( y>0 && (mxSafeFrame==0 || mxSafeFrame<y) ){
      if( y<pWal->hdr.mxFrame
       && (rc = walLockExclusive(pWal, WAL_READ_LOCK(i), 1))==SQLITE_OK
      ){
        pInfo->aReadMark[i] = 0;
        walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1);
      }else{
        mxSafeFrame = y;
      }
    }
  }

  if( pInfo->nBackfill<mxSafeFrame
   && (rc = walLockExclusive(pWal, WAL_READ_LOCK(0), 1))==SQLITE_OK
  ){
................................................................................
  ** time.
  */
  badHdr = walIndexTryHdr(pWal, pChanged);

  /* If the first attempt failed, it might have been due to a race
  ** with a writer.  So get a WRITE lock and try again.
  */
  assert( pWal->writeLock==0 );
  if( badHdr ){
    rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1);
    if( rc==SQLITE_OK ){
      pWal->writeLock = 1;
      badHdr = walIndexTryHdr(pWal, pChanged);
      if( badHdr ){
        /* If the wal-index header is still malformed even while holding
................................................................................
  if( mxI==0 ){
    /* If we get here, it means that all of the aReadMark[] entries between
    ** 1 and WAL_NREADER-1 are zero.  Try to initialize aReadMark[1] to
    ** be mxFrame, then retry.
    */
    rc = walLockExclusive(pWal, WAL_READ_LOCK(1), 1);
    if( rc==SQLITE_OK ){
      pInfo->aReadMark[1] = pWal->hdr.mxFrame;
      walUnlockExclusive(pWal, WAL_READ_LOCK(1), 1);
    }
    return WAL_RETRY;
  }else{
    if( mxReadMark < pWal->hdr.mxFrame ){
      for(i=0; i<WAL_NREADER; i++){
        rc = walLockExclusive(pWal, WAL_READ_LOCK(i), 1);
        if( rc==SQLITE_OK ){
          pInfo->aReadMark[i] = pWal->hdr.mxFrame;
          mxReadMark = pWal->hdr.mxFrame;
          mxI = i;
          walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1);
          break;
        }
      }
    }
................................................................................
    walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1);
    pWal->writeLock = 0;
    walIndexUnmap(pWal);
    return SQLITE_BUSY;
  }

  pInfo = walCkptInfo(pWal);

  if( pWal->readLock==0 && pInfo->nBackfill==pWal->hdr.mxFrame ){

    rc = walLockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1);
    if( rc==SQLITE_OK ){
      /* If all readers are using WAL_READ_LOCK(0) (in other words if no
      ** readers are currently using the WAL) */
      pWal->nCkpt++;
      pWal->hdr.mxFrame = 0;
      sqlite3Put4byte((u8*)pWal->hdr.aSalt,
                       1 + sqlite3Get4byte((u8*)pWal->hdr.aSalt));
      sqlite3_randomness(4, &pWal->hdr.aSalt[1]);
      walIndexWriteHdr(pWal);
      pInfo->nBackfill = 0;
      memset(&pInfo->aReadMark[1], 0, sizeof(pInfo->aReadMark)-sizeof(u32));
      rc = sqlite3OsTruncate(pWal->pDbFd, 
                             ((i64)pWal->hdr.nPage*(i64)pWal->szPage));
      walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1);

    }
    walUnlockShared(pWal, WAL_READ_LOCK(0));
    pWal->readLock = -1;
    do{
      int notUsed;
      rc = walTryBeginRead(pWal, &notUsed, 1);
    }while( rc==WAL_RETRY );
  }

  return rc;
}

/*
** End a write transaction.  The commit has already been done.  This
** routine merely releases the lock.
*/
int sqlite3WalEndWriteTransaction(Wal *pWal){
  walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1);

  return SQLITE_OK;
}

/*
** If any data has been written (but not committed) to the log file, this
** function moves the write-pointer back to the start of the transaction.
**
................................................................................
  int nBuf,                       /* Size of temporary buffer */
  u8 *zBuf                        /* Temporary buffer to use */
){
  int rc;                         /* Return code */
  int isChanged = 0;              /* True if a new wal-index header is loaded */

  assert( pWal->pWiData==0 );


  rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1);
  if( rc ){
    /* Usually this is SQLITE_BUSY meaning that another thread or process
    ** is already running a checkpoint, or maybe a recovery.  But it might
    ** also be SQLITE_IOERR. */
    return rc;
  }


  /* Copy data from the log to the database file. */
  rc = walIndexReadHdr(pWal, &isChanged);
  if( rc==SQLITE_OK ){
    rc = walCheckpoint(pWal, sync_flags, nBuf, zBuf);
  }
  if( isChanged ){
................................................................................
    */
    memset(&pWal->hdr, 0, sizeof(WalIndexHdr));
  }

  /* Release the locks. */
  walIndexUnmap(pWal);
  walUnlockExclusive(pWal, WAL_CKPT_LOCK, 1);

  return rc;
}

/* Return the value to pass to a sqlite3_wal_hook callback, the
** number of frames in the WAL at the point of the last commit since
** sqlite3WalCallback() was called.  If no commits have occurred since
** the last call, then return 0.

  }

  /* This routine only runs while holding SQLITE_SHM_CHECKPOINT.  No other
  ** thread is able to write to shared memory while this routine is
  ** running (or, indeed, while the WalIterator object exists).  Hence,
  ** we can cast off the volatile qualifacation from shared memory
  */
  assert( pWal->ckptLock || pWal->exclusiveMode );
  aData = (u32*)pWal->pWiData;

  /* Allocate space for the WalIterator object */
  iLast = pWal->hdr.mxFrame;
  nSegment = (iLast >> 8) + 1;
  nFinal = (iLast & 0x000000FF);
  nByte = sizeof(WalIterator) + (nSegment+1)*(sizeof(struct WalSegment)+256);
................................................................................
  }

  /* Compute in mxSafeFrame the index of the last frame of the WAL that is
  ** safe to write into the database.  Frames beyond mxSafeFrame might
  ** overwrite database pages that are in use by active readers and thus
  ** cannot be backfilled from the WAL.
  */
  mxSafeFrame = pWal->hdr.mxFrame;
  pHdr = (volatile WalIndexHdr*)pWal->pWiData;
  pInfo = (volatile WalCkptInfo*)&pHdr[2];
  assert( pInfo==walCkptInfo(pWal) );
  for(i=1; i<WAL_NREADER; i++){
    u32 y = pInfo->aReadMark[i];
    if( y>0 && (mxSafeFrame==0 || mxSafeFrame>=y) ){
      if( y<pWal->hdr.mxFrame
       && (rc = walLockExclusive(pWal, WAL_READ_LOCK(i), 1))==SQLITE_OK
      ){
        pInfo->aReadMark[i] = 0;
        walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1);
      }else{
        mxSafeFrame = y-1;
      }
    }
  }

  if( pInfo->nBackfill<mxSafeFrame
   && (rc = walLockExclusive(pWal, WAL_READ_LOCK(0), 1))==SQLITE_OK
  ){
................................................................................
  ** time.
  */
  badHdr = walIndexTryHdr(pWal, pChanged);

  /* If the first attempt failed, it might have been due to a race
  ** with a writer.  So get a WRITE lock and try again.
  */
  assert( badHdr==0 || pWal->writeLock==0 );
  if( badHdr ){
    rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1);
    if( rc==SQLITE_OK ){
      pWal->writeLock = 1;
      badHdr = walIndexTryHdr(pWal, pChanged);
      if( badHdr ){
        /* If the wal-index header is still malformed even while holding
................................................................................
  if( mxI==0 ){
    /* If we get here, it means that all of the aReadMark[] entries between
    ** 1 and WAL_NREADER-1 are zero.  Try to initialize aReadMark[1] to
    ** be mxFrame, then retry.
    */
    rc = walLockExclusive(pWal, WAL_READ_LOCK(1), 1);
    if( rc==SQLITE_OK ){
      pInfo->aReadMark[1] = pWal->hdr.mxFrame+1;
      walUnlockExclusive(pWal, WAL_READ_LOCK(1), 1);
    }
    return WAL_RETRY;
  }else{
    if( mxReadMark < pWal->hdr.mxFrame ){
      for(i=1; i<WAL_NREADER; i++){
        rc = walLockExclusive(pWal, WAL_READ_LOCK(i), 1);
        if( rc==SQLITE_OK ){
          pInfo->aReadMark[i] = pWal->hdr.mxFrame+1;
          mxReadMark = pWal->hdr.mxFrame;
          mxI = i;
          walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1);
          break;
        }
      }
    }
................................................................................
    walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1);
    pWal->writeLock = 0;
    walIndexUnmap(pWal);
    return SQLITE_BUSY;
  }

  pInfo = walCkptInfo(pWal);
  if( pWal->readLock==0 ){
    assert( pInfo->nBackfill==pWal->hdr.mxFrame );
    if( pInfo->nBackfill>0 ){
      rc = walLockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1);
      if( rc==SQLITE_OK ){
        /* If all readers are using WAL_READ_LOCK(0) (in other words if no
        ** readers are currently using the WAL) */
        pWal->nCkpt++;
        pWal->hdr.mxFrame = 0;
        sqlite3Put4byte((u8*)pWal->hdr.aSalt,
                         1 + sqlite3Get4byte((u8*)pWal->hdr.aSalt));
        sqlite3_randomness(4, &pWal->hdr.aSalt[1]);
        walIndexWriteHdr(pWal);
        pInfo->nBackfill = 0;
        memset(&pInfo->aReadMark[1], 0, sizeof(pInfo->aReadMark)-sizeof(u32));
        rc = sqlite3OsTruncate(pWal->pDbFd, 
                               ((i64)pWal->hdr.nPage*(i64)pWal->szPage));
        walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1);
      }
    }
    walUnlockShared(pWal, WAL_READ_LOCK(0));
    pWal->readLock = -1;
    do{
      int notUsed;
      rc = walTryBeginRead(pWal, &notUsed, 1);
    }while( rc==WAL_RETRY );
  }
  walIndexUnmap(pWal);
  return rc;
}

/*
** End a write transaction.  The commit has already been done.  This
** routine merely releases the lock.
*/
int sqlite3WalEndWriteTransaction(Wal *pWal){
  walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1);
  pWal->writeLock = 0;
  return SQLITE_OK;
}

/*
** If any data has been written (but not committed) to the log file, this
** function moves the write-pointer back to the start of the transaction.
**
................................................................................
  int nBuf,                       /* Size of temporary buffer */
  u8 *zBuf                        /* Temporary buffer to use */
){
  int rc;                         /* Return code */
  int isChanged = 0;              /* True if a new wal-index header is loaded */

  assert( pWal->pWiData==0 );
  assert( pWal->ckptLock==0 );

  rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1);
  if( rc ){
    /* Usually this is SQLITE_BUSY meaning that another thread or process
    ** is already running a checkpoint, or maybe a recovery.  But it might
    ** also be SQLITE_IOERR. */
    return rc;
  }
  pWal->ckptLock = 1;

  /* Copy data from the log to the database file. */
  rc = walIndexReadHdr(pWal, &isChanged);
  if( rc==SQLITE_OK ){
    rc = walCheckpoint(pWal, sync_flags, nBuf, zBuf);
  }
  if( isChanged ){
................................................................................
    */
    memset(&pWal->hdr, 0, sizeof(WalIndexHdr));
  }

  /* Release the locks. */
  walIndexUnmap(pWal);
  walUnlockExclusive(pWal, WAL_CKPT_LOCK, 1);
  pWal->ckptLock = 0;
  return rc;
}

/* Return the value to pass to a sqlite3_wal_hook callback, the
** number of frames in the WAL at the point of the last commit since
** sqlite3WalCallback() was called.  If no commits have occurred since
** the last call, then return 0.

Changes to test/walcrash2.test.

          INSERT INTO t1 SELECT * FROM t1 LIMIT 3;        -- 20 rows, 20 pages
      }
    } 
    close $C
    file size test.db-wal
  } [wal_file_size 16 1024]
}
do_test walcrash2-1.3 {
  sqlite3 db2 test.db













  execsql { SELECT count(*) FROM t1 } db2
} {0}
catch { db2 close }

finish_test








<
|
>
>
>
>
>
>
>
>
>
>
>
>
>

          INSERT INTO t1 SELECT * FROM t1 LIMIT 3;        -- 20 rows, 20 pages
      }
    } 
    close $C
    file size test.db-wal
  } [wal_file_size 16 1024]
}

sqlite3 db2 test.db
breakpoint
      db2 eval {
        PRAGMA cache_size = 15;
        BEGIN;
          INSERT INTO t1 VALUES(randomblob(900));         --  1 row,  1  page
          INSERT INTO t1 SELECT * FROM t1;                --  2 rows, 3  pages
          INSERT INTO t1 SELECT * FROM t1;                --  4 rows, 5  pages
          INSERT INTO t1 SELECT * FROM t1;                --  8 rows, 9  pages
          INSERT INTO t1 SELECT * FROM t1;                -- 16 rows, 17 pages
          INSERT INTO t1 SELECT * FROM t1 LIMIT 3;        -- 20 rows, 20 pages
      }

do_test walcrash2-1.3 {
  execsql { SELECT count(*) FROM t1 } db2
} {0}
catch { db2 close }

finish_test