SQLite

Check-in [28393c413c]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Remove the requirement to open the wal file before sqlite3_snapshot_recover() is called. Also add some comments to new functions.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | serializable-snapshot
Files: files | file ages | folders
SHA1: 28393c413cc4505b94411730e728583c5d4baaae
User & Date: dan 2016-11-19 18:31:37.017
Context
2016-11-22
21:11
Enclose the sqlite3WalSnapshotRecover() routine within (check-in: e7be3183eb user: drh tags: serializable-snapshot)
2016-11-19
18:31
Remove the requirement to open the wal file before sqlite3_snapshot_recover() is called. Also add some comments to new functions. (check-in: 28393c413c user: dan tags: serializable-snapshot)
17:30
Test some extra error conditions in sqlite3_recover_snapshot(). (check-in: db314213c0 user: dan tags: serializable-snapshot)
Changes
Unified Diff Ignore Whitespace Patch
Changes to src/main.c.
4060
4061
4062
4063
4064
4065
4066


4067


4068
4069
4070
4071
4072
4073
4074
#endif

  sqlite3_mutex_enter(db->mutex);
  iDb = sqlite3FindDbName(db, zDb);
  if( iDb==0 || iDb>1 ){
    Btree *pBt = db->aDb[iDb].pBt;
    if( 0==sqlite3BtreeIsInReadTrans(pBt) ){


      rc = sqlite3PagerSnapshotRecover(sqlite3BtreePager(pBt));


    }
  }
  sqlite3_mutex_leave(db->mutex);
#endif   /* SQLITE_OMIT_WAL */
  return rc;
}








>
>
|
>
>







4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
#endif

  sqlite3_mutex_enter(db->mutex);
  iDb = sqlite3FindDbName(db, zDb);
  if( iDb==0 || iDb>1 ){
    Btree *pBt = db->aDb[iDb].pBt;
    if( 0==sqlite3BtreeIsInReadTrans(pBt) ){
      rc = sqlite3BtreeBeginTrans(pBt, 0);
      if( rc==SQLITE_OK ){
        rc = sqlite3PagerSnapshotRecover(sqlite3BtreePager(pBt));
        sqlite3BtreeCommit(pBt);
      }
    }
  }
  sqlite3_mutex_leave(db->mutex);
#endif   /* SQLITE_OMIT_WAL */
  return rc;
}

Changes to src/sqlite.h.in.
8413
8414
8415
8416
8417
8418
8419
















8420
8421
8422
8423
8424
8425
8426
  sqlite3_snapshot *p1,
  sqlite3_snapshot *p2
);

/*
** CAPI3REF: Recover snapshots from a wal file
** EXPERIMENTAL
















*/
SQLITE_EXPERIMENTAL int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb);

/*
** Undo the hack that converts floating point types to integer for
** builds on processors without floating point support.
*/







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







8413
8414
8415
8416
8417
8418
8419
8420
8421
8422
8423
8424
8425
8426
8427
8428
8429
8430
8431
8432
8433
8434
8435
8436
8437
8438
8439
8440
8441
8442
  sqlite3_snapshot *p1,
  sqlite3_snapshot *p2
);

/*
** CAPI3REF: Recover snapshots from a wal file
** EXPERIMENTAL
**
** If all connections disconnect from a database file but do not perform
** a checkpoint, the existing wal file is opened along with the database
** file the next time the database is opened. At this point it is only
** possible to successfully call sqlite3_snapshot_open() to open the most
** recent snapshot of the database (the one at the head of the wal file),
** even though the wal file may contain other valid snapshots for which
** clients have sqlite3_snapshot handles.
**
** This function attempts to scan the wal file associated with database zDb
** of database handle db and make all valid snapshots available to
** sqlite3_snapshot_open(). It is an error if there is already a read
** transaction open on the database, or if the database is not a wal mode
** database.
**
** SQLITE_OK is returned if successful, or an SQLite error code otherwise.
*/
SQLITE_EXPERIMENTAL int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb);

/*
** Undo the hack that converts floating point types to integer for
** builds on processors without floating point support.
*/
Changes to src/wal.c.
2376
2377
2378
2379
2380
2381
2382

2383















2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
    assert( mxReadMark<=pWal->hdr.mxFrame );
    pWal->readLock = (i16)mxI;
  }
  return rc;
}

/*

** Recover as many snapshots as possible from the wal file.















*/
int sqlite3WalSnapshotRecover(Wal *pWal){
  int dummy;
  int rc;

  rc = sqlite3WalBeginReadTransaction(pWal, &dummy);
  if( rc==SQLITE_OK ){
    rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1);
    if( rc==SQLITE_OK ){
      volatile WalCkptInfo *pInfo = walCkptInfo(pWal);
      int szPage = (int)pWal->szPage;
      i64 szDb;                   /* Size of db file in bytes */

      rc = sqlite3OsFileSize(pWal->pDbFd, &szDb);
      if( rc==SQLITE_OK ){
        void *pBuf1 = sqlite3_malloc(szPage);
        void *pBuf2 = sqlite3_malloc(szPage);
        if( pBuf1==0 || pBuf2==0 ){
          rc = SQLITE_NOMEM;
        }else{
          u32 i = pInfo->nBackfillAttempted;
          for(i=pInfo->nBackfillAttempted; i>pInfo->nBackfill; i--){
            volatile ht_slot *dummy;
            volatile u32 *aPgno;      /* Array of page numbers */
            u32 iZero;                /* Frame corresponding to aPgno[0] */
            u32 pgno;                 /* Page number in db file */
            i64 iDbOff;               /* Offset of db file entry */
            i64 iWalOff;              /* Offset of wal file entry */

            rc = walHashGet(pWal, walFramePage(i), &dummy, &aPgno, &iZero);
            if( rc!=SQLITE_OK ) break;
            pgno = aPgno[i-iZero];
            iDbOff = (i64)(pgno-1) * szPage;

            if( iDbOff+szPage<=szDb ){
              iWalOff = walFrameOffset(i, szPage) + WAL_FRAME_HDRSIZE;
              rc = sqlite3OsRead(pWal->pWalFd, pBuf1, szPage, iWalOff);

              if( rc==SQLITE_OK ){
                rc = sqlite3OsRead(pWal->pDbFd, pBuf2, szPage, iDbOff);
              }

              if( rc!=SQLITE_OK || 0==memcmp(pBuf1, pBuf2, szPage) ){
                break;
              }
            }

            pInfo->nBackfillAttempted = i-1;
          }
        }

        sqlite3_free(pBuf1);
        sqlite3_free(pBuf2);
      }
      walUnlockExclusive(pWal, WAL_CKPT_LOCK, 1);
    }

    /* End the read transaction opened above. Also zero the cache of the
    ** wal-index header to force the pager-cache to be flushed when the next
    ** read transaction is open, as it may not match the current contents of
    ** pWal->hdr. */
    sqlite3WalEndReadTransaction(pWal);
    memset(&pWal->hdr, 0, sizeof(WalIndexHdr));
  }

  return rc;
}

/*
** Begin a read transaction on the database.







>
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>


<


|
<
|
|
|
|
|

|
|
|
|
|
|
|
|
|
|
|
|
|
|
|

|
|
|
|

|
|
|

|
|
|

|
|
|
|

|
|
|

|
|
|
|
<
<
<
<
<
<
<
<







2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401

2402
2403
2404

2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452








2453
2454
2455
2456
2457
2458
2459
    assert( mxReadMark<=pWal->hdr.mxFrame );
    pWal->readLock = (i16)mxI;
  }
  return rc;
}

/*
** Attempt to reduce the value of the WalCkptInfo.nBackfillAttempted 
** variable so that older snapshots can be accessed. To do this, loop
** through all wal frames from nBackfillAttempted to (nBackfill+1), 
** comparing their content to the corresponding page with the database
** file, if any. Set nBackfillAttempted to the frame number of the
** first frame for which the wal file content matches the db file.
**
** This is only really safe if the file-system is such that any page 
** writes made by earlier checkpointers were atomic operations, which 
** is not always true. It is also possible that nBackfillAttempted
** may be left set to a value larger than expected, if a wal frame
** contains content that duplicate of an earlier version of the same
** page.
**
** SQLITE_OK is returned if successful, or an SQLite error code if an
** error occurs. It is not an error if nBackfillAttempted cannot be
** decreased at all.
*/
int sqlite3WalSnapshotRecover(Wal *pWal){

  int rc;

  assert( pWal->readLock>=0 );

  rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1);
  if( rc==SQLITE_OK ){
    volatile WalCkptInfo *pInfo = walCkptInfo(pWal);
    int szPage = (int)pWal->szPage;
    i64 szDb;                   /* Size of db file in bytes */

    rc = sqlite3OsFileSize(pWal->pDbFd, &szDb);
    if( rc==SQLITE_OK ){
      void *pBuf1 = sqlite3_malloc(szPage);
      void *pBuf2 = sqlite3_malloc(szPage);
      if( pBuf1==0 || pBuf2==0 ){
        rc = SQLITE_NOMEM;
      }else{
        u32 i = pInfo->nBackfillAttempted;
        for(i=pInfo->nBackfillAttempted; i>pInfo->nBackfill; i--){
          volatile ht_slot *dummy;
          volatile u32 *aPgno;      /* Array of page numbers */
          u32 iZero;                /* Frame corresponding to aPgno[0] */
          u32 pgno;                 /* Page number in db file */
          i64 iDbOff;               /* Offset of db file entry */
          i64 iWalOff;              /* Offset of wal file entry */

          rc = walHashGet(pWal, walFramePage(i), &dummy, &aPgno, &iZero);
          if( rc!=SQLITE_OK ) break;
          pgno = aPgno[i-iZero];
          iDbOff = (i64)(pgno-1) * szPage;

          if( iDbOff+szPage<=szDb ){
            iWalOff = walFrameOffset(i, szPage) + WAL_FRAME_HDRSIZE;
            rc = sqlite3OsRead(pWal->pWalFd, pBuf1, szPage, iWalOff);

            if( rc==SQLITE_OK ){
              rc = sqlite3OsRead(pWal->pDbFd, pBuf2, szPage, iDbOff);
            }

            if( rc!=SQLITE_OK || 0==memcmp(pBuf1, pBuf2, szPage) ){
              break;
            }
          }

          pInfo->nBackfillAttempted = i-1;
        }
      }

      sqlite3_free(pBuf1);
      sqlite3_free(pBuf2);
    }
    walUnlockExclusive(pWal, WAL_CKPT_LOCK, 1);








  }

  return rc;
}

/*
** Begin a read transaction on the database.
Changes to test/snapshot2.test.
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141

do_test 2.5 {
  execsql { PRAGMA wal_checkpoint }
  sqlite3_db_config db NO_CKPT_ON_CLOSE 1
  db close
  sqlite3 db test.db

  execsql {SELECT * FROM sqlite_master}
  sqlite3_snapshot_recover db main
  execsql BEGIN
  list [catch { sqlite3_snapshot_open_blob db main $snap } msg] $msg
} {1 SQLITE_BUSY_SNAPSHOT}

#-------------------------------------------------------------------------
# Check that calling sqlite3_snapshot_recover() does not confuse the







<







127
128
129
130
131
132
133

134
135
136
137
138
139
140

do_test 2.5 {
  execsql { PRAGMA wal_checkpoint }
  sqlite3_db_config db NO_CKPT_ON_CLOSE 1
  db close
  sqlite3 db test.db


  sqlite3_snapshot_recover db main
  execsql BEGIN
  list [catch { sqlite3_snapshot_open_blob db main $snap } msg] $msg
} {1 SQLITE_BUSY_SNAPSHOT}

#-------------------------------------------------------------------------
# Check that calling sqlite3_snapshot_recover() does not confuse the