Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | If it is known that checksums will be recalculated on transaction commit, skip calculating checksums when appending frames to the wal file. When recalculating checksums, recalculate them starting with the first overwritten frame - not the first frame in the transaction. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | wal-overwrite-frames |
Files: | files | file ages | folders |
SHA1: |
16b34f2537bbc7846d8e6dc2b35daae5 |
User & Date: | dan 2016-01-09 18:57:35.280 |
Context
2016-01-09
| ||
23:55 | All WAL frame overwrites even if there are active savepoints. This is safe because a ROLLBACK TO will cause all reverted pages to be rewritten to the WAL file prior to COMMIT. (check-in: 99b31a6b49 user: drh tags: wal-overwrite-frames) | |
18:57 | If it is known that checksums will be recalculated on transaction commit, skip calculating checksums when appending frames to the wal file. When recalculating checksums, recalculate them starting with the first overwritten frame - not the first frame in the transaction. (check-in: 16b34f2537 user: dan tags: wal-overwrite-frames) | |
16:39 | If a single page is written to the wal file more than once, have each subsequent copy overwrite the original frame. (check-in: 5d113aef2c user: dan tags: wal-overwrite-frames) | |
Changes
Changes to src/wal.c.
︙ | ︙ | |||
441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 | u8 ckptLock; /* True if holding a checkpoint lock */ u8 readOnly; /* WAL_RDWR, WAL_RDONLY, or WAL_SHM_RDONLY */ u8 truncateOnCommit; /* True to truncate WAL file on commit */ u8 syncHeader; /* Fsync the WAL header if true */ u8 padToSectorBoundary; /* Pad transactions out to the next sector */ WalIndexHdr hdr; /* Wal-index header for current transaction */ u32 minFrame; /* Ignore wal frames before this one */ const char *zWalName; /* Name of WAL file */ u32 nCkpt; /* Checkpoint sequence counter in the wal-header */ #ifdef SQLITE_DEBUG u8 lockError; /* True if a locking error has occurred */ #endif #ifdef SQLITE_ENABLE_SNAPSHOT WalIndexHdr *pSnapshot; /* Start transaction here if not NULL */ #endif }; /* ** Candidate values for Wal.exclusiveMode. */ #define WAL_NORMAL_MODE 0 #define WAL_EXCLUSIVE_MODE 1 #define WAL_HEAPMEMORY_MODE 2 | > < < < < < < < | 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 | u8 ckptLock; /* True if holding a checkpoint lock */ u8 readOnly; /* WAL_RDWR, WAL_RDONLY, or WAL_SHM_RDONLY */ u8 truncateOnCommit; /* True to truncate WAL file on commit */ u8 syncHeader; /* Fsync the WAL header if true */ u8 padToSectorBoundary; /* Pad transactions out to the next sector */ WalIndexHdr hdr; /* Wal-index header for current transaction */ u32 minFrame; /* Ignore wal frames before this one */ u32 iReCksum; /* On commit, recalculate checksums from here */ const char *zWalName; /* Name of WAL file */ u32 nCkpt; /* Checkpoint sequence counter in the wal-header */ #ifdef SQLITE_DEBUG u8 lockError; /* True if a locking error has occurred */ #endif #ifdef SQLITE_ENABLE_SNAPSHOT WalIndexHdr *pSnapshot; /* Start transaction here if not NULL */ #endif }; /* ** Candidate values for Wal.exclusiveMode. */ #define WAL_NORMAL_MODE 0 #define WAL_EXCLUSIVE_MODE 1 #define WAL_HEAPMEMORY_MODE 2 /* ** Possible values for WAL.readOnly */ #define WAL_RDWR 0 /* Normal read/write connection */ #define WAL_RDONLY 1 /* The WAL file is readonly */ #define WAL_SHM_RDONLY 2 /* The SHM file is readonly */ |
︙ | ︙ | |||
701 702 703 704 705 706 707 | u8 *aFrame /* OUT: Write encoded frame here */ ){ int nativeCksum; /* True for native byte-order checksums */ u32 *aCksum = pWal->hdr.aFrameCksum; assert( WAL_FRAME_HDRSIZE==24 ); sqlite3Put4byte(&aFrame[0], iPage); sqlite3Put4byte(&aFrame[4], nTruncate); | > | | | | | | > | 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 | u8 *aFrame /* OUT: Write encoded frame here */ ){ int nativeCksum; /* True for native byte-order checksums */ u32 *aCksum = pWal->hdr.aFrameCksum; assert( WAL_FRAME_HDRSIZE==24 ); sqlite3Put4byte(&aFrame[0], iPage); sqlite3Put4byte(&aFrame[4], nTruncate); if( pWal->iReCksum==0 ){ memcpy(&aFrame[8], pWal->hdr.aSalt, 8); nativeCksum = (pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN); walChecksumBytes(nativeCksum, aFrame, 8, aCksum, aCksum); walChecksumBytes(nativeCksum, aData, pWal->szPage, aCksum, aCksum); sqlite3Put4byte(&aFrame[16], aCksum[0]); sqlite3Put4byte(&aFrame[20], aCksum[1]); } } /* ** Check to see if the frame with header in aFrame[] and content ** in aData[] is valid. If it is a valid frame, fill *piPage and ** *pnTruncate and return true. Return if the frame is not valid. */ |
︙ | ︙ | |||
2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 | */ int sqlite3WalBeginWriteTransaction(Wal *pWal){ int rc; /* Cannot start a write transaction without first holding a read ** transaction. */ assert( pWal->readLock>=0 ); if( pWal->readOnly ){ return SQLITE_READONLY; } /* Only one writer allowed at a time. Get the write lock. Return ** SQLITE_BUSY if unable. | > | 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 | */ int sqlite3WalBeginWriteTransaction(Wal *pWal){ int rc; /* Cannot start a write transaction without first holding a read ** transaction. */ assert( pWal->readLock>=0 ); assert( pWal->writeLock==0 && pWal->iReCksum==0 ); if( pWal->readOnly ){ return SQLITE_READONLY; } /* Only one writer allowed at a time. Get the write lock. Return ** SQLITE_BUSY if unable. |
︙ | ︙ | |||
2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 | ** End a write transaction. The commit has already been done. This ** routine merely releases the lock. */ int sqlite3WalEndWriteTransaction(Wal *pWal){ if( pWal->writeLock ){ walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); pWal->writeLock = 0; pWal->truncateOnCommit = 0; } return SQLITE_OK; } /* ** If any data has been written (but not committed) to the log file, this | > | 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 | ** End a write transaction. The commit has already been done. This ** routine merely releases the lock. */ int sqlite3WalEndWriteTransaction(Wal *pWal){ if( pWal->writeLock ){ walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); pWal->writeLock = 0; pWal->iReCksum = 0; pWal->truncateOnCommit = 0; } return SQLITE_OK; } /* ** If any data has been written (but not committed) to the log file, this |
︙ | ︙ | |||
2891 2892 2893 2894 2895 2896 2897 | rc = walWriteToLog(p, pData, p->szPage, iOffset+sizeof(aFrame)); return rc; } /* ** This function is called as part of committing a transaction within which ** one or more frames have been overwritten. It updates the checksums for | | | < < < < < | > | | | | | | | > > > > | | | < < | | > | | 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 | rc = walWriteToLog(p, pData, p->szPage, iOffset+sizeof(aFrame)); return rc; } /* ** This function is called as part of committing a transaction within which ** one or more frames have been overwritten. It updates the checksums for ** all frames written to the wal file by the current transaction starting ** with the earliest to have been overwritten. ** ** SQLITE_OK is returned if successful, or an SQLite error code otherwise. */ static int walRewriteChecksums(Wal *pWal, u32 iLast){ const int szPage = pWal->szPage;/* Database page size */ int rc = SQLITE_OK; /* Return code */ u8 *aBuf; /* Buffer to load data from wal file into */ u8 aFrame[WAL_FRAME_HDRSIZE]; /* Buffer to assemble frame-headers in */ u32 iRead; /* Next frame to read from wal file */ i64 iCksumOff; aBuf = sqlite3_malloc(szPage + WAL_FRAME_HDRSIZE); if( aBuf==0 ) return SQLITE_NOMEM; /* Find the checksum values to use as input for the recalculating the ** first checksum. If the first frame is frame 1 (implying that the current ** transaction restarted the wal file), these values must be read from the ** wal-file header. Otherwise, read them from the frame header of the ** previous frame. */ assert( pWal->iReCksum>0 ); if( pWal->iReCksum==1 ){ iCksumOff = 24; }else{ iCksumOff = walFrameOffset(pWal->iReCksum-1, szPage) + 16; } rc = sqlite3OsRead(pWal->pWalFd, aBuf, sizeof(u32)*2, iCksumOff); pWal->hdr.aFrameCksum[0] = sqlite3Get4byte(aBuf); pWal->hdr.aFrameCksum[1] = sqlite3Get4byte(&aBuf[sizeof(u32)]); iRead = pWal->iReCksum; pWal->iReCksum = 0; for(; rc==SQLITE_OK && iRead<=iLast; iRead++){ i64 iOff = walFrameOffset(iRead, szPage); rc = sqlite3OsRead(pWal->pWalFd, aBuf, szPage+WAL_FRAME_HDRSIZE, iOff); if( rc==SQLITE_OK ){ u32 iPgno, nDbSize; iPgno = sqlite3Get4byte(aBuf); nDbSize = sqlite3Get4byte(&aBuf[4]); |
︙ | ︙ | |||
3061 3062 3063 3064 3065 3066 3067 | ** checksums must be recomputed when the transaction is committed. */ if( iFirst && (p->pDirty || isCommit==0) ){ u32 iWrite = 0; rc = sqlite3WalFindFrame(pWal, p->pgno, &iWrite); if( rc ) return rc; if( iWrite>=iFirst ){ i64 iOff = walFrameOffset(iWrite, szPage) + WAL_FRAME_HDRSIZE; | > | > | | | 3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 | ** checksums must be recomputed when the transaction is committed. */ if( iFirst && (p->pDirty || isCommit==0) ){ u32 iWrite = 0; rc = sqlite3WalFindFrame(pWal, p->pgno, &iWrite); if( rc ) return rc; if( iWrite>=iFirst ){ i64 iOff = walFrameOffset(iWrite, szPage) + WAL_FRAME_HDRSIZE; if( pWal->iReCksum==0 || iWrite<pWal->iReCksum ){ pWal->iReCksum = iWrite; } rc = sqlite3OsWrite(pWal->pWalFd, p->pData, szPage, iOff); if( rc ) return rc; p->flags &= ~PGHDR_WAL_APPEND; continue; } } iFrame++; assert( iOffset==walFrameOffset(iFrame, szPage) ); nDbSize = (isCommit && p->pDirty==0) ? nTruncate : 0; rc = walWriteOneFrame(&w, p, nDbSize, iOffset); if( rc ) return rc; pLast = p; iOffset += szFrame; p->flags |= PGHDR_WAL_APPEND; } /* Recalculate checksums within the wal file if required. */ if( isCommit && pWal->iReCksum ){ rc = walRewriteChecksums(pWal, iFrame); if( rc ) return rc; } /* If this is the end of a transaction, then we might need to pad ** the transaction and/or sync the WAL file. ** ** Padding and syncing only occur if this set of frames complete a |
︙ | ︙ |