Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Merge enhancements from trunk. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | larger-databases |
Files: | files | file ages | folders |
SHA3-256: |
969c25bb14fbd99ca8523abf0ae78a75 |
User & Date: | drh 2020-07-28 17:51:48.981 |
Context
2020-07-28
| ||
20:32 | Earlier detection of out-of-range page numbers in the btree layer. (check-in: 805bb67a82 user: drh tags: larger-databases) | |
17:51 | Merge enhancements from trunk. (check-in: 969c25bb14 user: drh tags: larger-databases) | |
17:29 | If a writer crashes in WAL mode and leave the SHM file in an inconsistent state, subsequent transactions are now able to recover the SHM file even if there are active read transactions. (check-in: ee8a108058 user: drh tags: trunk) | |
2020-07-24
| ||
13:49 | Merge recent changes from trunk. (check-in: 22e8e6901a user: drh tags: larger-databases) | |
Changes
Changes to src/wal.c.
︙ | ︙ | |||
586 587 588 589 590 591 592 | if( !pWal->apWiData[iPage] ) rc = SQLITE_NOMEM_BKPT; }else{ rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ, pWal->writeLock, (void volatile **)&pWal->apWiData[iPage] ); assert( pWal->apWiData[iPage]!=0 || rc!=SQLITE_OK || pWal->writeLock==0 ); testcase( pWal->apWiData[iPage]==0 && rc==SQLITE_OK ); | > > | | 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 | if( !pWal->apWiData[iPage] ) rc = SQLITE_NOMEM_BKPT; }else{ rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ, pWal->writeLock, (void volatile **)&pWal->apWiData[iPage] ); assert( pWal->apWiData[iPage]!=0 || rc!=SQLITE_OK || pWal->writeLock==0 ); testcase( pWal->apWiData[iPage]==0 && rc==SQLITE_OK ); if( rc==SQLITE_OK ){ if( iPage>0 && sqlite3FaultSim(600) ) rc = SQLITE_NOMEM; }else if( (rc&0xff)==SQLITE_READONLY ){ pWal->readOnly |= WAL_SHM_RDONLY; if( rc==SQLITE_READONLY ){ rc = SQLITE_OK; } } } |
︙ | ︙ | |||
1157 1158 1159 1160 1161 1162 1163 | */ assert( pWal->ckptLock==1 || pWal->ckptLock==0 ); assert( WAL_ALL_BUT_WRITE==WAL_WRITE_LOCK+1 ); assert( WAL_CKPT_LOCK==WAL_ALL_BUT_WRITE ); assert( pWal->writeLock ); iLock = WAL_ALL_BUT_WRITE + pWal->ckptLock; rc = walLockExclusive(pWal, iLock, WAL_READ_LOCK(0)-iLock); | < < < < < < > < < > > | 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 | */ assert( pWal->ckptLock==1 || pWal->ckptLock==0 ); assert( WAL_ALL_BUT_WRITE==WAL_WRITE_LOCK+1 ); assert( WAL_CKPT_LOCK==WAL_ALL_BUT_WRITE ); assert( pWal->writeLock ); iLock = WAL_ALL_BUT_WRITE + pWal->ckptLock; rc = walLockExclusive(pWal, iLock, WAL_READ_LOCK(0)-iLock); if( rc ){ return rc; } WALTRACE(("WAL%p: recovery begin...\n", pWal)); memset(&pWal->hdr, 0, sizeof(WalIndexHdr)); rc = sqlite3OsFileSize(pWal->pWalFd, &nSize); if( rc!=SQLITE_OK ){ goto recovery_error; } if( nSize>WAL_HDRSIZE ){ u8 aBuf[WAL_HDRSIZE]; /* Buffer to load WAL header into */ u32 *aPrivate = 0; /* Heap copy of *-shm hash being populated */ u8 *aFrame = 0; /* Malloc'd buffer to load entire frame */ int szFrame; /* Number of bytes in buffer aFrame[] */ u8 *aData; /* Pointer to data part of aFrame buffer */ int szPage; /* Page size according to the log */ u32 magic; /* Magic value read from WAL header */ u32 version; /* Magic value read from WAL header */ int isValid; /* True if this frame is valid */ int iPg; /* Current 32KB wal-index page */ int iLastFrame; /* Last frame in wal, based on nSize alone */ /* Read in the WAL header. */ rc = sqlite3OsRead(pWal->pWalFd, aBuf, WAL_HDRSIZE, 0); if( rc!=SQLITE_OK ){ goto recovery_error; } |
︙ | ︙ | |||
1233 1234 1235 1236 1237 1238 1239 | if( version!=WAL_MAX_VERSION ){ rc = SQLITE_CANTOPEN_BKPT; goto finished; } /* Malloc a buffer to read frames into. */ szFrame = szPage + WAL_FRAME_HDRSIZE; | | > | > > > > > > > > > | > > | | | < | | | | | | | | | | | | | | | | > > > > > > | | | > > > | > > > > > > > > < | 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 | if( version!=WAL_MAX_VERSION ){ rc = SQLITE_CANTOPEN_BKPT; goto finished; } /* Malloc a buffer to read frames into. */ szFrame = szPage + WAL_FRAME_HDRSIZE; aFrame = (u8 *)sqlite3_malloc64(szFrame + WALINDEX_PGSZ); if( !aFrame ){ rc = SQLITE_NOMEM_BKPT; goto recovery_error; } aData = &aFrame[WAL_FRAME_HDRSIZE]; aPrivate = (u32*)&aData[szPage]; /* Read all frames from the log file. */ iLastFrame = (nSize - WAL_HDRSIZE) / szFrame; for(iPg=0; iPg<=walFramePage(iLastFrame); iPg++){ u32 *aShare; int iFrame; /* Index of last frame read */ int iLast = MIN(iLastFrame, HASHTABLE_NPAGE_ONE+iPg*HASHTABLE_NPAGE); int iFirst = 1 + (iPg==0?0:HASHTABLE_NPAGE_ONE+(iPg-1)*HASHTABLE_NPAGE); int nHdr, nHdr32; rc = walIndexPage(pWal, iPg, (volatile u32**)&aShare); if( rc ) break; pWal->apWiData[iPg] = aPrivate; for(iFrame=iFirst; iFrame<=iLast; iFrame++){ i64 iOffset = walFrameOffset(iFrame, szPage); u32 pgno; /* Database page number for frame */ u32 nTruncate; /* dbsize field from frame header */ /* Read and decode the next log frame. */ rc = sqlite3OsRead(pWal->pWalFd, aFrame, szFrame, iOffset); if( rc!=SQLITE_OK ) break; isValid = walDecodeFrame(pWal, &pgno, &nTruncate, aData, aFrame); if( !isValid ) break; rc = walIndexAppend(pWal, iFrame, pgno); if( NEVER(rc!=SQLITE_OK) ) break; /* If nTruncate is non-zero, this is a commit record. */ if( nTruncate ){ pWal->hdr.mxFrame = iFrame; pWal->hdr.nPage = nTruncate; pWal->hdr.szPage = (u16)((szPage&0xff00) | (szPage>>16)); testcase( szPage<=32768 ); testcase( szPage>=65536 ); aFrameCksum[0] = pWal->hdr.aFrameCksum[0]; aFrameCksum[1] = pWal->hdr.aFrameCksum[1]; } } pWal->apWiData[iPg] = aShare; nHdr = (iPg==0 ? WALINDEX_HDR_SIZE : 0); nHdr32 = nHdr / sizeof(u32); memcpy(&aShare[nHdr32], &aPrivate[nHdr32], WALINDEX_PGSZ-nHdr); if( iFrame<=iLast ) break; } sqlite3_free(aFrame); } finished: if( rc==SQLITE_OK ){ volatile WalCkptInfo *pInfo; int i; pWal->hdr.aFrameCksum[0] = aFrameCksum[0]; pWal->hdr.aFrameCksum[1] = aFrameCksum[1]; walIndexWriteHdr(pWal); /* Reset the checkpoint-header. This is safe because this thread is ** currently holding locks that exclude all other writers and ** checkpointers. Then set the values of read-mark slots 1 through N. */ pInfo = walCkptInfo(pWal); pInfo->nBackfill = 0; pInfo->nBackfillAttempted = pWal->hdr.mxFrame; pInfo->aReadMark[0] = 0; for(i=1; i<WAL_NREADER; i++){ rc = walLockExclusive(pWal, WAL_READ_LOCK(i), 1); if( rc==SQLITE_OK ){ if( i==1 && pWal->hdr.mxFrame ){ pInfo->aReadMark[i] = pWal->hdr.mxFrame; }else{ pInfo->aReadMark[i] = READMARK_NOT_USED; } walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); }else if( rc!=SQLITE_BUSY ){ goto recovery_error; } } /* If more than one frame was recovered from the log file, report an ** event via sqlite3_log(). This is to help with identifying performance ** problems caused by applications routinely shutting down without ** checkpointing the log file. */ if( pWal->hdr.nPage ){ sqlite3_log(SQLITE_NOTICE_RECOVER_WAL, "recovered %d frames from WAL file %s", pWal->hdr.mxFrame, pWal->zWalName ); } } recovery_error: WALTRACE(("WAL%p: recovery %s\n", pWal, rc ? "failed" : "ok")); walUnlockExclusive(pWal, iLock, WAL_READ_LOCK(0)-iLock); return rc; } /* ** Close an open wal-index. */ static void walIndexClose(Wal *pWal, int isDelete){ |
︙ | ︙ |
Changes to test/wal2.test.
︙ | ︙ | |||
118 119 120 121 122 123 124 | } } {4 10} do_test wal2-1.1 { execsql { SELECT count(a), sum(a) FROM t1 } db2 } {4 10} set RECOVER [list \ | | > > > > | | 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 | } } {4 10} do_test wal2-1.1 { execsql { SELECT count(a), sum(a) FROM t1 } db2 } {4 10} set RECOVER [list \ {0 1 lock exclusive} {1 2 lock exclusive} \ {4 1 lock exclusive} {4 1 unlock exclusive} \ {5 1 lock exclusive} {5 1 unlock exclusive} \ {6 1 lock exclusive} {6 1 unlock exclusive} \ {7 1 lock exclusive} {7 1 unlock exclusive} \ {1 2 unlock exclusive} {0 1 unlock exclusive} \ ] set READ [list \ {4 1 lock shared} {4 1 unlock shared} \ ] set INITSLOT [list \ {4 1 lock exclusive} {4 1 unlock exclusive} \ ] |
︙ | ︙ | |||
390 391 392 393 394 395 396 | # required the client grabs all exclusive locks (just as it would for a # recovery performed as a pre-cursor to a normal database transaction). # set expected_locks [list] lappend expected_locks {1 1 lock exclusive} ;# Lock checkpoint lappend expected_locks {0 1 lock exclusive} ;# Lock writer lappend expected_locks {2 1 lock exclusive} ;# Lock recovery | | > > > > > > > > | | 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 | # required the client grabs all exclusive locks (just as it would for a # recovery performed as a pre-cursor to a normal database transaction). # set expected_locks [list] lappend expected_locks {1 1 lock exclusive} ;# Lock checkpoint lappend expected_locks {0 1 lock exclusive} ;# Lock writer lappend expected_locks {2 1 lock exclusive} ;# Lock recovery # lappend expected_locks {4 4 lock exclusive} ;# Lock all aReadMark[] lappend expected_locks {4 1 lock exclusive} ;# Lock aReadMark[1] lappend expected_locks {4 1 unlock exclusive} ;# Unlock aReadMark[1] lappend expected_locks {5 1 lock exclusive} lappend expected_locks {5 1 unlock exclusive} lappend expected_locks {6 1 lock exclusive} lappend expected_locks {6 1 unlock exclusive} lappend expected_locks {7 1 lock exclusive} lappend expected_locks {7 1 unlock exclusive} lappend expected_locks {2 1 unlock exclusive} ;# Unlock recovery # lappend expected_locks {4 4 unlock exclusive} ;# Unlock all aReadMark[] lappend expected_locks {0 1 unlock exclusive} ;# Unlock writer lappend expected_locks {3 1 lock exclusive} ;# Lock aReadMark[0] lappend expected_locks {3 1 unlock exclusive} ;# Unlock aReadMark[0] lappend expected_locks {1 1 unlock exclusive} ;# Unlock checkpoint do_test wal2-5.1 { proc tvfs_cb {method args} { set ::shm_file [lindex $args 0] |
︙ | ︙ | |||
621 622 623 624 625 626 627 | testvfs tvfs tvfs script tvfs_cb sqlite3 db test.db -vfs tvfs set {} {} } {} set RECOVERY { | | > > > > | | 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 | testvfs tvfs tvfs script tvfs_cb sqlite3 db test.db -vfs tvfs set {} {} } {} set RECOVERY { {0 1 lock exclusive} {1 2 lock exclusive} {4 1 lock exclusive} {4 1 unlock exclusive} {5 1 lock exclusive} {5 1 unlock exclusive} {6 1 lock exclusive} {6 1 unlock exclusive} {7 1 lock exclusive} {7 1 unlock exclusive} {1 2 unlock exclusive} {0 1 unlock exclusive} } set READMARK0_READ { {3 1 lock shared} {3 1 unlock shared} } set READMARK0_WRITE { {3 1 lock shared} {0 1 lock exclusive} {3 1 unlock shared} |
︙ | ︙ |
Changes to test/walprotocol.test.
︙ | ︙ | |||
48 49 50 51 52 53 54 | do_test 1.1 { testvfs T T filter xShmLock T script lock_callback set ::locks [list] sqlite3 db test.db -vfs T execsql { SELECT * FROM x } | | | > > > > | > | | > > > > | > | 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 | do_test 1.1 { testvfs T T filter xShmLock T script lock_callback set ::locks [list] sqlite3 db test.db -vfs T execsql { SELECT * FROM x } lrange $::locks 0 11 } [list {0 1 lock exclusive} {1 2 lock exclusive} \ {4 1 lock exclusive} {4 1 unlock exclusive} \ {5 1 lock exclusive} {5 1 unlock exclusive} \ {6 1 lock exclusive} {6 1 unlock exclusive} \ {7 1 lock exclusive} {7 1 unlock exclusive} \ {1 2 unlock exclusive} \ {0 1 unlock exclusive} \ ] do_test 1.2 { db close set ::locks [list] sqlite3 db test.db -vfs T execsql { SELECT * FROM x } lrange $::locks 0 11 } [list {0 1 lock exclusive} {1 2 lock exclusive} \ {4 1 lock exclusive} {4 1 unlock exclusive} \ {5 1 lock exclusive} {5 1 unlock exclusive} \ {6 1 lock exclusive} {6 1 unlock exclusive} \ {7 1 lock exclusive} {7 1 unlock exclusive} \ {1 2 unlock exclusive} \ {0 1 unlock exclusive} \ ] proc lock_callback {method filename handle lock} { if {$lock == "1 2 lock exclusive"} { return SQLITE_BUSY } return SQLITE_OK } puts "# Warning: This next test case causes SQLite to call xSleep(1) 100 times." puts "# Normally this equates to a delay of roughly 10 seconds, but if SQLite" |
︙ | ︙ | |||
97 98 99 100 101 102 103 | return SQLITE_OK } do_test 1.5 { db close set ::locks [list] sqlite3 db test.db -vfs T catchsql { SELECT * FROM x } | | | 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 | return SQLITE_OK } do_test 1.5 { db close set ::locks [list] sqlite3 db test.db -vfs T catchsql { SELECT * FROM x } } {0 z} db close T delete #------------------------------------------------------------------------- # do_test 2.1 { forcedelete test.db test.db-journal test.db wal |
︙ | ︙ | |||
156 157 158 159 160 161 162 | sqlite3 db2 test.db puts "# Warning: Another slow test!" do_test 2.5 { execsql { SELECT * FROM b } } {Tehran Qom Markazi Qazvin Gilan Ardabil} do_test 2.6 { set ::r | | | 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 | sqlite3 db2 test.db puts "# Warning: Another slow test!" do_test 2.5 { execsql { SELECT * FROM b } } {Tehran Qom Markazi Qazvin Gilan Ardabil} do_test 2.6 { set ::r } {0 {Tehran Qom Markazi Qazvin Gilan Ardabil}} db close db2 close faultsim_restore_and_reopen sqlite3 db2 test.db T filter xShmLock |
︙ | ︙ | |||
178 179 180 181 182 183 184 | unset ::r puts "# Warning: Last one!" do_test 2.7 { execsql { SELECT * FROM b } } {Tehran Qom Markazi Qazvin Gilan Ardabil} do_test 2.8 { set ::r | | | 188 189 190 191 192 193 194 195 196 197 198 199 200 201 | unset ::r puts "# Warning: Last one!" do_test 2.7 { execsql { SELECT * FROM b } } {Tehran Qom Markazi Qazvin Gilan Ardabil} do_test 2.8 { set ::r } {0 {Tehran Qom Markazi Qazvin Gilan Ardabil}} db close db2 close T delete finish_test |