Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
| Comment: | Fixes for locking issues in WAL mode. |
|---|---|
| Downloads: | Tarball | ZIP archive |
| Timelines: | family | ancestors | descendants | both | wal |
| Files: | files | file ages | folders |
| SHA1: |
a9617eff39177250e2f118f25fdd4b3a |
| User & Date: | dan 2010-04-14 11:23:30.000 |
Context
|
2010-04-14
| ||
| 15:49 | Improve the logLockRegion() function in log.c. (check-in: 5e9dd3bd8e user: dan tags: wal) | |
| 11:23 | Fixes for locking issues in WAL mode. (check-in: a9617eff39 user: dan tags: wal) | |
|
2010-04-13
| ||
| 19:27 | Add experimental locking scheme. (check-in: 3f958e87c3 user: dan tags: wal) | |
Changes
Changes to src/log.c.
| ︙ | ︙ | |||
613 614 615 616 617 618 619 |
** exist it is created by this call.
*/
int sqlite3LogOpen(
sqlite3_vfs *pVfs, /* vfs module to open log file with */
const char *zDb, /* Name of database file */
Log **ppLog /* OUT: Allocated Log handle */
){
| | | 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 |
** exist it is created by this call.
*/
int sqlite3LogOpen(
sqlite3_vfs *pVfs, /* vfs module to open log file with */
const char *zDb, /* Name of database file */
Log **ppLog /* OUT: Allocated Log handle */
){
int rc = SQLITE_OK; /* Return Code */
Log *pRet; /* Object to allocate and return */
LogSummary *pSummary = 0; /* Summary object */
sqlite3_mutex *mutex = 0; /* LOG_SUMMARY_MUTEX mutex */
int flags; /* Flags passed to OsOpen() */
char *zWal = 0; /* Path to WAL file */
int nWal; /* Length of zWal in bytes */
|
| ︙ | ︙ | |||
802 803 804 805 806 807 808 |
sqlite3_file *pFd, /* File descriptor open on db file */
u8 *zBuf /* Temporary buffer to use */
){
int rc; /* Return code */
int pgsz = pLog->hdr.pgsz; /* Database page-size */
LogCheckpoint *pIter = 0; /* Log iterator context */
u32 iDbpage = 0; /* Next database page to write */
| | | 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 |
sqlite3_file *pFd, /* File descriptor open on db file */
u8 *zBuf /* Temporary buffer to use */
){
int rc; /* Return code */
int pgsz = pLog->hdr.pgsz; /* Database page-size */
LogCheckpoint *pIter = 0; /* Log iterator context */
u32 iDbpage = 0; /* Next database page to write */
u32 iFrame = 0; /* Log frame containing data for iDbpage */
/* Allocate the iterator */
pIter = logCheckpointInit(pLog);
if( !pIter ) return SQLITE_NOMEM;
/* Sync the log file to disk */
rc = sqlite3OsSync(pLog->pFd, pLog->sync_flags);
|
| ︙ | ︙ | |||
1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 |
}
}
pLog->lock.mLock = mNewLock;
sqlite3_mutex_leave(pSummary->mutex);
return SQLITE_OK;
}
/*
** Lock a snapshot.
**
** If this call obtains a new read-lock and the database contents have been
** modified since the most recent call to LogCloseSnapshot() on this Log
** connection, then *pChanged is set to 1 before returning. Otherwise, it
| > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 |
}
}
pLog->lock.mLock = mNewLock;
sqlite3_mutex_leave(pSummary->mutex);
return SQLITE_OK;
}
/*
** Try to read the log-summary header. Attempt to verify the header
** checksum. If the checksum can be verified, copy the log-summary
** header into structure pLog->hdr. If the contents of pLog->hdr are
** modified by this and pChanged is not NULL, set *pChanged to 1.
** Otherwise leave *pChanged unmodified.
**
** If the checksum cannot be verified return SQLITE_ERROR.
*/
int logSummaryTryHdr(Log *pLog, int *pChanged){
u32 aCksum[2] = {1, 1};
u32 aHdr[LOGSUMMARY_HDR_NFIELD+2];
/* First try to read the header without a lock. Verify the checksum
** before returning. This will almost always work.
*/
memcpy(aHdr, pLog->pSummary->aData, sizeof(aHdr));
logChecksumBytes((u8*)aHdr, sizeof(u32)*LOGSUMMARY_HDR_NFIELD, aCksum);
if( aCksum[0]!=aHdr[LOGSUMMARY_HDR_NFIELD]
|| aCksum[1]!=aHdr[LOGSUMMARY_HDR_NFIELD+1]
){
return SQLITE_ERROR;
}
if( memcmp(&pLog->hdr, aHdr, sizeof(LogSummaryHdr)) ){
if( pChanged ){
*pChanged = 1;
}
memcpy(&pLog->hdr, aHdr, sizeof(LogSummaryHdr));
}
return SQLITE_OK;
}
/*
** Read the log-summary header from the log-summary file into structure
** pLog->hdr. If attempting to verify the header checksum fails, try
** to recover the log before returning.
**
** If the log-summary header is successfully read, return SQLITE_OK.
** Otherwise an SQLite error code.
*/
int logSummaryReadHdr(Log *pLog, int *pChanged){
int rc;
/* First try to read the header without a lock. Verify the checksum
** before returning. This will almost always work.
*/
if( SQLITE_OK==logSummaryTryHdr(pLog, pChanged) ){
return SQLITE_OK;
}
/* If the first attempt to read the header failed, lock the log-summary
** file and try again. If the header checksum verification fails this
** time as well, run log recovery.
*/
if( SQLITE_OK==(rc = logEnterMutex(pLog)) ){
if( SQLITE_OK!=logSummaryTryHdr(pLog, pChanged) ){
if( pChanged ){
*pChanged = 1;
}
rc = logSummaryRecover(pLog->pSummary, pLog->pFd);
if( rc==SQLITE_OK ){
rc = logSummaryTryHdr(pLog, 0);
}
}
logLeaveMutex(pLog);
}
return rc;
}
/*
** Lock a snapshot.
**
** If this call obtains a new read-lock and the database contents have been
** modified since the most recent call to LogCloseSnapshot() on this Log
** connection, then *pChanged is set to 1 before returning. Otherwise, it
|
| ︙ | ︙ | |||
1118 1119 1120 1121 1122 1123 1124 |
pLog->isLocked = LOG_REGION_A;
}
}
if( rc!=SQLITE_OK ){
return rc;
}
| < < < < < < < < < < < < | < < < < < < < < < < < < | 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 |
pLog->isLocked = LOG_REGION_A;
}
}
if( rc!=SQLITE_OK ){
return rc;
}
rc = logSummaryReadHdr(pLog, pChanged);
if( rc!=SQLITE_OK ){
/* An error occured while attempting log recovery. */
sqlite3LogCloseSnapshot(pLog);
}
}
return rc;
}
|
| ︙ | ︙ | |||
1405 1406 1407 1408 1409 1410 1411 |
logLeaveMutex(pLog);
}
return SQLITE_OK;
}
/*
| | > > > | > > | > > | > > | > > > > | | > > | 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 |
logLeaveMutex(pLog);
}
return SQLITE_OK;
}
/*
** Checkpoint the database:
**
** 1. Wait for an EXCLUSIVE lock on regions B and C.
** 2. Wait for an EXCLUSIVE lock on region A.
** 3. Copy the contents of the log into the database file.
** 4. Zero the log-summary header (so new readers will ignore the log).
** 5. Drop the locks obtained in steps 1 and 2.
*/
int sqlite3LogCheckpoint(
Log *pLog, /* Log connection */
sqlite3_file *pFd, /* File descriptor open on db file */
u8 *zBuf, /* Temporary buffer to use */
int (*xBusyHandler)(void *), /* Pointer to busy-handler function */
void *pBusyHandlerArg /* Argument to pass to xBusyHandler */
){
int rc; /* Return code */
/* Wait for a write-lock on regions B and C. */
do {
rc = logLockRegion(pLog, LOG_REGION_B|LOG_REGION_C, LOG_WRLOCK);
}while( rc==SQLITE_BUSY && xBusyHandler(pBusyHandlerArg) );
if( rc!=SQLITE_OK ) return rc;
/* Wait for a write-lock on region A. */
do {
rc = logLockRegion(pLog, LOG_REGION_A, LOG_WRLOCK);
}while( rc==SQLITE_BUSY && xBusyHandler(pBusyHandlerArg) );
if( rc!=SQLITE_OK ){
logLockRegion(pLog, LOG_REGION_B|LOG_REGION_C, LOG_UNLOCK);
return rc;
}
/* Copy data from the log to the database file. */
rc = logSummaryReadHdr(pLog, 0);
if( rc==SQLITE_OK ){
rc = logCheckpoint(pLog, pFd, zBuf);
}
/* Release the locks. */
logLockRegion(pLog, LOG_REGION_A|LOG_REGION_B|LOG_REGION_C, LOG_UNLOCK);
return rc;
}
|
Changes to src/pragma.c.
| ︙ | ︙ | |||
1382 1383 1384 1385 1386 1387 1388 |
sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1);
}
}else
#endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */
if( sqlite3StrICmp(zLeft, "checkpoint")==0 ){
sqlite3VdbeUsesBtree(v, iDb);
| < | 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 |
sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1);
}
}else
#endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */
if( sqlite3StrICmp(zLeft, "checkpoint")==0 ){
sqlite3VdbeUsesBtree(v, iDb);
sqlite3VdbeAddOp3(v, OP_Checkpoint, iDb, 0, 0);
}else
#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST)
/*
** Report the current state of file logs for all databases
*/
|
| ︙ | ︙ |
Changes to test/wal.test.
| ︙ | ︙ | |||
23 24 25 26 27 28 29 |
lappend ret $i
}
}
set ret
}
proc reopen_db {} {
| | | 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 |
lappend ret $i
}
}
set ret
}
proc reopen_db {} {
catch { db close }
file delete -force test.db test.db-wal
sqlite3_wal db test.db
#register_logtest
}
proc register_logtest {{db db}} {
register_logsummary_module $db
execsql { CREATE VIRTUAL TABLE temp.logsummary USING logsummary } $db
|
| ︙ | ︙ | |||
219 220 221 222 223 224 225 |
file delete -force test.db test.db-wal
sqlite3_wal db test.db
execsql {
PRAGMA page_size = 1024;
CREATE TABLE t1(a, b);
INSERT INTO t1 VALUES(1, 2);
}
| < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | | | | | | | < > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 |
file delete -force test.db test.db-wal
sqlite3_wal db test.db
execsql {
PRAGMA page_size = 1024;
CREATE TABLE t1(a, b);
INSERT INTO t1 VALUES(1, 2);
}
list [file size test.db] [file size test.db-wal]
} [list 0 [expr (1024+20)*3]]
do_test wal-7.2 {
execsql { PRAGMA checkpoint }
list [file size test.db] [file size test.db-wal]
} [list 2048 [expr (1024+20)*3]]
# Execute some transactions in auto-vacuum mode to test database file
# truncation.
#
do_test wal-8.1 {
reopen_db
execsql {
PRAGMA auto_vacuum = 1;
PRAGMA auto_vacuum;
}
} {1}
do_test wal-8.2 {
execsql {
PRAGMA page_size = 1024;
CREATE TABLE t1(x);
INSERT INTO t1 VALUES(randomblob(900));
INSERT INTO t1 VALUES(randomblob(900));
INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 4 */
INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 8 */
INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 16 */
INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 32 */
INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 64 */
PRAGMA checkpoint;
}
file size test.db
} [expr 68*1024]
do_test wal-8.3 {
execsql {
DELETE FROM t1 WHERE rowid<54;
PRAGMA checkpoint;
}
file size test.db
} [expr 14*1024]
# Run some "warm-body" tests to ensure that log-summary files with more
# than 256 entries (log summaries that contain index blocks) work Ok.
#
do_test wal-9.1 {
reopen_db
execsql {
PRAGMA page_size = 1024;
CREATE TABLE t1(x PRIMARY KEY);
INSERT INTO t1 VALUES(randomblob(900));
INSERT INTO t1 VALUES(randomblob(900));
INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 4 */
INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 8 */
INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 16 */
INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 32 */
INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 64 */
INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 128 */
INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 256 */
}
file size test.db
} 0
do_test wal-9.2 {
sqlite3_wal db2 test.db
execsql {PRAGMA integrity_check } db2
} {ok}
do_test wal-9.3 {
file delete -force test2.db test2.db-wal
file copy test.db test2.db
file copy test.db-wal test2.db-wal
sqlite3_wal db3 test2.db
execsql {PRAGMA integrity_check } db3
} {ok}
db3 close
do_test wal-9.4 {
execsql { PRAGMA checkpoint }
db2 close
sqlite3_wal db2 test.db
execsql {PRAGMA integrity_check } db2
} {ok}
foreach handle {db db2 db3} { catch { $handle close } }
unset handle
#-------------------------------------------------------------------------
# The following block of tests - wal-10.* - test that the WAL locking
# scheme works for clients in a single process.
#
reopen_db
sqlite3_wal db2 test.db
sqlite3_wal db3 test.db
do_test wal-10.1 {
execsql {
CREATE TABLE t1(a, b);
INSERT INTO t1 VALUES(1, 2);
BEGIN;
INSERT INTO t1 VALUES(3, 4);
}
execsql "SELECT * FROM t1" db2
} {1 2}
do_test wal-10.2 {
execsql { COMMIT }
execsql "SELECT * FROM t1" db2
} {1 2 3 4}
do_test wal-10.3 {
execsql {
BEGIN;
SELECT * FROM t1;
} db2
} {1 2 3 4}
do_test wal-10.4 {
catchsql { PRAGMA checkpoint }
} {1 {database is locked}}
do_test wal-10.5 {
execsql { INSERT INTO t1 VALUES(5, 6) }
execsql { SELECT * FROM t1 } db2
} {1 2 3 4}
# Connection [db2] is holding a lock on a snapshot, preventing [db] from
# checkpointing the database. Add a busy-handler to [db]. If [db2] completes
# its transaction from within the busy-handler, [db] is able to complete
# the checkpoint operation.
#
proc busyhandler x {
if {$x==4} {
execsql { COMMIT } db2
}
if {$x<5} {return 0}
return 1
}
db busy busyhandler
do_test wal-10.6 {
execsql { PRAGMA checkpoint }
} {}
# Similar to the test above. Except this time, a new read transaction is
# started (db3) while the checkpointer is waiting for an old one to finish.
# The checkpointer can finish, but any subsequent write operations must
# wait until after db3 has closed the read transaction.
#
db busy {}
do_test wal-10.7 {
execsql {
BEGIN;
SELECT * FROM t1;
} db2
} {1 2 3 4 5 6}
do_test wal-10.8 {
execsql { INSERT INTO t1 VALUES(7, 8) }
catchsql { PRAGMA checkpoint }
} {1 {database is locked}}
proc busyhandler x {
if {$x==3} { execsql { BEGIN; SELECT * FROM t1 } db3 }
if {$x==4} { execsql { COMMIT } db2 }
if {$x<5} { return 0 }
return 1
}
db busy busyhandler
do_test wal-10.9 {
execsql { PRAGMA checkpoint }
} {}
do_test wal-10.10 {
execsql { SELECT * FROM t1 } db3
} {1 2 3 4 5 6 7 8}
do_test wal-10.11 {
catchsql { INSERT INTO t1 VALUES(9, 10) }
} {1 {database is locked}}
do_test wal-10.12 {
execsql { SELECT * FROM t1 }
} {1 2 3 4 5 6 7 8}
do_test wal-10.13 {
execsql { COMMIT } db3
} {}
do_test wal-10.14 {
execsql { INSERT INTO t1 VALUES(9, 10) }
execsql { SELECT * FROM t1 }
} {1 2 3 4 5 6 7 8 9 10}
foreach handle {db db2 db3} { catch { $handle close } }
unset handle
finish_test
|
Changes to test/walthread.test.
| ︙ | ︙ | |||
58 59 60 61 62 63 64 | # 1) Modifying the contents of t1 (inserting, updating, deleting rows). # 2) Appending a new row to the table containing the md5sum() of all # rows in the table. # # Each of the N threads runs N read transactions followed by a single write # transaction in a loop as fast as possible. # | | | 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 |
# 1) Modifying the contents of t1 (inserting, updating, deleting rows).
# 2) Appending a new row to the table containing the md5sum() of all
# rows in the table.
#
# Each of the N threads runs N read transactions followed by a single write
# transaction in a loop as fast as possible.
#
# Ther is also a single checkpointer thread. It runs the following loop:
#
# 1) Execute "CHECKPOINT main 32 -1 1"
# 2) Sleep for 500 ms.
#
set thread_program {
proc rest {ms} {
|
| ︙ | ︙ | |||
81 82 83 84 85 86 87 |
set rc [sqlite3_step $stmt]
if {$rc eq "SQLITE_ROW"} {
set res [sqlite3_column_text $stmt 0]
}
set rc [sqlite3_finalize $stmt]
if {$rc ne "SQLITE_OK"} {
| | | 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 |
set rc [sqlite3_step $stmt]
if {$rc eq "SQLITE_ROW"} {
set res [sqlite3_column_text $stmt 0]
}
set rc [sqlite3_finalize $stmt]
if {$rc ne "SQLITE_OK"} {
error "$rc: [sqlite3_errmsg $DB]"
}
return $res
}
proc read_transaction {DB} {
dosql $DB BEGIN
|
| ︙ | ︙ |