/ Check-in [a9617eff]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fixes for locking issues in WAL mode.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | wal
Files: files | file ages | folders
SHA1: a9617eff39177250e2f118f25fdd4b3acb8b0478
User & Date: dan 2010-04-14 11:23:30
Context
2010-04-14
15:49
Improve the logLockRegion() function in log.c. check-in: 5e9dd3bd user: dan tags: wal
11:23
Fixes for locking issues in WAL mode. check-in: a9617eff user: dan tags: wal
2010-04-13
19:27
Add experimental locking scheme. check-in: 3f958e87 user: dan tags: wal
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/log.c.

613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
...
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
....
1075
1076
1077
1078
1079
1080
1081







































































1082
1083
1084
1085
1086
1087
1088
....
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
....
1405
1406
1407
1408
1409
1410
1411
1412
1413





1414
1415
1416
1417
1418
1419
1420
1421
1422
1423

1424
1425
1426
1427
1428

1429
1430
1431
1432


1433




1434
1435


1436
1437
1438
1439
** exist it is created by this call.
*/
int sqlite3LogOpen(
  sqlite3_vfs *pVfs,              /* vfs module to open log file with */
  const char *zDb,                /* Name of database file */
  Log **ppLog                     /* OUT: Allocated Log handle */
){
  int rc;                         /* Return Code */
  Log *pRet;                      /* Object to allocate and return */
  LogSummary *pSummary = 0;       /* Summary object */
  sqlite3_mutex *mutex = 0;       /* LOG_SUMMARY_MUTEX mutex */
  int flags;                      /* Flags passed to OsOpen() */
  char *zWal = 0;                 /* Path to WAL file */
  int nWal;                       /* Length of zWal in bytes */

................................................................................
  sqlite3_file *pFd,              /* File descriptor open on db file */
  u8 *zBuf                        /* Temporary buffer to use */
){
  int rc;                         /* Return code */
  int pgsz = pLog->hdr.pgsz;      /* Database page-size */
  LogCheckpoint *pIter = 0;       /* Log iterator context */
  u32 iDbpage = 0;                /* Next database page to write */
  u32 iFrame;                     /* Log frame containing data for iDbpage */

  /* Allocate the iterator */
  pIter = logCheckpointInit(pLog);
  if( !pIter ) return SQLITE_NOMEM;

  /* Sync the log file to disk */
  rc = sqlite3OsSync(pLog->pFd, pLog->sync_flags);
................................................................................
    }
  }

  pLog->lock.mLock = mNewLock;
  sqlite3_mutex_leave(pSummary->mutex);
  return SQLITE_OK;
}








































































/*
** Lock a snapshot.
**
** If this call obtains a new read-lock and the database contents have been
** modified since the most recent call to LogCloseSnapshot() on this Log
** connection, then *pChanged is set to 1 before returning. Otherwise, it 
................................................................................
        pLog->isLocked = LOG_REGION_A;
      }
    }
    if( rc!=SQLITE_OK ){
      return rc;
    }

    if( SQLITE_OK==(rc = logEnterMutex(pLog)) ){
      u32 aCksum[2] = {1, 1};
      u32 aHdr[LOGSUMMARY_HDR_NFIELD+2];
      memcpy(aHdr, pLog->pSummary->aData, sizeof(aHdr));

      /* Verify the checksum on the log-summary header. If it fails,
      ** recover the log-summary from the log file.
      */
      logChecksumBytes((u8*)aHdr, sizeof(u32)*LOGSUMMARY_HDR_NFIELD, aCksum);
      if( aCksum[0]!=aHdr[LOGSUMMARY_HDR_NFIELD]
       || aCksum[1]!=aHdr[LOGSUMMARY_HDR_NFIELD+1]
      ){
        rc = logSummaryRecover(pLog->pSummary, pLog->pFd);
        memcpy(aHdr, pLog->pSummary->aData, sizeof(aHdr));
        *pChanged = 1;
      }
      if( rc==SQLITE_OK ){
        if( memcmp(&pLog->hdr, aHdr, sizeof(LogSummaryHdr)) ){
          *pChanged = 1;
          memcpy(&pLog->hdr, aHdr, LOGSUMMARY_HDR_NFIELD*sizeof(u32));
        }
      }
      logLeaveMutex(pLog);
    }

    if( rc!=SQLITE_OK ){
      /* An error occured while attempting log recovery. */
      sqlite3LogCloseSnapshot(pLog);
    }
  }
  return rc;
}
................................................................................
    logLeaveMutex(pLog);
  }

  return SQLITE_OK;
}

/* 
** Checkpoint the database. When this function is called the caller
** must hold an exclusive lock on the database file.





*/
int sqlite3LogCheckpoint(
  Log *pLog,                      /* Log connection */
  sqlite3_file *pFd,              /* File descriptor open on db file */
  u8 *zBuf,                       /* Temporary buffer to use */
  int (*xBusyHandler)(void *),    /* Pointer to busy-handler function */
  void *pBusyHandlerArg           /* Argument to pass to xBusyHandler */
){
  int rc;


  do {
    rc = logLockRegion(pLog, LOG_REGION_B|LOG_REGION_C, LOG_WRLOCK);
  }while( rc==SQLITE_BUSY && xBusyHandler(pBusyHandlerArg) );
  if( rc!=SQLITE_OK ) return rc;


  do {
    rc = logLockRegion(pLog, LOG_REGION_A, LOG_WRLOCK);
  }while( rc==SQLITE_BUSY && xBusyHandler(pBusyHandlerArg) );
  if( rc!=SQLITE_OK ) return rc;


  




  rc = logCheckpoint(pLog, pFd, zBuf);



  logLockRegion(pLog, LOG_REGION_A|LOG_REGION_B|LOG_REGION_C, LOG_UNLOCK);
  return rc;
}








|







 







|







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







<
<
<
<
<
<
<
<
<
<
<
<
|
<
<
<
<
<
<
<
<
<
<
<
<







 







|
|
>
>
>
>
>








|

>





>



|
>
>
|
>
>
>
>
|
|
>
>




613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
...
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
....
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
....
1189
1190
1191
1192
1193
1194
1195












1196












1197
1198
1199
1200
1201
1202
1203
....
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
** exist it is created by this call.
*/
int sqlite3LogOpen(
  sqlite3_vfs *pVfs,              /* vfs module to open log file with */
  const char *zDb,                /* Name of database file */
  Log **ppLog                     /* OUT: Allocated Log handle */
){
  int rc = SQLITE_OK;             /* Return Code */
  Log *pRet;                      /* Object to allocate and return */
  LogSummary *pSummary = 0;       /* Summary object */
  sqlite3_mutex *mutex = 0;       /* LOG_SUMMARY_MUTEX mutex */
  int flags;                      /* Flags passed to OsOpen() */
  char *zWal = 0;                 /* Path to WAL file */
  int nWal;                       /* Length of zWal in bytes */

................................................................................
  sqlite3_file *pFd,              /* File descriptor open on db file */
  u8 *zBuf                        /* Temporary buffer to use */
){
  int rc;                         /* Return code */
  int pgsz = pLog->hdr.pgsz;      /* Database page-size */
  LogCheckpoint *pIter = 0;       /* Log iterator context */
  u32 iDbpage = 0;                /* Next database page to write */
  u32 iFrame = 0;                 /* Log frame containing data for iDbpage */

  /* Allocate the iterator */
  pIter = logCheckpointInit(pLog);
  if( !pIter ) return SQLITE_NOMEM;

  /* Sync the log file to disk */
  rc = sqlite3OsSync(pLog->pFd, pLog->sync_flags);
................................................................................
    }
  }

  pLog->lock.mLock = mNewLock;
  sqlite3_mutex_leave(pSummary->mutex);
  return SQLITE_OK;
}

/*
** Try to read the log-summary header. Attempt to verify the header
** checksum. If the checksum can be verified, copy the log-summary
** header into structure pLog->hdr. If the contents of pLog->hdr are
** modified by this and pChanged is not NULL, set *pChanged to 1. 
** Otherwise leave *pChanged unmodified.
**
** If the checksum cannot be verified return SQLITE_ERROR.
*/
int logSummaryTryHdr(Log *pLog, int *pChanged){
  u32 aCksum[2] = {1, 1};
  u32 aHdr[LOGSUMMARY_HDR_NFIELD+2];

  /* First try to read the header without a lock. Verify the checksum
  ** before returning. This will almost always work.  
  */
  memcpy(aHdr, pLog->pSummary->aData, sizeof(aHdr));
  logChecksumBytes((u8*)aHdr, sizeof(u32)*LOGSUMMARY_HDR_NFIELD, aCksum);
  if( aCksum[0]!=aHdr[LOGSUMMARY_HDR_NFIELD]
   || aCksum[1]!=aHdr[LOGSUMMARY_HDR_NFIELD+1]
  ){
    return SQLITE_ERROR;
  }

  if( memcmp(&pLog->hdr, aHdr, sizeof(LogSummaryHdr)) ){
    if( pChanged ){
      *pChanged = 1;
    }
    memcpy(&pLog->hdr, aHdr, sizeof(LogSummaryHdr));
  }
  return SQLITE_OK;
}

/*
** Read the log-summary header from the log-summary file into structure 
** pLog->hdr. If attempting to verify the header checksum fails, try
** to recover the log before returning.
**
** If the log-summary header is successfully read, return SQLITE_OK. 
** Otherwise an SQLite error code.
*/
int logSummaryReadHdr(Log *pLog, int *pChanged){
  int rc;

  /* First try to read the header without a lock. Verify the checksum
  ** before returning. This will almost always work.  
  */
  if( SQLITE_OK==logSummaryTryHdr(pLog, pChanged) ){
    return SQLITE_OK;
  }

  /* If the first attempt to read the header failed, lock the log-summary
  ** file and try again. If the header checksum verification fails this
  ** time as well, run log recovery.
  */
  if( SQLITE_OK==(rc = logEnterMutex(pLog)) ){
    if( SQLITE_OK!=logSummaryTryHdr(pLog, pChanged) ){
      if( pChanged ){
        *pChanged = 1;
      }
      rc = logSummaryRecover(pLog->pSummary, pLog->pFd);
      if( rc==SQLITE_OK ){
        rc = logSummaryTryHdr(pLog, 0);
      }
    }
    logLeaveMutex(pLog);
  }

  return rc;
}

/*
** Lock a snapshot.
**
** If this call obtains a new read-lock and the database contents have been
** modified since the most recent call to LogCloseSnapshot() on this Log
** connection, then *pChanged is set to 1 before returning. Otherwise, it 
................................................................................
        pLog->isLocked = LOG_REGION_A;
      }
    }
    if( rc!=SQLITE_OK ){
      return rc;
    }













    rc = logSummaryReadHdr(pLog, pChanged);












    if( rc!=SQLITE_OK ){
      /* An error occured while attempting log recovery. */
      sqlite3LogCloseSnapshot(pLog);
    }
  }
  return rc;
}
................................................................................
    logLeaveMutex(pLog);
  }

  return SQLITE_OK;
}

/* 
** Checkpoint the database:
**
**   1. Wait for an EXCLUSIVE lock on regions B and C.
**   2. Wait for an EXCLUSIVE lock on region A.
**   3. Copy the contents of the log into the database file.
**   4. Zero the log-summary header (so new readers will ignore the log).
**   5. Drop the locks obtained in steps 1 and 2.
*/
int sqlite3LogCheckpoint(
  Log *pLog,                      /* Log connection */
  sqlite3_file *pFd,              /* File descriptor open on db file */
  u8 *zBuf,                       /* Temporary buffer to use */
  int (*xBusyHandler)(void *),    /* Pointer to busy-handler function */
  void *pBusyHandlerArg           /* Argument to pass to xBusyHandler */
){
  int rc;                         /* Return code */

  /* Wait for a write-lock on regions B and C. */
  do {
    rc = logLockRegion(pLog, LOG_REGION_B|LOG_REGION_C, LOG_WRLOCK);
  }while( rc==SQLITE_BUSY && xBusyHandler(pBusyHandlerArg) );
  if( rc!=SQLITE_OK ) return rc;

  /* Wait for a write-lock on region A. */
  do {
    rc = logLockRegion(pLog, LOG_REGION_A, LOG_WRLOCK);
  }while( rc==SQLITE_BUSY && xBusyHandler(pBusyHandlerArg) );
  if( rc!=SQLITE_OK ){
    logLockRegion(pLog, LOG_REGION_B|LOG_REGION_C, LOG_UNLOCK);
    return rc;
  }

  /* Copy data from the log to the database file. */
  rc = logSummaryReadHdr(pLog, 0);
  if( rc==SQLITE_OK ){
    rc = logCheckpoint(pLog, pFd, zBuf);
  }

  /* Release the locks. */
  logLockRegion(pLog, LOG_REGION_A|LOG_REGION_B|LOG_REGION_C, LOG_UNLOCK);
  return rc;
}

Changes to src/pragma.c.

1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
      sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1);
    }
  }else
#endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */

  if( sqlite3StrICmp(zLeft, "checkpoint")==0 ){
    sqlite3VdbeUsesBtree(v, iDb);
    sqlite3VdbeAddOp2(v, OP_Transaction, iDb, 1);
    sqlite3VdbeAddOp3(v, OP_Checkpoint, iDb, 0, 0);
  }else

#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST)
  /*
  ** Report the current state of file logs for all databases
  */







<







1382
1383
1384
1385
1386
1387
1388

1389
1390
1391
1392
1393
1394
1395
      sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1);
    }
  }else
#endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */

  if( sqlite3StrICmp(zLeft, "checkpoint")==0 ){
    sqlite3VdbeUsesBtree(v, iDb);

    sqlite3VdbeAddOp3(v, OP_Checkpoint, iDb, 0, 0);
  }else

#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST)
  /*
  ** Report the current state of file logs for all databases
  */

Changes to test/wal.test.

23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
...
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
...
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
...
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566

































































































567
568
      lappend ret $i
    }
  }
  set ret
}

proc reopen_db {} {
  db close
  file delete -force test.db test.db-wal
  sqlite3_wal db test.db
  #register_logtest
}
proc register_logtest {{db db}} {
  register_logsummary_module $db
  execsql { CREATE VIRTUAL TABLE temp.logsummary USING logsummary } $db
................................................................................
  file delete -force test.db test.db-wal
  sqlite3_wal db test.db
  execsql {
    PRAGMA page_size = 1024;
    CREATE TABLE t1(a, b);
    INSERT INTO t1 VALUES(1, 2);
  }

  list [file size test.db] [file size test.db-wal]
} [list 0 [expr (1024+20)*3]]
do_test wal-7.2 {
  execsql { PRAGMA checkpoint }
  list [file size test.db] [file size test.db-wal]
} [list 2048 [expr (1024+20)*3]]

# db close
# sqlite3_wal db test.db
# register_logsummary_module db
# # Warm-body tests of the virtual tables used for testing.
# # 
# do_test wal-8.1 {
#   execsql { CREATE VIRTUAL TABLE temp.logsummary USING logsummary }
#   execsql { CREATE VIRTUAL TABLE temp.logcontent USING logcontent }
#   execsql { CREATE VIRTUAL TABLE temp.loglock USING loglock }
#   execsql { SELECT * FROM logsummary }
# } [list [file join [pwd] test.db] 1 0 0 0 0 0 0]
# 
# do_test wal-8.2 {
#   sqlite3_wal db2 test.db
#   execsql { SELECT * FROM logsummary }
# } [list [file join [pwd] test.db] 2 0 0 0 0 0 0]
# do_test wal-8.3 {
#   db2 close
#   execsql { SELECT * FROM logsummary }
# } [list [file join [pwd] test.db] 1 0 0 0 0 0 0]
# do_test wal-8.4 {
#   execsql { INSERT INTO t1 VALUES(3, 4) }
#   execsql { SELECT * FROM logsummary }
# } [list [file join [pwd] test.db] 1 1024 2 2 3 0 0]
# do_test wal-8.5 {
#   execsql { PRAGMA checkpoint }
#   execsql { SELECT * FROM logsummary }
# } [list [file join [pwd] test.db] 1 1024 0 0 0 0 0]
# 
# do_test wal-8.6 {
#   execsql { INSERT INTO t1 VALUES(5, 6) }
#   execsql { PRAGMA checkpoint('1 1 1') }
#   execsql { SELECT * FROM logsummary }
# } [list [file join [pwd] test.db] 1 1024 2 0 3 0 0]
# do_test wal-8.7 {
#   execsql { SELECT logpage, dbpage FROM logcontent }
# } {}
# do_test wal-8.8 {
#   execsql { INSERT INTO t1 VALUES(7, 8) }
#   execsql { SELECT logpage, dbpage FROM logcontent }
# } {4 T:4 5 2}
# do_test wal-8.9 {
#   execsql { SELECT * FROM logsummary }
# } [list [file join [pwd] test.db] 1 1024 2 4 5 0 0]
# do_test wal-8.10 {
#   execsql { SELECT * FROM loglock }
# } [list [file join [pwd] test.db] 0 0 0]
# do_test wal-8.11 {
#   execsql { BEGIN; SELECT * FROM t1; }
#   execsql { SELECT * FROM loglock }
# } [list [file join [pwd] test.db] 0 0 4]
# 
# # Try making the log wrap around.
# #
# reopen_db
# 
# do_test wal-9.1 {
#   execsql {
#     BEGIN;
#     CREATE TABLE t1(a PRIMARY KEY, b);
#   }
#   for {set i 0} {$i < 100} {incr i} {
#     execsql { INSERT INTO t1 VALUES($i, randomblob(100)) }
#   }
#   execsql COMMIT
#   execsql { SELECT * FROM logsummary }
# } [list [file join [pwd] test.db] 1 1024 2 2 17 0 0]
# do_test wal-9.2 {
#   execsql { SELECT logpage, dbpage FROM logcontent }
# } {2 T:2 3 1 4 2 5 3 6 4 7 5 8 6 9 7 10 8 11 9 12 10 13 11 14 12 15 13 16 14 17 15}
# do_test wal-9.3 {
#   execsql { PRAGMA checkpoint('12, 12') }
#   execsql { SELECT * FROM logsummary }
# } [list [file join [pwd] test.db] 1 1024 2 15 17 0 0]
# do_test wal-9.4 {
#   execsql { SELECT logpage, dbpage FROM logcontent }
# } {15 13 16 14 17 15}
# do_test wal-9.5 {
#   execsql { SELECT count(*) FROM t1 }
# } {100}
# do_test wal-9.6 {
#   execsql { INSERT INTO t1 VALUES(100, randomblob(100)) }
#   execsql { SELECT * FROM logsummary }
# } [list [file join [pwd] test.db] 1 1024 2 15 20 0 0]
# 
# do_test wal-9.7 {
#   execsql { SELECT count(*) FROM t1 }
# } {101}
# do_test wal-9.8 {
#   db close
#   sqlite3_wal db test.db
#   register_logtest
#   execsql { SELECT count(*) FROM t1 }
# } {101}
# do_test wal-9.9 {
#   execsql { SELECT * FROM logsummary }
# } [list [file join [pwd] test.db] 1 0 0 0 0 0 0]
# 
# reopen_db
# do_test wal-10.1 {
#   execsql {
#     PRAGMA page_size = 1024;
#     CREATE TABLE t1(x PRIMARY KEY);
#     INSERT INTO t1 VALUES(randomblob(900));
#     INSERT INTO t1 VALUES(randomblob(900));
#     INSERT INTO t1 SELECT randomblob(900) FROM t1;         -- 4
#     INSERT INTO t1 SELECT randomblob(900) FROM t1;         -- 8
#     INSERT INTO t1 SELECT randomblob(900) FROM t1;         -- 16
#   }
#   list [file size test.db] [file size test.db-wal]
# } {0 55296}
# do_test wal-10.2 {
#   execsql { PRAGMA checkpoint('20 30') }
# } {}
# do_test wal-10.3 {
#   execsql { SELECT * FROM logsummary }
# } [list [file join [pwd] test.db] 1 1024 34 38 54 0 0]
# do_test wal-10.4 {
#   execsql { SELECT dbpage FROM logcontent }
# } {21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37}
# do_test wal-10.5 {
#   execsql { INSERT INTO t1 VALUES(randomblob(900)) }
#   execsql { SELECT * FROM logsummary }
# } [list [file join [pwd] test.db] 1 1024 34 38 6 34 54]
# do_test wal-10.6 {
#   execsql { SELECT count(*) FROM t1 WHERE x NOT NULL }
# } {17}
# do_test wal-10.8 {
#   execsql { SELECT logpage FROM logcontent }
# } [range 38 54  1 6]
# do_test wal-10.9 {
#   execsql { INSERT INTO t1 SELECT randomblob(900) FROM t1 }
#   execsql { SELECT * FROM logsummary }
# } [list [file join [pwd] test.db] 1 1024 34 38 68 34 54]
# 
# do_test wal-10.10 {
#   execsql { SELECT logpage FROM logcontent }
# } [range 38 54  1 33  55 68]
# 
# do_test wal-10.11 {
#   execsql { SELECT count(*) FROM t1 WHERE x NOT NULL }
# } {34}
# 
# do_test wal-10.12 {
#   execsql { PRAGMA checkpoint('35 35') }
# } {}
# do_test wal-10.13 {
#   execsql { SELECT logpage FROM logcontent }
# } [range 22 68]
# do_test wal-10.13a {
#   execsql { SELECT dbpage FROM logcontent }
# } [list \
#   50 51 52 53 54 55 56 57 58 59 60 61    \
#   0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 \
#   62 63 64 65 66 67 68 69 70 71 72 73 74 75 \
# ]
# 
# do_test wal-10.14 {
#   execsql { SELECT count(*) FROM t1 WHERE x NOT NULL }
# } {34}
# do_test wal-10.15 {
#   execsql { PRAGMA integrity_check }
# } {ok}
# do_test wal-10.16 {
#   execsql { PRAGMA checkpoint('20 20') }
#   execsql { SELECT * FROM logsummary }
# } [list [file join [pwd] test.db] 1 1024 7 63 68 0 0]
# do_test wal-10.17 {
#   execsql { SELECT logpage FROM logcontent }
# } [range 63 68]
# do_test wal-10.17a {
#   execsql { SELECT dbpage FROM logcontent }
# } {70 71 72 73 74 75}
# 
# do_test wal-10.18 {
#   execsql { INSERT INTO t1 SELECT randomblob(900) FROM t1 }
#   execsql { SELECT logpage FROM logcontent }
# } [range 63 147]
# integrity_check wal-10.19
# 
# do_test wal-10.20 {
#   execsql { PRAGMA checkpoint('52 52') }
#   execsql { SELECT logpage FROM logcontent }
# } [range 116 147]
# do_test wal-10.20a {
#   execsql { SELECT * FROM logsummary }
# } [list [file join [pwd] test.db] 1 1024 69 116 147 0 0]
# integrity_check wal-10.20.integrity
# 
# do_test wal-10.21 {
#   execsql { INSERT INTO t1 VALUES( randomblob(900) ) }
#   execsql { SELECT logpage FROM logcontent }
# } [range 116 152]
# do_test wal-10.22 {
#   execsql { PRAGMA integrity_check }
# } {ok}
# 
# file delete -force testX.db testX.db-wal
# file copy test.db testX.db
# file copy test.db-wal testX.db-wal
# do_test wal-10.23 {
#   sqlite3_wal db2 testX.db
#   register_logtest db2
#   execsql { SELECT logpage FROM logcontent WHERE db LIKE '%testX%' } db2
# } [range 34 54  1 33  55 152]
# 
# do_test wal-10.24 {
#   execsql { PRAGMA integrity_check } db2
# } {ok}
# db2 close
# 
# do_test wal-11.1 {
#   reopen_db
#   sqlite3_wal db2 test.db
# 
#   execsql {
#     BEGIN;
#       CREATE TABLE t1(x);
#       CREATE TABLE t2(x PRIMARY KEY);
#       INSERT INTO t1 VALUES(randomblob(900));
#       INSERT INTO t1 VALUES(randomblob(900));
#       INSERT INTO t1 SELECT randomblob(900) FROM t1;       /*  4 */
#       INSERT INTO t1 SELECT randomblob(900) FROM t1;       /*  8 */
#       INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 16 */
#       INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 32 */
#       INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 64 */
# 
#       INSERT INTO t2 VALUES('x');
#       INSERT INTO t2 VALUES('y');
#       INSERT INTO t2 VALUES('z');
#     COMMIT;
#     SELECT * FROM logsummary;
#   }
# } [list [file join [pwd] test.db] 2 1024 2 2 70 0 0]
# 
# do_test wal-11.2 {
#   execsql {
#     BEGIN; SELECT x FROM t2;
#   } db2
# } {x y z}
# do_test wal-11.2 {
#   execsql {
#     INSERT INTO t1 VALUES(randomblob(900));
#     PRAGMA checkpoint('10 100');
#     INSERT INTO t1 VALUES(randomblob(900));
#     INSERT INTO t2 VALUES('0');
#     SELECT * FROM logsummary;
#   }
# } [list [file join [pwd] test.db] 2 1024 71 71 7 71 73]
# do_test wal-12.3 {
#   execsql { PRAGMA integrity_check } db2
# } {ok}
# db2 close


# Execute some transactions in auto-vacuum mode to test database file
# truncation.
#
do_test wal-12.1 {
  reopen_db
  execsql {
    PRAGMA auto_vacuum = 1;
    PRAGMA auto_vacuum;
  }
} {1}
do_test wal-12.2 {
  execsql {
    PRAGMA page_size = 1024;
    CREATE TABLE t1(x);
    INSERT INTO t1 VALUES(randomblob(900));
    INSERT INTO t1 VALUES(randomblob(900));
    INSERT INTO t1 SELECT randomblob(900) FROM t1;       /*  4 */
    INSERT INTO t1 SELECT randomblob(900) FROM t1;       /*  8 */
................................................................................
    INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 16 */
    INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 32 */
    INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 64 */
    PRAGMA checkpoint;
  }
  file size test.db
} [expr 68*1024]
do_test wal-12.3 {
  execsql { 
    DELETE FROM t1 WHERE rowid<54;
    PRAGMA checkpoint;
  }
  file size test.db
} [expr 14*1024]

# Run some "warm-body" tests to ensure that log-summary files with more
# than 256 entries (log summaries that contain index blocks) work Ok.
#
do_test wal-13.1 {
  reopen_db
  execsql {
    PRAGMA page_size = 1024;
    CREATE TABLE t1(x PRIMARY KEY);
    INSERT INTO t1 VALUES(randomblob(900));
    INSERT INTO t1 VALUES(randomblob(900));
    INSERT INTO t1 SELECT randomblob(900) FROM t1;       /*  4 */
................................................................................
    INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 32 */
    INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 64 */
    INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 128 */
    INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 256 */
  }
  file size test.db
} 0
do_test wal-13.2 {
  sqlite3_wal db2 test.db
  execsql {PRAGMA integrity_check } db2
} {ok}

do_test wal-13.3 {
  file delete -force test2.db test2.db-wal
  file copy test.db test2.db
  file copy test.db-wal test2.db-wal
  sqlite3_wal db3 test2.db 
  execsql {PRAGMA integrity_check } db3
} {ok}
db3 close

do_test wal-13.4 {
breakpoint
  execsql { PRAGMA checkpoint }
  db2 close
  sqlite3_wal db2 test.db
  execsql {PRAGMA integrity_check } db2
} {ok}

foreach handle {db db2 db3} { catch { $handle close } }
unset handle


































































































finish_test








|







 







<







<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<



|






|







 







|










|







 







|




|








|
<









>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>


23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
...
219
220
221
222
223
224
225

226
227
228
229
230
231
232































































































































































































































































233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
...
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
...
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300

301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
      lappend ret $i
    }
  }
  set ret
}

proc reopen_db {} {
  catch { db close }
  file delete -force test.db test.db-wal
  sqlite3_wal db test.db
  #register_logtest
}
proc register_logtest {{db db}} {
  register_logsummary_module $db
  execsql { CREATE VIRTUAL TABLE temp.logsummary USING logsummary } $db
................................................................................
  file delete -force test.db test.db-wal
  sqlite3_wal db test.db
  execsql {
    PRAGMA page_size = 1024;
    CREATE TABLE t1(a, b);
    INSERT INTO t1 VALUES(1, 2);
  }

  list [file size test.db] [file size test.db-wal]
} [list 0 [expr (1024+20)*3]]
do_test wal-7.2 {
  execsql { PRAGMA checkpoint }
  list [file size test.db] [file size test.db-wal]
} [list 2048 [expr (1024+20)*3]]
































































































































































































































































# Execute some transactions in auto-vacuum mode to test database file
# truncation.
#
do_test wal-8.1 {
  reopen_db
  execsql {
    PRAGMA auto_vacuum = 1;
    PRAGMA auto_vacuum;
  }
} {1}
do_test wal-8.2 {
  execsql {
    PRAGMA page_size = 1024;
    CREATE TABLE t1(x);
    INSERT INTO t1 VALUES(randomblob(900));
    INSERT INTO t1 VALUES(randomblob(900));
    INSERT INTO t1 SELECT randomblob(900) FROM t1;       /*  4 */
    INSERT INTO t1 SELECT randomblob(900) FROM t1;       /*  8 */
................................................................................
    INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 16 */
    INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 32 */
    INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 64 */
    PRAGMA checkpoint;
  }
  file size test.db
} [expr 68*1024]
do_test wal-8.3 {
  execsql { 
    DELETE FROM t1 WHERE rowid<54;
    PRAGMA checkpoint;
  }
  file size test.db
} [expr 14*1024]

# Run some "warm-body" tests to ensure that log-summary files with more
# than 256 entries (log summaries that contain index blocks) work Ok.
#
do_test wal-9.1 {
  reopen_db
  execsql {
    PRAGMA page_size = 1024;
    CREATE TABLE t1(x PRIMARY KEY);
    INSERT INTO t1 VALUES(randomblob(900));
    INSERT INTO t1 VALUES(randomblob(900));
    INSERT INTO t1 SELECT randomblob(900) FROM t1;       /*  4 */
................................................................................
    INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 32 */
    INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 64 */
    INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 128 */
    INSERT INTO t1 SELECT randomblob(900) FROM t1;       /* 256 */
  }
  file size test.db
} 0
do_test wal-9.2 {
  sqlite3_wal db2 test.db
  execsql {PRAGMA integrity_check } db2
} {ok}

do_test wal-9.3 {
  file delete -force test2.db test2.db-wal
  file copy test.db test2.db
  file copy test.db-wal test2.db-wal
  sqlite3_wal db3 test2.db 
  execsql {PRAGMA integrity_check } db3
} {ok}
db3 close

do_test wal-9.4 {

  execsql { PRAGMA checkpoint }
  db2 close
  sqlite3_wal db2 test.db
  execsql {PRAGMA integrity_check } db2
} {ok}

foreach handle {db db2 db3} { catch { $handle close } }
unset handle

#-------------------------------------------------------------------------
# The following block of tests - wal-10.* - test that the WAL locking 
# scheme works for clients in a single process.
#
reopen_db
sqlite3_wal db2 test.db
sqlite3_wal db3 test.db

do_test wal-10.1 {
  execsql {
    CREATE TABLE t1(a, b);
    INSERT INTO t1 VALUES(1, 2);
    BEGIN;
      INSERT INTO t1 VALUES(3, 4);
  }
  execsql "SELECT * FROM t1" db2
} {1 2}
do_test wal-10.2 {
  execsql { COMMIT }
  execsql "SELECT * FROM t1" db2
} {1 2 3 4}
do_test wal-10.3 {
  execsql { 
    BEGIN;
      SELECT * FROM t1;
  } db2
} {1 2 3 4}
do_test wal-10.4 {
  catchsql { PRAGMA checkpoint } 
} {1 {database is locked}}
do_test wal-10.5 {
  execsql { INSERT INTO t1 VALUES(5, 6) }
  execsql { SELECT * FROM t1 } db2
} {1 2 3 4}

# Connection [db2] is holding a lock on a snapshot, preventing [db] from
# checkpointing the database. Add a busy-handler to [db]. If [db2] completes
# its transaction from within the busy-handler, [db] is able to complete
# the checkpoint operation.
#
proc busyhandler x {
  if {$x==4} {
    execsql { COMMIT } db2
  }
  if {$x<5} {return 0}
  return 1
}
db busy busyhandler
do_test wal-10.6 {
  execsql { PRAGMA checkpoint } 
} {}

# Similar to the test above. Except this time, a new read transaction is
# started (db3) while the checkpointer is waiting for an old one to finish.
# The checkpointer can finish, but any subsequent write operations must
# wait until after db3 has closed the read transaction.
#
db busy {}
do_test wal-10.7 {
  execsql { 
    BEGIN;
      SELECT * FROM t1;
  } db2
} {1 2 3 4 5 6}
do_test wal-10.8 {
  execsql { INSERT INTO t1 VALUES(7, 8) }
  catchsql { PRAGMA checkpoint } 
} {1 {database is locked}}
proc busyhandler x {
  if {$x==3} { execsql { BEGIN; SELECT * FROM t1 } db3 }
  if {$x==4} { execsql { COMMIT } db2 }
  if {$x<5}  { return 0 }
  return 1
}
db busy busyhandler
do_test wal-10.9 {
  execsql { PRAGMA checkpoint } 
} {}
do_test wal-10.10 {
  execsql { SELECT * FROM t1 } db3
} {1 2 3 4 5 6 7 8}
do_test wal-10.11 {
  catchsql { INSERT INTO t1 VALUES(9, 10) }
} {1 {database is locked}}
do_test wal-10.12 {
  execsql { SELECT * FROM t1 }
} {1 2 3 4 5 6 7 8}
do_test wal-10.13 {
  execsql { COMMIT } db3
} {}
do_test wal-10.14 {
  execsql { INSERT INTO t1 VALUES(9, 10) }
  execsql { SELECT * FROM t1 }
} {1 2 3 4 5 6 7 8 9 10}

foreach handle {db db2 db3} { catch { $handle close } }
unset handle
finish_test

Changes to test/walthread.test.

58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
..
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
#   1) Modifying the contents of t1 (inserting, updating, deleting rows).
#   2) Appending a new row to the table containing the md5sum() of all
#      rows in the table.
#
# Each of the N threads runs N read transactions followed by a single write
# transaction in a loop as fast as possible.
#
# There is also a single checkpointer thread. It runs the following loop:
#
#   1) Execute "CHECKPOINT main 32 -1 1"
#   2) Sleep for 500 ms.
#

set thread_program {
  proc rest {ms} {
................................................................................
    set rc [sqlite3_step $stmt]
    if {$rc eq "SQLITE_ROW"} {
      set res [sqlite3_column_text $stmt 0]
    }
    set rc [sqlite3_finalize $stmt]

    if {$rc ne "SQLITE_OK"} {
      error $rc 
    }
    return $res
  }

  proc read_transaction {DB} {
    dosql $DB BEGIN








|







 







|







58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
..
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
#   1) Modifying the contents of t1 (inserting, updating, deleting rows).
#   2) Appending a new row to the table containing the md5sum() of all
#      rows in the table.
#
# Each of the N threads runs N read transactions followed by a single write
# transaction in a loop as fast as possible.
#
# Ther is also a single checkpointer thread. It runs the following loop:
#
#   1) Execute "CHECKPOINT main 32 -1 1"
#   2) Sleep for 500 ms.
#

set thread_program {
  proc rest {ms} {
................................................................................
    set rc [sqlite3_step $stmt]
    if {$rc eq "SQLITE_ROW"} {
      set res [sqlite3_column_text $stmt 0]
    }
    set rc [sqlite3_finalize $stmt]

    if {$rc ne "SQLITE_OK"} {
      error "$rc: [sqlite3_errmsg $DB]"
    }
    return $res
  }

  proc read_transaction {DB} {
    dosql $DB BEGIN