/ Check-in [3d159939]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add mutexes to fix a race condition in wal.c. This isn't a very good fix.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | wal
Files: files | file ages | folders
SHA1: 3d159939cc2beb18c4ca0c8e9a99a75d4107e6e4
User & Date: dan 2010-04-26 10:40:52
Context
2010-04-26
12:39
Add the "wal" permutation to run existing test files savepoint.test and savepoint2.test in WAL mode. check-in: 205e5d8a user: dan tags: wal
10:40
Add mutexes to fix a race condition in wal.c. This isn't a very good fix. check-in: 3d159939 user: dan tags: wal
00:19
Change the names of the log.c and log.h source files to wal.c and wal.h. check-in: 56fe5d76 user: drh tags: wal
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/test1.c.

5109
5110
5111
5112
5113
5114
5115

5116
5117
5118
5119
5120
5121
5122
....
5168
5169
5170
5171
5172
5173
5174


5175
5176
5177
5178
5179
5180
5181
  extern int sqlite3WhereTrace;
  extern int sqlite3OSTrace;
  extern int sqlite3VdbeAddopTrace;
#endif
#ifdef SQLITE_TEST
  extern char sqlite3_query_plan[];
  static char *query_plan = sqlite3_query_plan;

#ifdef SQLITE_ENABLE_FTS3
  extern int sqlite3_fts3_enable_parentheses;
#endif
#endif

  for(i=0; i<sizeof(aCmd)/sizeof(aCmd[0]); i++){
    Tcl_CreateCommand(interp, aCmd[i].zName, aCmd[i].xProc, 0, 0);
................................................................................
#if SQLITE_OS_WIN
  Tcl_LinkVar(interp, "sqlite_os_type",
      (char*)&sqlite3_os_type, TCL_LINK_INT);
#endif
#ifdef SQLITE_TEST
  Tcl_LinkVar(interp, "sqlite_query_plan",
      (char*)&query_plan, TCL_LINK_STRING|TCL_LINK_READ_ONLY);


#endif
#ifdef SQLITE_DEBUG
  Tcl_LinkVar(interp, "sqlite_addop_trace",
      (char*)&sqlite3VdbeAddopTrace, TCL_LINK_INT);
  Tcl_LinkVar(interp, "sqlite_where_trace",
      (char*)&sqlite3WhereTrace, TCL_LINK_INT);
  Tcl_LinkVar(interp, "sqlite_os_trace",







>







 







>
>







5109
5110
5111
5112
5113
5114
5115
5116
5117
5118
5119
5120
5121
5122
5123
....
5169
5170
5171
5172
5173
5174
5175
5176
5177
5178
5179
5180
5181
5182
5183
5184
  extern int sqlite3WhereTrace;
  extern int sqlite3OSTrace;
  extern int sqlite3VdbeAddopTrace;
#endif
#ifdef SQLITE_TEST
  extern char sqlite3_query_plan[];
  static char *query_plan = sqlite3_query_plan;
  extern int sqlite3_walsummary_mmap_incr;                 /* In wal.c */
#ifdef SQLITE_ENABLE_FTS3
  extern int sqlite3_fts3_enable_parentheses;
#endif
#endif

  for(i=0; i<sizeof(aCmd)/sizeof(aCmd[0]); i++){
    Tcl_CreateCommand(interp, aCmd[i].zName, aCmd[i].xProc, 0, 0);
................................................................................
#if SQLITE_OS_WIN
  Tcl_LinkVar(interp, "sqlite_os_type",
      (char*)&sqlite3_os_type, TCL_LINK_INT);
#endif
#ifdef SQLITE_TEST
  Tcl_LinkVar(interp, "sqlite_query_plan",
      (char*)&query_plan, TCL_LINK_STRING|TCL_LINK_READ_ONLY);
  Tcl_LinkVar(interp, "sqlite_walsummary_mmap_incr",
      (char*)&sqlite3_walsummary_mmap_incr, TCL_LINK_INT);
#endif
#ifdef SQLITE_DEBUG
  Tcl_LinkVar(interp, "sqlite_addop_trace",
      (char*)&sqlite3VdbeAddopTrace, TCL_LINK_INT);
  Tcl_LinkVar(interp, "sqlite_where_trace",
      (char*)&sqlite3WhereTrace, TCL_LINK_INT);
  Tcl_LinkVar(interp, "sqlite_os_trace",

Changes to src/wal.c.

115
116
117
118
119
120
121




122

123
124
125
126
127
128
129
...
607
608
609
610
611
612
613
614
615
616

617
618
619
620
621
622
623
....
1373
1374
1375
1376
1377
1378
1379
1380
1381



1382
1383
1384
1385
1386
1387
1388
....
1406
1407
1408
1409
1410
1411
1412




1413

1414
1415
1416

1417
1418
1419
1420
1421
1422
1423
....
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508



1509
1510
1511
1512
1513
1514
1515
....
1544
1545
1546
1547
1548
1549
1550


1551
1552
1553
1554
1555
1556
1557
/*
** If using mmap() to access a shared (or otherwise) log-summary file, then
** the mapping size is incremented in units of the following size.
**
** A 64 KB log-summary mapping corresponds to a log file containing over
** 13000 frames, so the mapping size does not need to be increased often.
*/




#define LOGSUMMARY_MMAP_INCREMENT (64*1024)


/*
** There is one instance of this structure for each log-summary object
** that this process has a connection to. They are stored in a linked
** list starting at pLogSummary (global variable).
**
** TODO: LogSummary.fd is a unix file descriptor. Unix APIs are used 
................................................................................
** value of iFrame is always exactly one more than the value passed to
** the previous call), but that restriction is not enforced or asserted
** here.
*/
static void logSummaryAppend(LogSummary *pSummary, u32 iFrame, u32 iPage){
  u32 iSlot = logSummaryEntry(iFrame);

  if( (iSlot+128)>=pSummary->nData ){
    int nByte = pSummary->nData*4 + LOGSUMMARY_MMAP_INCREMENT;


    sqlite3_mutex_enter(pSummary->mutex);
    munmap(pSummary->aData, pSummary->nData*4);
    pSummary->aData = 0;
    logSummaryMap(pSummary, nByte);
    sqlite3_mutex_leave(pSummary->mutex);
  }

................................................................................
**
** If the checksum cannot be verified return SQLITE_ERROR.
*/
int logSummaryTryHdr(Log *pLog, int *pChanged){
  u32 aCksum[2] = {1, 1};
  u32 aHdr[LOGSUMMARY_HDR_NFIELD+2];

  /* First try to read the header without a lock. Verify the checksum
  ** before returning. This will almost always work.  



  */
  memcpy(aHdr, pLog->pSummary->aData, sizeof(aHdr));
  logChecksumBytes((u8*)aHdr, sizeof(u32)*LOGSUMMARY_HDR_NFIELD, aCksum);
  if( aCksum[0]!=aHdr[LOGSUMMARY_HDR_NFIELD]
   || aCksum[1]!=aHdr[LOGSUMMARY_HDR_NFIELD+1]
  ){
    return SQLITE_ERROR;
................................................................................
** Otherwise an SQLite error code.
*/
int logSummaryReadHdr(Log *pLog, int *pChanged){
  int rc;

  /* First try to read the header without a lock. Verify the checksum
  ** before returning. This will almost always work.  




  */

  if( SQLITE_OK==logSummaryTryHdr(pLog, pChanged) ){
    return SQLITE_OK;
  }


  /* If the first attempt to read the header failed, lock the log-summary
  ** file and try again. If the header checksum verification fails this
  ** time as well, run log recovery.
  */
  if( SQLITE_OK==(rc = logEnterMutex(pLog)) ){
    if( SQLITE_OK!=logSummaryTryHdr(pLog, pChanged) ){
................................................................................
}

/* 
** Read a page from the log, if it is present. 
*/
int sqlite3WalRead(Log *pLog, Pgno pgno, int *pInLog, u8 *pOut){
  u32 iRead = 0;
  u32 *aData = pLog->pSummary->aData;
  int iFrame = (pLog->hdr.iLastPg & 0xFFFFFF00);

  assert( pLog->isLocked );




  /* Do a linear search of the unindexed block of page-numbers (if any) 
  ** at the end of the log-summary. An alternative to this would be to
  ** build an index in private memory each time a read transaction is
  ** opened on a new snapshot.
  */
  if( pLog->hdr.iLastPg ){
................................................................................
        iLow = iTest+1;
      }else{
        iHigh = iTest-1;
      }
    }
  }
  assert( iRead==0 || aData[logSummaryEntry(iRead)]==pgno );



  /* If iRead is non-zero, then it is the log frame number that contains the
  ** required page. Read and return data from the log file.
  */
  if( iRead ){
    i64 iOffset = logFrameOffset(iRead, pLog->hdr.pgsz) + LOG_FRAME_HDRSIZE;
    *pInLog = 1;







>
>
>
>
|
>







 







|


>







 







|
|
>
>
>







 







>
>
>
>

>



>







 







|



>
>
>







 







>
>







115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
...
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
....
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
....
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
....
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
....
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
/*
** If using mmap() to access a shared (or otherwise) log-summary file, then
** the mapping size is incremented in units of the following size.
**
** A 64 KB log-summary mapping corresponds to a log file containing over
** 13000 frames, so the mapping size does not need to be increased often.
*/
#ifdef SQLITE_TEST
int sqlite3_walsummary_mmap_incr = 128;
# define LOGSUMMARY_MMAP_INCREMENT sqlite3_walsummary_mmap_incr
#else
# define LOGSUMMARY_MMAP_INCREMENT (64*1024)
#endif

/*
** There is one instance of this structure for each log-summary object
** that this process has a connection to. They are stored in a linked
** list starting at pLogSummary (global variable).
**
** TODO: LogSummary.fd is a unix file descriptor. Unix APIs are used 
................................................................................
** value of iFrame is always exactly one more than the value passed to
** the previous call), but that restriction is not enforced or asserted
** here.
*/
static void logSummaryAppend(LogSummary *pSummary, u32 iFrame, u32 iPage){
  u32 iSlot = logSummaryEntry(iFrame);

  while( (iSlot+128)>=pSummary->nData ){
    int nByte = pSummary->nData*4 + LOGSUMMARY_MMAP_INCREMENT;

    /* Unmap and remap the log-summary file. */
    sqlite3_mutex_enter(pSummary->mutex);
    munmap(pSummary->aData, pSummary->nData*4);
    pSummary->aData = 0;
    logSummaryMap(pSummary, nByte);
    sqlite3_mutex_leave(pSummary->mutex);
  }

................................................................................
**
** If the checksum cannot be verified return SQLITE_ERROR.
*/
int logSummaryTryHdr(Log *pLog, int *pChanged){
  u32 aCksum[2] = {1, 1};
  u32 aHdr[LOGSUMMARY_HDR_NFIELD+2];

  /* Read the header. The caller may or may not have locked the log-summary
  ** file, meaning it is possible that an inconsistent snapshot is read
  ** from the file. If this happens, return SQLITE_ERROR. The caller will
  ** retry. Or, if the caller has already locked the file and the header
  ** still looks inconsistent, it will run recovery.
  */
  memcpy(aHdr, pLog->pSummary->aData, sizeof(aHdr));
  logChecksumBytes((u8*)aHdr, sizeof(u32)*LOGSUMMARY_HDR_NFIELD, aCksum);
  if( aCksum[0]!=aHdr[LOGSUMMARY_HDR_NFIELD]
   || aCksum[1]!=aHdr[LOGSUMMARY_HDR_NFIELD+1]
  ){
    return SQLITE_ERROR;
................................................................................
** Otherwise an SQLite error code.
*/
int logSummaryReadHdr(Log *pLog, int *pChanged){
  int rc;

  /* First try to read the header without a lock. Verify the checksum
  ** before returning. This will almost always work.  
  **
  ** TODO: Doing this causes a race-condition with the code that resizes
  ** the mapping. Unless Log.pSummary->mutex is held, it is possible that 
  ** LogSummary.aData is invalid.
  */
#if 0
  if( SQLITE_OK==logSummaryTryHdr(pLog, pChanged) ){
    return SQLITE_OK;
  }
#endif

  /* If the first attempt to read the header failed, lock the log-summary
  ** file and try again. If the header checksum verification fails this
  ** time as well, run log recovery.
  */
  if( SQLITE_OK==(rc = logEnterMutex(pLog)) ){
    if( SQLITE_OK!=logSummaryTryHdr(pLog, pChanged) ){
................................................................................
}

/* 
** Read a page from the log, if it is present. 
*/
int sqlite3WalRead(Log *pLog, Pgno pgno, int *pInLog, u8 *pOut){
  u32 iRead = 0;
  u32 *aData; 
  int iFrame = (pLog->hdr.iLastPg & 0xFFFFFF00);

  assert( pLog->isLocked );

  sqlite3_mutex_enter(pLog->pSummary->mutex);
  aData = pLog->pSummary->aData;

  /* Do a linear search of the unindexed block of page-numbers (if any) 
  ** at the end of the log-summary. An alternative to this would be to
  ** build an index in private memory each time a read transaction is
  ** opened on a new snapshot.
  */
  if( pLog->hdr.iLastPg ){
................................................................................
        iLow = iTest+1;
      }else{
        iHigh = iTest-1;
      }
    }
  }
  assert( iRead==0 || aData[logSummaryEntry(iRead)]==pgno );

  sqlite3_mutex_leave(pLog->pSummary->mutex);

  /* If iRead is non-zero, then it is the log frame number that contains the
  ** required page. Read and return data from the log file.
  */
  if( iRead ){
    i64 iOffset = logFrameOffset(iRead, pLog->hdr.pgsz) + LOG_FRAME_HDRSIZE;
    *pInLog = 1;

Changes to test/wal.test.

783
784
785
786
787
788
789
790
791

792
793
794
795
do_test wal-13.5 {
  for {set i 0} {$i < 15} {incr i} {
    execsql { INSERT INTO t2 SELECT randomblob(400), randomblob(400) FROM t2 }
  }
  execsql { SELECT count(*) FROM t2 }
} [expr int(pow(2, 15))]
do_test wal-13.6 {
  file size test.db-wal-summary
} [expr 192*1024]



finish_test








|
|
>




783
784
785
786
787
788
789
790
791
792
793
794
795
796
do_test wal-13.5 {
  for {set i 0} {$i < 15} {incr i} {
    execsql { INSERT INTO t2 SELECT randomblob(400), randomblob(400) FROM t2 }
  }
  execsql { SELECT count(*) FROM t2 }
} [expr int(pow(2, 15))]
do_test wal-13.6 {
  set sz [file size test.db-wal-summary]
  expr {$sz<=(3*64*1024) && $sz>(2*64*1024)}
} {1}


finish_test

Changes to test/walthread.test.

6
7
8
9
10
11
12
13
14
15
16
17
18
19
20





21
22
23
24
25
26
27
..
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file implements regression tests for SQLite library.  The
# focus of this file is testing the operation of the library in
# "PRAGMA journal_mode=WAL" mode.
#

set testdir [file dirname $argv0]

source $testdir/tester.tcl
if {[run_thread_tests]==0} { finish_test ; return }






do_test walthread-1.1 {
  execsql {
    PRAGMA journal_mode = WAL;
    PRAGMA lock_status;
    CREATE TABLE t1(x PRIMARY KEY);
    PRAGMA lock_status;
    INSERT INTO t1 VALUES(randomblob(100));
................................................................................
# transaction in a loop as fast as possible.
#
# Ther is also a single checkpointer thread. It runs the following loop:
#
#   1) Execute "CHECKPOINT main 32 -1 1"
#   2) Sleep for 500 ms.
#

set thread_program {
  proc rest {ms} {
    set ::rest 0
    after $ms {set ::rest 1}
    vwait ::rest
  }








|







>
>
>
>
>







 







<







6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
..
70
71
72
73
74
75
76

77
78
79
80
81
82
83
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file implements regression tests for SQLite library.  The
# focus of this file is testing the operation of the library in
# "PRAGMA journal_mode=WAL" mode with multiple threads.
#

set testdir [file dirname $argv0]

source $testdir/tester.tcl
if {[run_thread_tests]==0} { finish_test ; return }

set sqlite_walsummary_mmap_incr 64

#--------------------------------------------------------------------------
# Initialize the database used for the multi-thread test.
#
do_test walthread-1.1 {
  execsql {
    PRAGMA journal_mode = WAL;
    PRAGMA lock_status;
    CREATE TABLE t1(x PRIMARY KEY);
    PRAGMA lock_status;
    INSERT INTO t1 VALUES(randomblob(100));
................................................................................
# transaction in a loop as fast as possible.
#
# Ther is also a single checkpointer thread. It runs the following loop:
#
#   1) Execute "CHECKPOINT main 32 -1 1"
#   2) Sleep for 500 ms.
#

set thread_program {
  proc rest {ms} {
    set ::rest 0
    after $ms {set ::rest 1}
    vwait ::rest
  }