SQLite

Check-in [3d159939cc]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add mutexes to fix a race condition in wal.c. This isn't a very good fix.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | wal
Files: files | file ages | folders
SHA1: 3d159939cc2beb18c4ca0c8e9a99a75d4107e6e4
User & Date: dan 2010-04-26 10:40:52.000
Context
2010-04-26
12:39
Add the "wal" permutation to run existing test files savepoint.test and savepoint2.test in WAL mode. (check-in: 205e5d8ac0 user: dan tags: wal)
10:40
Add mutexes to fix a race condition in wal.c. This isn't a very good fix. (check-in: 3d159939cc user: dan tags: wal)
00:19
Change the names of the log.c and log.h source files to wal.c and wal.h. (check-in: 56fe5d7624 user: drh tags: wal)
Changes
Unified Diff Ignore Whitespace Patch
Changes to src/test1.c.
5109
5110
5111
5112
5113
5114
5115

5116
5117
5118
5119
5120
5121
5122
  extern int sqlite3WhereTrace;
  extern int sqlite3OSTrace;
  extern int sqlite3VdbeAddopTrace;
#endif
#ifdef SQLITE_TEST
  extern char sqlite3_query_plan[];
  static char *query_plan = sqlite3_query_plan;

#ifdef SQLITE_ENABLE_FTS3
  extern int sqlite3_fts3_enable_parentheses;
#endif
#endif

  for(i=0; i<sizeof(aCmd)/sizeof(aCmd[0]); i++){
    Tcl_CreateCommand(interp, aCmd[i].zName, aCmd[i].xProc, 0, 0);







>







5109
5110
5111
5112
5113
5114
5115
5116
5117
5118
5119
5120
5121
5122
5123
  extern int sqlite3WhereTrace;
  extern int sqlite3OSTrace;
  extern int sqlite3VdbeAddopTrace;
#endif
#ifdef SQLITE_TEST
  extern char sqlite3_query_plan[];
  static char *query_plan = sqlite3_query_plan;
  extern int sqlite3_walsummary_mmap_incr;                 /* In wal.c */
#ifdef SQLITE_ENABLE_FTS3
  extern int sqlite3_fts3_enable_parentheses;
#endif
#endif

  for(i=0; i<sizeof(aCmd)/sizeof(aCmd[0]); i++){
    Tcl_CreateCommand(interp, aCmd[i].zName, aCmd[i].xProc, 0, 0);
5168
5169
5170
5171
5172
5173
5174


5175
5176
5177
5178
5179
5180
5181
#if SQLITE_OS_WIN
  Tcl_LinkVar(interp, "sqlite_os_type",
      (char*)&sqlite3_os_type, TCL_LINK_INT);
#endif
#ifdef SQLITE_TEST
  Tcl_LinkVar(interp, "sqlite_query_plan",
      (char*)&query_plan, TCL_LINK_STRING|TCL_LINK_READ_ONLY);


#endif
#ifdef SQLITE_DEBUG
  Tcl_LinkVar(interp, "sqlite_addop_trace",
      (char*)&sqlite3VdbeAddopTrace, TCL_LINK_INT);
  Tcl_LinkVar(interp, "sqlite_where_trace",
      (char*)&sqlite3WhereTrace, TCL_LINK_INT);
  Tcl_LinkVar(interp, "sqlite_os_trace",







>
>







5169
5170
5171
5172
5173
5174
5175
5176
5177
5178
5179
5180
5181
5182
5183
5184
#if SQLITE_OS_WIN
  Tcl_LinkVar(interp, "sqlite_os_type",
      (char*)&sqlite3_os_type, TCL_LINK_INT);
#endif
#ifdef SQLITE_TEST
  Tcl_LinkVar(interp, "sqlite_query_plan",
      (char*)&query_plan, TCL_LINK_STRING|TCL_LINK_READ_ONLY);
  Tcl_LinkVar(interp, "sqlite_walsummary_mmap_incr",
      (char*)&sqlite3_walsummary_mmap_incr, TCL_LINK_INT);
#endif
#ifdef SQLITE_DEBUG
  Tcl_LinkVar(interp, "sqlite_addop_trace",
      (char*)&sqlite3VdbeAddopTrace, TCL_LINK_INT);
  Tcl_LinkVar(interp, "sqlite_where_trace",
      (char*)&sqlite3WhereTrace, TCL_LINK_INT);
  Tcl_LinkVar(interp, "sqlite_os_trace",
Changes to src/wal.c.
115
116
117
118
119
120
121




122

123
124
125
126
127
128
129
/*
** If using mmap() to access a shared (or otherwise) log-summary file, then
** the mapping size is incremented in units of the following size.
**
** A 64 KB log-summary mapping corresponds to a log file containing over
** 13000 frames, so the mapping size does not need to be increased often.
*/




#define LOGSUMMARY_MMAP_INCREMENT (64*1024)


/*
** There is one instance of this structure for each log-summary object
** that this process has a connection to. They are stored in a linked
** list starting at pLogSummary (global variable).
**
** TODO: LogSummary.fd is a unix file descriptor. Unix APIs are used 







>
>
>
>
|
>







115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
/*
** If using mmap() to access a shared (or otherwise) log-summary file, then
** the mapping size is incremented in units of the following size.
**
** A 64 KB log-summary mapping corresponds to a log file containing over
** 13000 frames, so the mapping size does not need to be increased often.
*/
#ifdef SQLITE_TEST
int sqlite3_walsummary_mmap_incr = 128;
# define LOGSUMMARY_MMAP_INCREMENT sqlite3_walsummary_mmap_incr
#else
# define LOGSUMMARY_MMAP_INCREMENT (64*1024)
#endif

/*
** There is one instance of this structure for each log-summary object
** that this process has a connection to. They are stored in a linked
** list starting at pLogSummary (global variable).
**
** TODO: LogSummary.fd is a unix file descriptor. Unix APIs are used 
607
608
609
610
611
612
613
614
615
616

617
618
619
620
621
622
623
** value of iFrame is always exactly one more than the value passed to
** the previous call), but that restriction is not enforced or asserted
** here.
*/
static void logSummaryAppend(LogSummary *pSummary, u32 iFrame, u32 iPage){
  u32 iSlot = logSummaryEntry(iFrame);

  if( (iSlot+128)>=pSummary->nData ){
    int nByte = pSummary->nData*4 + LOGSUMMARY_MMAP_INCREMENT;


    sqlite3_mutex_enter(pSummary->mutex);
    munmap(pSummary->aData, pSummary->nData*4);
    pSummary->aData = 0;
    logSummaryMap(pSummary, nByte);
    sqlite3_mutex_leave(pSummary->mutex);
  }








|


>







612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
** value of iFrame is always exactly one more than the value passed to
** the previous call), but that restriction is not enforced or asserted
** here.
*/
static void logSummaryAppend(LogSummary *pSummary, u32 iFrame, u32 iPage){
  u32 iSlot = logSummaryEntry(iFrame);

  while( (iSlot+128)>=pSummary->nData ){
    int nByte = pSummary->nData*4 + LOGSUMMARY_MMAP_INCREMENT;

    /* Unmap and remap the log-summary file. */
    sqlite3_mutex_enter(pSummary->mutex);
    munmap(pSummary->aData, pSummary->nData*4);
    pSummary->aData = 0;
    logSummaryMap(pSummary, nByte);
    sqlite3_mutex_leave(pSummary->mutex);
  }

1373
1374
1375
1376
1377
1378
1379
1380



1381
1382
1383
1384
1385
1386
1387
1388
**
** If the checksum cannot be verified return SQLITE_ERROR.
*/
int logSummaryTryHdr(Log *pLog, int *pChanged){
  u32 aCksum[2] = {1, 1};
  u32 aHdr[LOGSUMMARY_HDR_NFIELD+2];

  /* First try to read the header without a lock. Verify the checksum



  ** before returning. This will almost always work.  
  */
  memcpy(aHdr, pLog->pSummary->aData, sizeof(aHdr));
  logChecksumBytes((u8*)aHdr, sizeof(u32)*LOGSUMMARY_HDR_NFIELD, aCksum);
  if( aCksum[0]!=aHdr[LOGSUMMARY_HDR_NFIELD]
   || aCksum[1]!=aHdr[LOGSUMMARY_HDR_NFIELD+1]
  ){
    return SQLITE_ERROR;







|
>
>
>
|







1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
**
** If the checksum cannot be verified return SQLITE_ERROR.
*/
int logSummaryTryHdr(Log *pLog, int *pChanged){
  u32 aCksum[2] = {1, 1};
  u32 aHdr[LOGSUMMARY_HDR_NFIELD+2];

  /* Read the header. The caller may or may not have locked the log-summary
  ** file, meaning it is possible that an inconsistent snapshot is read
  ** from the file. If this happens, return SQLITE_ERROR. The caller will
  ** retry. Or, if the caller has already locked the file and the header
  ** still looks inconsistent, it will run recovery.
  */
  memcpy(aHdr, pLog->pSummary->aData, sizeof(aHdr));
  logChecksumBytes((u8*)aHdr, sizeof(u32)*LOGSUMMARY_HDR_NFIELD, aCksum);
  if( aCksum[0]!=aHdr[LOGSUMMARY_HDR_NFIELD]
   || aCksum[1]!=aHdr[LOGSUMMARY_HDR_NFIELD+1]
  ){
    return SQLITE_ERROR;
1406
1407
1408
1409
1410
1411
1412




1413

1414
1415
1416

1417
1418
1419
1420
1421
1422
1423
** Otherwise an SQLite error code.
*/
int logSummaryReadHdr(Log *pLog, int *pChanged){
  int rc;

  /* First try to read the header without a lock. Verify the checksum
  ** before returning. This will almost always work.  




  */

  if( SQLITE_OK==logSummaryTryHdr(pLog, pChanged) ){
    return SQLITE_OK;
  }


  /* If the first attempt to read the header failed, lock the log-summary
  ** file and try again. If the header checksum verification fails this
  ** time as well, run log recovery.
  */
  if( SQLITE_OK==(rc = logEnterMutex(pLog)) ){
    if( SQLITE_OK!=logSummaryTryHdr(pLog, pChanged) ){







>
>
>
>

>



>







1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
** Otherwise an SQLite error code.
*/
int logSummaryReadHdr(Log *pLog, int *pChanged){
  int rc;

  /* First try to read the header without a lock. Verify the checksum
  ** before returning. This will almost always work.  
  **
  ** TODO: Doing this causes a race-condition with the code that resizes
  ** the mapping. Unless Log.pSummary->mutex is held, it is possible that 
  ** LogSummary.aData is invalid.
  */
#if 0
  if( SQLITE_OK==logSummaryTryHdr(pLog, pChanged) ){
    return SQLITE_OK;
  }
#endif

  /* If the first attempt to read the header failed, lock the log-summary
  ** file and try again. If the header checksum verification fails this
  ** time as well, run log recovery.
  */
  if( SQLITE_OK==(rc = logEnterMutex(pLog)) ){
    if( SQLITE_OK!=logSummaryTryHdr(pLog, pChanged) ){
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508



1509
1510
1511
1512
1513
1514
1515
}

/* 
** Read a page from the log, if it is present. 
*/
int sqlite3WalRead(Log *pLog, Pgno pgno, int *pInLog, u8 *pOut){
  u32 iRead = 0;
  u32 *aData = pLog->pSummary->aData;
  int iFrame = (pLog->hdr.iLastPg & 0xFFFFFF00);

  assert( pLog->isLocked );




  /* Do a linear search of the unindexed block of page-numbers (if any) 
  ** at the end of the log-summary. An alternative to this would be to
  ** build an index in private memory each time a read transaction is
  ** opened on a new snapshot.
  */
  if( pLog->hdr.iLastPg ){







|



>
>
>







1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
}

/* 
** Read a page from the log, if it is present. 
*/
int sqlite3WalRead(Log *pLog, Pgno pgno, int *pInLog, u8 *pOut){
  u32 iRead = 0;
  u32 *aData; 
  int iFrame = (pLog->hdr.iLastPg & 0xFFFFFF00);

  assert( pLog->isLocked );

  sqlite3_mutex_enter(pLog->pSummary->mutex);
  aData = pLog->pSummary->aData;

  /* Do a linear search of the unindexed block of page-numbers (if any) 
  ** at the end of the log-summary. An alternative to this would be to
  ** build an index in private memory each time a read transaction is
  ** opened on a new snapshot.
  */
  if( pLog->hdr.iLastPg ){
1544
1545
1546
1547
1548
1549
1550


1551
1552
1553
1554
1555
1556
1557
        iLow = iTest+1;
      }else{
        iHigh = iTest-1;
      }
    }
  }
  assert( iRead==0 || aData[logSummaryEntry(iRead)]==pgno );



  /* If iRead is non-zero, then it is the log frame number that contains the
  ** required page. Read and return data from the log file.
  */
  if( iRead ){
    i64 iOffset = logFrameOffset(iRead, pLog->hdr.pgsz) + LOG_FRAME_HDRSIZE;
    *pInLog = 1;







>
>







1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
        iLow = iTest+1;
      }else{
        iHigh = iTest-1;
      }
    }
  }
  assert( iRead==0 || aData[logSummaryEntry(iRead)]==pgno );

  sqlite3_mutex_leave(pLog->pSummary->mutex);

  /* If iRead is non-zero, then it is the log frame number that contains the
  ** required page. Read and return data from the log file.
  */
  if( iRead ){
    i64 iOffset = logFrameOffset(iRead, pLog->hdr.pgsz) + LOG_FRAME_HDRSIZE;
    *pInLog = 1;
Changes to test/wal.test.
783
784
785
786
787
788
789
790

791
792
793
794
795
do_test wal-13.5 {
  for {set i 0} {$i < 15} {incr i} {
    execsql { INSERT INTO t2 SELECT randomblob(400), randomblob(400) FROM t2 }
  }
  execsql { SELECT count(*) FROM t2 }
} [expr int(pow(2, 15))]
do_test wal-13.6 {
  file size test.db-wal-summary

} [expr 192*1024]


finish_test








|
>
|




783
784
785
786
787
788
789
790
791
792
793
794
795
796
do_test wal-13.5 {
  for {set i 0} {$i < 15} {incr i} {
    execsql { INSERT INTO t2 SELECT randomblob(400), randomblob(400) FROM t2 }
  }
  execsql { SELECT count(*) FROM t2 }
} [expr int(pow(2, 15))]
do_test wal-13.6 {
  set sz [file size test.db-wal-summary]
  expr {$sz<=(3*64*1024) && $sz>(2*64*1024)}
} {1}


finish_test

Changes to test/walthread.test.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20





21
22
23
24
25
26
27
# 2010 April 13
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file implements regression tests for SQLite library.  The
# focus of this file is testing the operation of the library in
# "PRAGMA journal_mode=WAL" mode.
#

set testdir [file dirname $argv0]

source $testdir/tester.tcl
if {[run_thread_tests]==0} { finish_test ; return }






do_test walthread-1.1 {
  execsql {
    PRAGMA journal_mode = WAL;
    PRAGMA lock_status;
    CREATE TABLE t1(x PRIMARY KEY);
    PRAGMA lock_status;
    INSERT INTO t1 VALUES(randomblob(100));












|







>
>
>
>
>







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
# 2010 April 13
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file implements regression tests for SQLite library.  The
# focus of this file is testing the operation of the library in
# "PRAGMA journal_mode=WAL" mode with multiple threads.
#

set testdir [file dirname $argv0]

source $testdir/tester.tcl
if {[run_thread_tests]==0} { finish_test ; return }

set sqlite_walsummary_mmap_incr 64

#--------------------------------------------------------------------------
# Initialize the database used for the multi-thread test.
#
do_test walthread-1.1 {
  execsql {
    PRAGMA journal_mode = WAL;
    PRAGMA lock_status;
    CREATE TABLE t1(x PRIMARY KEY);
    PRAGMA lock_status;
    INSERT INTO t1 VALUES(randomblob(100));
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
# transaction in a loop as fast as possible.
#
# Ther is also a single checkpointer thread. It runs the following loop:
#
#   1) Execute "CHECKPOINT main 32 -1 1"
#   2) Sleep for 500 ms.
#

set thread_program {
  proc rest {ms} {
    set ::rest 0
    after $ms {set ::rest 1}
    vwait ::rest
  }








<







70
71
72
73
74
75
76

77
78
79
80
81
82
83
# transaction in a loop as fast as possible.
#
# Ther is also a single checkpointer thread. It runs the following loop:
#
#   1) Execute "CHECKPOINT main 32 -1 1"
#   2) Sleep for 500 ms.
#

set thread_program {
  proc rest {ms} {
    set ::rest 0
    after $ms {set ::rest 1}
    vwait ::rest
  }