/ Check-in [b733afc1]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add some support for wal mode to the hack on this branch.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | server-edition
Files: files | file ages | folders
SHA3-256: b733afc1d0abc09861903ce8e27a8f2462ec871967f5d3dc2847b31bb28b55f3
User & Date: dan 2017-05-08 20:15:23
Context
2017-05-09
16:32
Fix a problem with wrapping the log file in server mode. check-in: 270b7d1e user: dan tags: server-edition
2017-05-08
20:15
Add some support for wal mode to the hack on this branch. check-in: b733afc1 user: dan tags: server-edition
2017-05-06
16:04
Update this branch with latest trunk changes. check-in: ed6bad67 user: dan tags: server-edition
Changes
Hide Diffs Unified Diffs Show Whitespace Changes Patch

Changes to src/pager.c.

5368
5369
5370
5371
5372
5373
5374


5375
5376
5377
5378

5379

5380
5381
5382
5383
5384
5385
5386
5387
5388
5389
5390
5391
5392
5393
5394
5395
5396
5397
5398
5399
5400
5401
....
7451
7452
7453
7454
7455
7456
7457
7458
7459
7460
7461
7462
7463
7464
7465
....
7546
7547
7548
7549
7550
7551
7552

7553
7554
7555
7556
7557
7558
7559
        ** to be the right size but is not actually valid. Avoid this
        ** possibility by unmapping the db here. */
        if( USEFETCH(pPager) ){
          sqlite3OsUnfetch(pPager->fd, 0, 0);
        }
      }
    }



    /* If there is a WAL file in the file-system, open this database in WAL
    ** mode. Otherwise, the following function call is a no-op.
    */

    rc = pagerOpenWalIfPresent(pPager);

#ifndef SQLITE_OMIT_WAL
    assert( pPager->pWal==0 || rc==SQLITE_OK );
#endif

    if( rc==SQLITE_OK && pagerUseWal(pPager)==0 ){
      rc = pagerServerConnect(pPager);
    }
  }

#ifdef SQLITE_SERVER_EDITION
  if( pagerIsServer(pPager) ){
    assert( rc==SQLITE_OK );
    pager_reset(pPager);
    rc = sqlite3ServerBegin(pPager->pServer);
  }else
#endif
  if( pagerUseWal(pPager) ){
    assert( rc==SQLITE_OK );
    rc = pagerBeginReadTransaction(pPager);
  }

  if( pPager->tempFile==0 && pPager->eState==PAGER_OPEN && rc==SQLITE_OK ){
................................................................................

/*
** Return true if the underlying VFS for the given pager supports the
** primitives necessary for write-ahead logging.
*/
int sqlite3PagerWalSupported(Pager *pPager){
  const sqlite3_io_methods *pMethods = pPager->fd->pMethods;
  if( pPager->noLock ) return 0;
  return pPager->exclusiveMode || (pMethods->iVersion>=2 && pMethods->xShmMap);
}

/*
** Attempt to take an exclusive lock on the database file. If a PENDING lock
** is obtained instead, immediately release it.
*/
................................................................................
    /* Close any rollback journal previously open */
    sqlite3OsClose(pPager->jfd);

    rc = pagerOpenWal(pPager);
    if( rc==SQLITE_OK ){
      pPager->journalMode = PAGER_JOURNALMODE_WAL;
      pPager->eState = PAGER_OPEN;

    }
  }else{
    *pbOpen = 1;
  }

  return rc;
}







>
>




>
|
>



<
<
<
<







|







 







|







 







>







5368
5369
5370
5371
5372
5373
5374
5375
5376
5377
5378
5379
5380
5381
5382
5383
5384
5385
5386




5387
5388
5389
5390
5391
5392
5393
5394
5395
5396
5397
5398
5399
5400
5401
....
7451
7452
7453
7454
7455
7456
7457
7458
7459
7460
7461
7462
7463
7464
7465
....
7546
7547
7548
7549
7550
7551
7552
7553
7554
7555
7556
7557
7558
7559
7560
        ** to be the right size but is not actually valid. Avoid this
        ** possibility by unmapping the db here. */
        if( USEFETCH(pPager) ){
          sqlite3OsUnfetch(pPager->fd, 0, 0);
        }
      }
    }

    rc = pagerServerConnect(pPager);

    /* If there is a WAL file in the file-system, open this database in WAL
    ** mode. Otherwise, the following function call is a no-op.
    */
    if( rc==SQLITE_OK ){
      rc = pagerOpenWalIfPresent(pPager);
    }
#ifndef SQLITE_OMIT_WAL
    assert( pPager->pWal==0 || rc==SQLITE_OK );
#endif




  }

#ifdef SQLITE_SERVER_EDITION
  if( pagerIsServer(pPager) ){
    assert( rc==SQLITE_OK );
    pager_reset(pPager);
    rc = sqlite3ServerBegin(pPager->pServer);
  }
#endif
  if( pagerUseWal(pPager) ){
    assert( rc==SQLITE_OK );
    rc = pagerBeginReadTransaction(pPager);
  }

  if( pPager->tempFile==0 && pPager->eState==PAGER_OPEN && rc==SQLITE_OK ){
................................................................................

/*
** Return true if the underlying VFS for the given pager supports the
** primitives necessary for write-ahead logging.
*/
int sqlite3PagerWalSupported(Pager *pPager){
  const sqlite3_io_methods *pMethods = pPager->fd->pMethods;
  if( pPager->noLock && !pagerIsServer(pPager) ) return 0;
  return pPager->exclusiveMode || (pMethods->iVersion>=2 && pMethods->xShmMap);
}

/*
** Attempt to take an exclusive lock on the database file. If a PENDING lock
** is obtained instead, immediately release it.
*/
................................................................................
    /* Close any rollback journal previously open */
    sqlite3OsClose(pPager->jfd);

    rc = pagerOpenWal(pPager);
    if( rc==SQLITE_OK ){
      pPager->journalMode = PAGER_JOURNALMODE_WAL;
      pPager->eState = PAGER_OPEN;
      sqlite3WalServer(pPager->pWal, pPager->pServer);
    }
  }else{
    *pbOpen = 1;
  }

  return rc;
}

Changes to src/wal.c.

450
451
452
453
454
455
456



457






458
459
460
461
462
463
464
....
1256
1257
1258
1259
1260
1261
1262








1263
1264
1265
1266
1267
1268
1269
....
2245
2246
2247
2248
2249
2250
2251



2252
2253
2254
2255
2256
2257
2258
....
2584
2585
2586
2587
2588
2589
2590
2591





2592
2593
2594
2595
2596
2597
2598
....
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
....
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733





2734
2735
2736
2737
2738
2739
2740
....
3054
3055
3056
3057
3058
3059
3060





3061






3062
3063
3064
3065
3066
3067
3068
  u32 nCkpt;                 /* Checkpoint sequence counter in the wal-header */
#ifdef SQLITE_DEBUG
  u8 lockError;              /* True if a locking error has occurred */
#endif
#ifdef SQLITE_ENABLE_SNAPSHOT
  WalIndexHdr *pSnapshot;    /* Start transaction here if not NULL */
#endif



};







/*
** Candidate values for Wal.exclusiveMode.
*/
#define WAL_NORMAL_MODE     0
#define WAL_EXCLUSIVE_MODE  1     
#define WAL_HEAPMEMORY_MODE 2
................................................................................
      sqlite3_free((void *)pWal->apWiData[i]);
      pWal->apWiData[i] = 0;
    }
  }else{
    sqlite3OsShmUnmap(pWal->pDbFd, isDelete);
  }
}









/* 
** Open a connection to the WAL file zWalName. The database file must 
** already be opened on connection pDbFd. The buffer that zWalName points
** to must remain valid for the lifetime of the returned Wal* handle.
**
** A SHARED lock should be held on the database file when this function
................................................................................
      }
    }
    if( rc!=SQLITE_OK ){
      return rc;
    }
  }




  pInfo = walCkptInfo(pWal);
  if( !useWal && pInfo->nBackfill==pWal->hdr.mxFrame 
#ifdef SQLITE_ENABLE_SNAPSHOT
   && (pWal->pSnapshot==0 || pWal->hdr.mxFrame==0
     || 0==memcmp(&pWal->hdr, pWal->pSnapshot, sizeof(WalIndexHdr)))
#endif
  ){
................................................................................
){
  u32 iRead = 0;                  /* If !=0, WAL frame to return data from */
  u32 iLast = pWal->hdr.mxFrame;  /* Last page in WAL for this reader */
  int iHash;                      /* Used to loop through N hash tables */
  int iMinHash;

  /* This routine is only be called from within a read transaction. */
  assert( pWal->readLock>=0 || pWal->lockError );






  /* If the "last page" field of the wal-index header snapshot is 0, then
  ** no data will be read from the wal under any circumstances. Return early
  ** in this case as an optimization.  Likewise, if pWal->readLock==0, 
  ** then the WAL is ignored by the reader so return early, as if the 
  ** WAL were empty.
  */
................................................................................
  return sqlite3OsRead(pWal->pWalFd, pOut, (nOut>sz ? sz : nOut), iOffset);
}

/* 
** Return the size of the database in pages (or zero, if unknown).
*/
Pgno sqlite3WalDbsize(Wal *pWal){
  if( pWal && ALWAYS(pWal->readLock>=0) ){
    return pWal->hdr.nPage;
  }
  return 0;
}


/* 
................................................................................
** There can only be a single writer active at a time.
*/
int sqlite3WalBeginWriteTransaction(Wal *pWal){
  int rc;

  /* Cannot start a write transaction without first holding a read
  ** transaction. */
  assert( pWal->readLock>=0 );
  assert( pWal->writeLock==0 && pWal->iReCksum==0 );

  if( pWal->readOnly ){
    return SQLITE_READONLY;
  }






  /* Only one writer allowed at a time.  Get the write lock.  Return
  ** SQLITE_BUSY if unable.
  */
  rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1);
  if( rc ){
    return rc;
................................................................................
  int szFrame;                    /* The size of a single frame */
  i64 iOffset;                    /* Next byte to write in WAL file */
  WalWriter w;                    /* The writer */
  u32 iFirst = 0;                 /* First frame that may be overwritten */
  WalIndexHdr *pLive;             /* Pointer to shared header */

  assert( pList );





  assert( pWal->writeLock );







  /* If this frame set completes a transaction, then nTruncate>0.  If
  ** nTruncate==0 then this frame set does not complete the transaction. */
  assert( (isCommit!=0)==(nTruncate!=0) );

#if defined(SQLITE_TEST) && defined(SQLITE_DEBUG)
  { int cnt; for(cnt=0, p=pList; p; p=p->pDirty, cnt++){}







>
>
>

>
>
>
>
>
>







 







>
>
>
>
>
>
>
>







 







>
>
>







 







|
>
>
>
>
>







 







|







 







|





>
>
>
>
>







 







>
>
>
>
>
|
>
>
>
>
>
>







450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
....
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
....
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
....
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
....
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
....
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
....
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
  u32 nCkpt;                 /* Checkpoint sequence counter in the wal-header */
#ifdef SQLITE_DEBUG
  u8 lockError;              /* True if a locking error has occurred */
#endif
#ifdef SQLITE_ENABLE_SNAPSHOT
  WalIndexHdr *pSnapshot;    /* Start transaction here if not NULL */
#endif
#ifdef SQLITE_SERVER_EDITION
  Server *pServer;
#endif
};

#ifdef SQLITE_SERVER_EDITION
# define walIsServer(p) ((p)->pServer!=0)
#else
# define walIsServer(p) 0
#endif

/*
** Candidate values for Wal.exclusiveMode.
*/
#define WAL_NORMAL_MODE     0
#define WAL_EXCLUSIVE_MODE  1     
#define WAL_HEAPMEMORY_MODE 2
................................................................................
      sqlite3_free((void *)pWal->apWiData[i]);
      pWal->apWiData[i] = 0;
    }
  }else{
    sqlite3OsShmUnmap(pWal->pDbFd, isDelete);
  }
}

#ifdef SQLITE_SERVER_EDITION
int sqlite3WalServer(Wal *pWal, Server *pServer){
  assert( pWal->pServer==0 );
  pWal->pServer = pServer;
  return SQLITE_OK;
}
#endif

/* 
** Open a connection to the WAL file zWalName. The database file must 
** already be opened on connection pDbFd. The buffer that zWalName points
** to must remain valid for the lifetime of the returned Wal* handle.
**
** A SHARED lock should be held on the database file when this function
................................................................................
      }
    }
    if( rc!=SQLITE_OK ){
      return rc;
    }
  }

  assert( rc==SQLITE_OK );
  if( walIsServer(pWal) ) return SQLITE_OK;

  pInfo = walCkptInfo(pWal);
  if( !useWal && pInfo->nBackfill==pWal->hdr.mxFrame 
#ifdef SQLITE_ENABLE_SNAPSHOT
   && (pWal->pSnapshot==0 || pWal->hdr.mxFrame==0
     || 0==memcmp(&pWal->hdr, pWal->pSnapshot, sizeof(WalIndexHdr)))
#endif
  ){
................................................................................
){
  u32 iRead = 0;                  /* If !=0, WAL frame to return data from */
  u32 iLast = pWal->hdr.mxFrame;  /* Last page in WAL for this reader */
  int iHash;                      /* Used to loop through N hash tables */
  int iMinHash;

  /* This routine is only be called from within a read transaction. */
  assert( walIsServer(pWal) || pWal->readLock>=0 || pWal->lockError );

  if( walIsServer(pWal) ){
    /* A server mode connection must read from the most recent snapshot. */
    iLast = walIndexHdr(pWal)->mxFrame;
  }

  /* If the "last page" field of the wal-index header snapshot is 0, then
  ** no data will be read from the wal under any circumstances. Return early
  ** in this case as an optimization.  Likewise, if pWal->readLock==0, 
  ** then the WAL is ignored by the reader so return early, as if the 
  ** WAL were empty.
  */
................................................................................
  return sqlite3OsRead(pWal->pWalFd, pOut, (nOut>sz ? sz : nOut), iOffset);
}

/* 
** Return the size of the database in pages (or zero, if unknown).
*/
Pgno sqlite3WalDbsize(Wal *pWal){
  if( pWal && (walIsServer(pWal) || ALWAYS(pWal->readLock>=0)) ){
    return pWal->hdr.nPage;
  }
  return 0;
}


/* 
................................................................................
** There can only be a single writer active at a time.
*/
int sqlite3WalBeginWriteTransaction(Wal *pWal){
  int rc;

  /* Cannot start a write transaction without first holding a read
  ** transaction. */
  assert( walIsServer(pWal) || pWal->readLock>=0 );
  assert( pWal->writeLock==0 && pWal->iReCksum==0 );

  if( pWal->readOnly ){
    return SQLITE_READONLY;
  }

  /* For a server connection, do nothing at this point. */
  if( walIsServer(pWal) ){
    return SQLITE_OK;
  }

  /* Only one writer allowed at a time.  Get the write lock.  Return
  ** SQLITE_BUSY if unable.
  */
  rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1);
  if( rc ){
    return rc;
................................................................................
  int szFrame;                    /* The size of a single frame */
  i64 iOffset;                    /* Next byte to write in WAL file */
  WalWriter w;                    /* The writer */
  u32 iFirst = 0;                 /* First frame that may be overwritten */
  WalIndexHdr *pLive;             /* Pointer to shared header */

  assert( pList );
  assert( pWal->writeLock || walIsServer(pWal) );
  if( pWal->writeLock==0 ){
    int bDummy = 0;
    rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1);
    if( rc==SQLITE_OK ){
      pWal->writeLock = 1;
      rc = walIndexTryHdr(pWal, &bDummy);
    }
    if( rc!=SQLITE_OK ){
      return rc;
    }
  }

  /* If this frame set completes a transaction, then nTruncate>0.  If
  ** nTruncate==0 then this frame set does not complete the transaction. */
  assert( (isCommit!=0)==(nTruncate!=0) );

#if defined(SQLITE_TEST) && defined(SQLITE_DEBUG)
  { int cnt; for(cnt=0, p=pList; p; p=p->pDirty, cnt++){}

Changes to src/wal.h.

139
140
141
142
143
144
145




146
147
148
** stored in each frame (i.e. the db page-size when the WAL was created).
*/
int sqlite3WalFramesize(Wal *pWal);
#endif

/* Return the sqlite3_file object for the WAL file */
sqlite3_file *sqlite3WalFile(Wal *pWal);





#endif /* ifndef SQLITE_OMIT_WAL */
#endif /* SQLITE_WAL_H */







>
>
>
>



139
140
141
142
143
144
145
146
147
148
149
150
151
152
** stored in each frame (i.e. the db page-size when the WAL was created).
*/
int sqlite3WalFramesize(Wal *pWal);
#endif

/* Return the sqlite3_file object for the WAL file */
sqlite3_file *sqlite3WalFile(Wal *pWal);

#ifdef SQLITE_SERVER_EDITION
int sqlite3WalServer(Wal *pWal, Server *pServer);
#endif

#endif /* ifndef SQLITE_OMIT_WAL */
#endif /* SQLITE_WAL_H */

Added test/serverwal.test.

































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
# 2017 April 25
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file implements regression tests for SQLite library.  The
# focus of this script is testing the server mode of SQLite.
#


set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix serverwal

# Check files are created and deleted as expected.
#
do_execsql_test 1.0 {
  PRAGMA journal_mode = wal;
} {wal}
do_execsql_test 1.1 {
  CREATE TABLE t1(a, b);
}
do_execsql_test 1.2 {
  SELECT * FROM t1;
} {}
do_test 1.3 {
  lsort [glob test.db*]
} {test.db test.db-hma test.db-shm test.db-wal}
do_test 1.4 {
  db close
  glob test.db*
} {test.db}

# Two concurrent transactions.
#
do_test 2.0 {
  sqlite3 db  test.db
  sqlite3 db2 test.db
  db eval {
    CREATE TABLE t2(a, b);
  }
} {}
do_test 2.1 {
  execsql {
    BEGIN;
      INSERT INTO t1 VALUES(1, 2);
  } db
  execsql {
    BEGIN;
      INSERT INTO t2 VALUES(1, 2);
  } db2
} {}
do_test 2.2 {
  execsql COMMIT db
  execsql COMMIT db2
} {}


finish_test