/ Check-in [a8115f95]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add support for crash recovery in multi-process mode. And add test cases for the same.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | server-process-edition
Files: files | file ages | folders
SHA3-256:a8115f95e80cc90c095fdd0a151da51f4d3ee427defdc34780e977585a68776d
User & Date: dan 2017-08-17 19:32:02
Context
2017-08-18
16:04
Do not search for locks to clear when connecting to a db in multi-process mode unless it looks like the previous user of the client-id crashed. check-in: 66fb9e1c user: dan tags: server-process-edition
2017-08-17
19:32
Add support for crash recovery in multi-process mode. And add test cases for the same. check-in: a8115f95 user: dan tags: server-process-edition
2017-08-16
17:06
Update this branch with the latest changes from trunk. check-in: 380a7b7a user: dan tags: server-process-edition
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/server.c.

209
210
211
212
213
214
215






















216
217
218
219
220
221
222
223
224
225

226
227
228
229






230
231
232
233
234
235
236
...
247
248
249
250
251
252
253


















254
255
256
257
258
259
260
...
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
...
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
...
360
361
362
363
364
365
366

367
368
369
370
371
372
373
374
375
376
377
...
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
...
450
451
452
453
454
455
456










457
458
459
460
461
462
463
...
714
715
716
717
718
719
720






















721
722
723
724
725
726
727
...
764
765
766
767
768
769
770
771
772
773
774
775
776






777
778
779
780
781
782
783
  }else{
    p->nClient++;
  }
  pNew->pDb = p;
  serverLeaveMutex();
  return rc;
}























/*
** Free all resources allocated by serverInitDatabase() associated with the
** object passed as the only argument.
*/
static void serverShutdownDatabase(
  ServerDb *pDb, 
  sqlite3_file *dbfd, 
  int bDelete
){

  int i;

  for(i=0; i<HMA_MAX_TRANSACTIONID; i++){
    ServerJournal *pJ = &pDb->aJrnl[i];






    if( pJ->jfd ){
      sqlite3OsClose(pJ->jfd);
      if( bDelete ) sqlite3OsDelete(pDb->pVfs, pJ->zJournal, 0);
    }
    sqlite3_free(pJ->zJournal);
  }
  memset(pDb->aJrnl, 0, sizeof(ServerJournal)*HMA_MAX_TRANSACTIONID);
................................................................................
    sqlite3OsFileControl(dbfd, SQLITE_FCNTL_SERVER_SHMCLOSE, (void*)&arg);
  }else{
    sqlite3_free(pDb->aSlot);
  }
  pDb->aSlot = 0;
  pDb->bInit = 0;
}



















/*
** This function is called when the very first connection to a database
** is established. It is responsible for rolling back any hot journal
** files found in the file-system.
*/
static int serverInitDatabase(Server *pNew, int eServer){
................................................................................
      pJ->zJournal = sqlite3_mprintf("%s-journal/%d-journal", zFilename, i);
      if( pJ->zJournal==0 ){
        rc = SQLITE_NOMEM_BKPT;
        break;
      }

      if( bRollback ){
        int bExist = 0;
        rc = sqlite3OsAccess(pVfs, pJ->zJournal, SQLITE_ACCESS_EXISTS, &bExist);
        if( rc==SQLITE_OK && bExist ){
          int flags = SQLITE_OPEN_READWRITE|SQLITE_OPEN_MAIN_JOURNAL;
          rc = sqlite3OsOpen(pVfs, pJ->zJournal, pJ->jfd, flags, &flags);
          if( rc==SQLITE_OK ){
            rc = sqlite3PagerRollbackJournal(pNew->pPager, pJ->jfd);
          }
        }
      }
    }
  }

  if( rc==SQLITE_OK && pDb->pServerShm && bRollback ){
    ServerFcntlArg arg;
    arg.h = pDb->pServerShm;
................................................................................
    arg.i2 = 0;
    rc = sqlite3OsFileControl(dbfd, SQLITE_FCNTL_SERVER_SHMOPEN2, (void*)&arg);
  }

  if( rc==SQLITE_OK ){
    pDb->bInit = 1;
  }else{
    serverShutdownDatabase(pNew->pDb, dbfd, eServer==1);
  }
  return rc;
}

/*
** Take (bLock==1) or release (bLock==0) a server shmlock on slot iSlot.
** Return SQLITE_OK if successful, or SQLITE_BUSY if the lock cannot be
................................................................................
*/
void sqlite3ServerDisconnect(Server *p, sqlite3_file *dbfd){
  ServerDb *pDb = p->pDb;

  /* In a multi-process setup, release the lock on the client slot and
  ** clear the bit in the ServerDb.transmask bitmask. */
  if( pDb->pServerShm && p->iTransId>=0 ){

    sqlite3_mutex_enter(pDb->mutex);
    pDb->transmask &= ~((u32)1 << p->iTransId);
    sqlite3_mutex_leave(pDb->mutex);
    serverFcntlLock(p, p->iTransId, 0);
  }

  serverEnterMutex();
  pDb->nClient--;
  if( pDb->nClient==0 ){
    sqlite3_file *dbfd = sqlite3PagerFile(p->pPager);
    ServerPage *pFree;
................................................................................
    if( pDb->pServerShm ){
      int res;
      res = sqlite3OsLock(dbfd, EXCLUSIVE_LOCK);
      if( res==SQLITE_OK ) bDelete = 1;
    }else{
      bDelete = 1;
    }
    serverShutdownDatabase(pDb, dbfd, bDelete);

    for(pp=&g_server.pDb; *pp!=pDb; pp=&((*pp)->pNext));
    *pp = pDb->pNext;
    sqlite3_mutex_free(pDb->mutex);
    while( (pFree = pDb->pFree) ){
      pDb->pFree = pFree->pNext;
      sqlite3_free(pFree);
................................................................................
                pNew->iTransId = i;
                pDb->transmask |= ((u32)1 << i);
              }
            }
          }
        }
        sqlite3_mutex_leave(pNew->pDb->mutex);










      }
    }else{
      rc = SQLITE_NOMEM_BKPT;
    }
  }

  if( rc!=SQLITE_OK && pNew ){
................................................................................
/*
** Release all write-locks.
*/
int sqlite3ServerReleaseWriteLocks(Server *p){
  int rc = SQLITE_OK;
  return rc;
}























/*
** Lock page pgno for reading (bWrite==0) or writing (bWrite==1).
**
** If parameter bBlock is non-zero, then make this a blocking lock if
** possible.
*/
................................................................................
      );

      iWriter = slotGetWriter(o);
      if( iWriter==p->iTransId || (bWrite==0 && (o & (1<<p->iTransId))) ){
        bSkip = 1;
        break;
      }else if( iWriter>=0 ){
        rc = SQLITE_BUSY_DEADLOCK;
      }else if( bWrite ){
        if( (slotReaderMask(o) & ~(1 << p->iTransId))==0 ){
          n += ((p->iTransId + 1) << HMA_MAX_TRANSACTIONID);
        }else{
          rc = SQLITE_BUSY_DEADLOCK;






        }
      }else{
        n |= (1 << p->iTransId);
      }

      assert( slotGetWriter(n)<0 
          || slotReaderMask(n)==0 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>






|



>




>
>
>
>
>
>







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







|
<
<
<
<
<
<
<
<







 







|







 







>



<







 







|







 







>
>
>
>
>
>
>
>
>
>







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







|




|
>
>
>
>
>
>







209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
...
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
...
351
352
353
354
355
356
357
358








359
360
361
362
363
364
365
...
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
...
399
400
401
402
403
404
405
406
407
408
409

410
411
412
413
414
415
416
...
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
...
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
...
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
...
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
  }else{
    p->nClient++;
  }
  pNew->pDb = p;
  serverLeaveMutex();
  return rc;
}

static int serverClientRollback(Server *p, int iClient){
  ServerDb *pDb = p->pDb;
  ServerJournal *pJ = &pDb->aJrnl[iClient];
  int bExist = 1;
  int rc = SQLITE_OK;

  if( pJ->jfd->pMethods==0 ){
    bExist = 0;
    rc = sqlite3OsAccess(pDb->pVfs, pJ->zJournal, SQLITE_ACCESS_EXISTS,&bExist);
    if( bExist && rc==SQLITE_OK ){
      int flags = SQLITE_OPEN_READWRITE|SQLITE_OPEN_MAIN_JOURNAL;
      rc = sqlite3OsOpen(pDb->pVfs, pJ->zJournal, pJ->jfd, flags, &flags);
    }
  }

  if( bExist && rc==SQLITE_OK ){
    rc = sqlite3PagerRollbackJournal(p->pPager, pJ->jfd);
  }
  return rc;
}


/*
** Free all resources allocated by serverInitDatabase() associated with the
** object passed as the only argument.
*/
static void serverShutdownDatabase(
  Server *p, 
  sqlite3_file *dbfd, 
  int bDelete
){
  ServerDb *pDb = p->pDb;
  int i;

  for(i=0; i<HMA_MAX_TRANSACTIONID; i++){
    ServerJournal *pJ = &pDb->aJrnl[i];

    if( pDb->pServerShm && bDelete ){
      int rc = serverClientRollback(p, i);
      if( rc!=SQLITE_OK ) bDelete = 0;
    }

    if( pJ->jfd ){
      sqlite3OsClose(pJ->jfd);
      if( bDelete ) sqlite3OsDelete(pDb->pVfs, pJ->zJournal, 0);
    }
    sqlite3_free(pJ->zJournal);
  }
  memset(pDb->aJrnl, 0, sizeof(ServerJournal)*HMA_MAX_TRANSACTIONID);
................................................................................
    sqlite3OsFileControl(dbfd, SQLITE_FCNTL_SERVER_SHMCLOSE, (void*)&arg);
  }else{
    sqlite3_free(pDb->aSlot);
  }
  pDb->aSlot = 0;
  pDb->bInit = 0;
}

static void serverClientUnlock(Server *p, int iClient){
  ServerDb *pDb = p->pDb;
  int i;

  assert( pDb->pServerShm );
  for(i=0; i<HMA_PAGELOCK_SLOTS; i++){
    u32 *pSlot = &pDb->aSlot[i];
    while( 1 ){
      u32 o = *pSlot;
      u32 n = o & ~((u32)1 << iClient);
      if( slotGetWriter(n)==iClient ){
        n -= ((iClient + 1) << HMA_MAX_TRANSACTIONID);
      }
      if( o==n || serverCompareAndSwap(pSlot, o, n) ) break;
    }
  }
}

/*
** This function is called when the very first connection to a database
** is established. It is responsible for rolling back any hot journal
** files found in the file-system.
*/
static int serverInitDatabase(Server *pNew, int eServer){
................................................................................
      pJ->zJournal = sqlite3_mprintf("%s-journal/%d-journal", zFilename, i);
      if( pJ->zJournal==0 ){
        rc = SQLITE_NOMEM_BKPT;
        break;
      }

      if( bRollback ){
        rc = serverClientRollback(pNew, i);








      }
    }
  }

  if( rc==SQLITE_OK && pDb->pServerShm && bRollback ){
    ServerFcntlArg arg;
    arg.h = pDb->pServerShm;
................................................................................
    arg.i2 = 0;
    rc = sqlite3OsFileControl(dbfd, SQLITE_FCNTL_SERVER_SHMOPEN2, (void*)&arg);
  }

  if( rc==SQLITE_OK ){
    pDb->bInit = 1;
  }else{
    serverShutdownDatabase(pNew, dbfd, eServer==1);
  }
  return rc;
}

/*
** Take (bLock==1) or release (bLock==0) a server shmlock on slot iSlot.
** Return SQLITE_OK if successful, or SQLITE_BUSY if the lock cannot be
................................................................................
*/
void sqlite3ServerDisconnect(Server *p, sqlite3_file *dbfd){
  ServerDb *pDb = p->pDb;

  /* In a multi-process setup, release the lock on the client slot and
  ** clear the bit in the ServerDb.transmask bitmask. */
  if( pDb->pServerShm && p->iTransId>=0 ){
    serverFcntlLock(p, p->iTransId, 0);
    sqlite3_mutex_enter(pDb->mutex);
    pDb->transmask &= ~((u32)1 << p->iTransId);
    sqlite3_mutex_leave(pDb->mutex);

  }

  serverEnterMutex();
  pDb->nClient--;
  if( pDb->nClient==0 ){
    sqlite3_file *dbfd = sqlite3PagerFile(p->pPager);
    ServerPage *pFree;
................................................................................
    if( pDb->pServerShm ){
      int res;
      res = sqlite3OsLock(dbfd, EXCLUSIVE_LOCK);
      if( res==SQLITE_OK ) bDelete = 1;
    }else{
      bDelete = 1;
    }
    serverShutdownDatabase(p, dbfd, bDelete);

    for(pp=&g_server.pDb; *pp!=pDb; pp=&((*pp)->pNext));
    *pp = pDb->pNext;
    sqlite3_mutex_free(pDb->mutex);
    while( (pFree = pDb->pFree) ){
      pDb->pFree = pFree->pNext;
      sqlite3_free(pFree);
................................................................................
                pNew->iTransId = i;
                pDb->transmask |= ((u32)1 << i);
              }
            }
          }
        }
        sqlite3_mutex_leave(pNew->pDb->mutex);

        /* If this is a multi-process database, it may be that the previous
        ** user of client-id pNew->iTransId crashed mid transaction. Roll
        ** back any hot journal file in the file-system and release 
        ** page locks held by any crashed process. TODO: The call to
        ** serverClientUnlock() is expensive.  */
        if( rc==SQLITE_OK && pDb->pServerShm ){
          serverClientUnlock(pNew, pNew->iTransId);
          rc = serverClientRollback(pNew, pNew->iTransId);
        }
      }
    }else{
      rc = SQLITE_NOMEM_BKPT;
    }
  }

  if( rc!=SQLITE_OK && pNew ){
................................................................................
/*
** Release all write-locks.
*/
int sqlite3ServerReleaseWriteLocks(Server *p){
  int rc = SQLITE_OK;
  return rc;
}

static int serverCheckClient(Server *p, int iClient){
  ServerDb *pDb = p->pDb;
  int rc = SQLITE_BUSY_DEADLOCK;
  if( pDb->pServerShm && 0==(pDb->transmask & (1 << iClient)) ){

    /* At this point it is know that client iClient, if it exists, resides in
    ** some other process. Check that it is still alive by attempting to lock
    ** its client slot. If the client is not alive, clear all its locks and
    ** rollback its journal.  */
    rc = serverFcntlLock(p, iClient, 1);
    if( rc==SQLITE_OK ){
      serverClientUnlock(p, iClient);
      rc = serverClientRollback(p, iClient);
      serverFcntlLock(p, iClient, 0);
      pDb->transmask &= ~(1 << iClient);
    }else if( rc==SQLITE_BUSY ){
      rc = SQLITE_BUSY_DEADLOCK;
    }
  }
  return rc;
}

/*
** Lock page pgno for reading (bWrite==0) or writing (bWrite==1).
**
** If parameter bBlock is non-zero, then make this a blocking lock if
** possible.
*/
................................................................................
      );

      iWriter = slotGetWriter(o);
      if( iWriter==p->iTransId || (bWrite==0 && (o & (1<<p->iTransId))) ){
        bSkip = 1;
        break;
      }else if( iWriter>=0 ){
        rc = serverCheckClient(p, iWriter);
      }else if( bWrite ){
        if( (slotReaderMask(o) & ~(1 << p->iTransId))==0 ){
          n += ((p->iTransId + 1) << HMA_MAX_TRANSACTIONID);
        }else{
          int i;
          for(i=0; i<HMA_MAX_TRANSACTIONID; i++){
            if( o & (1 << i) ){
              rc = serverCheckClient(p, i);
              break;
            }
          }
        }
      }else{
        n |= (1 << p->iTransId);
      }

      assert( slotGetWriter(n)<0 
          || slotReaderMask(n)==0 

Changes to test/permutations.test.

274
275
276
277
278
279
280
281

282
283
284
285
286
287
288
  All FTS5 tests.
} -files [glob -nocomplain $::testdir/../ext/fts5/test/*.test]

test_suite "server" -prefix "" -description {
  All server-edition tests.
} -files [
  test_set \
      select1.test server2.test server3.test server4.test server5.test

]

test_suite "fts5-light" -prefix "" -description {
  All FTS5 tests.
} -files [
  test_set \
      [glob -nocomplain $::testdir/../ext/fts5/test/*.test] \







|
>







274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
  All FTS5 tests.
} -files [glob -nocomplain $::testdir/../ext/fts5/test/*.test]

test_suite "server" -prefix "" -description {
  All server-edition tests.
} -files [
  test_set \
      select1.test server2.test server3.test server4.test server5.test \
      servercrash.test
]

test_suite "fts5-light" -prefix "" -description {
  All FTS5 tests.
} -files [
  test_set \
      [glob -nocomplain $::testdir/../ext/fts5/test/*.test] \

Changes to test/servercrash.test.

16
17
18
19
20
21
22
23







24
25
26
27
28
29
30
31
32
..
59
60
61
62
63
64
65












66
67
68
set testprefix servercrash

ifcapable !crashtest {
  finish_test
  return
}
do_not_use_codec








do_execsql_test 1.0 {
  PRAGMA page_siBlockze = 4096;
  PRAGMA auto_vacuum = OFF;
  CREATE TABLE t1(a, b);
  CREATE TABLE t2(c, d);

  INSERT INTO t1 VALUES(1, 2), (3, 4);
  INSERT INTO t2 VALUES(1, 2), (3, 4);
}
................................................................................
    crashsql -delay 1 -file test.db { INSERT INTO t1 VALUES(5, 6) }
  } {1 {child process exited abnormally}}

  sqlite3 db test.db
  do_execsql_test 3.$i.2 { SELECT * FROM t1 } {1 2 3 4}
  db close
}













finish_test









>
>
>
>
>
>
>

|







 







>
>
>
>
>
>
>
>
>
>
>
>



16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
..
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
set testprefix servercrash

ifcapable !crashtest {
  finish_test
  return
}
do_not_use_codec

source $testdir/server_common.tcl
return_if_no_server
db close

server_set_vfs unix
server_reset_db

do_execsql_test 1.0 {
  PRAGMA page_size = 4096;
  PRAGMA auto_vacuum = OFF;
  CREATE TABLE t1(a, b);
  CREATE TABLE t2(c, d);

  INSERT INTO t1 VALUES(1, 2), (3, 4);
  INSERT INTO t2 VALUES(1, 2), (3, 4);
}
................................................................................
    crashsql -delay 1 -file test.db { INSERT INTO t1 VALUES(5, 6) }
  } {1 {child process exited abnormally}}

  sqlite3 db test.db
  do_execsql_test 3.$i.2 { SELECT * FROM t1 } {1 2 3 4}
  db close
}

sqlite3 db test.db
db eval {SELECT * FROM t1}
for {set i 0} {$i < 10} {incr i} {
  do_test 4.$i.1 {
    crashsql -delay 1 -file test.db { INSERT INTO t1 VALUES(5, 6) }
  } {1 {child process exited abnormally}}

  db close
  sqlite3 db test.db
  do_execsql_test 4.$i.2 { SELECT * FROM t1 } {1 2 3 4}
}

finish_test