SQLite

Check-in [fa46fa3bfc]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add the ".integrity_check" command to tserver.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | begin-concurrent-wal2
Files: files | file ages | folders
SHA3-256: fa46fa3bfcd4b6d1d219db4179ce69d0edb0786a521acaa034fdb74b312274ff
User & Date: dan 2018-12-18 16:24:21.581
Context
2018-12-18
17:32
Merge latest trunk changes into this branch. (check-in: b3a163b46c user: dan tags: begin-concurrent-wal2)
16:24
Add the ".integrity_check" command to tserver. (check-in: fa46fa3bfc user: dan tags: begin-concurrent-wal2)
2018-12-17
18:26
Add wal2 related tests to this branch. (check-in: 5645822039 user: dan tags: begin-concurrent-wal2)
Changes
Unified Diff Ignore Whitespace Patch
Changes to tool/tserver.c.
31
32
33
34
35
36
37


38
39
40
41
42
43
44
**   .quit                    Disconnect.
**   .run                     Run all SQL statements in the list.
**   .repeats N               Configure the number of repeats per ".run".
**   .seconds N               Configure the number of seconds to ".run" for.
**   .mutex_commit            Add a "COMMIT" protected by a g.commit_mutex
**                            to the current SQL.
**   .stop                    Stop the tserver process - exit(0).


**
** Example input:
**
**   BEGIN;
**     INSERT INTO t1 VALUES(randomblob(10), randomblob(100));
**     INSERT INTO t1 VALUES(randomblob(10), randomblob(100));
**     INSERT INTO t1 VALUES(randomblob(10), randomblob(100));







>
>







31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
**   .quit                    Disconnect.
**   .run                     Run all SQL statements in the list.
**   .repeats N               Configure the number of repeats per ".run".
**   .seconds N               Configure the number of seconds to ".run" for.
**   .mutex_commit            Add a "COMMIT" protected by a g.commit_mutex
**                            to the current SQL.
**   .stop                    Stop the tserver process - exit(0).
**   .checkpoint N
**   .integrity_check
**
** Example input:
**
**   BEGIN;
**     INSERT INTO t1 VALUES(randomblob(10), randomblob(100));
**     INSERT INTO t1 VALUES(randomblob(10), randomblob(100));
**     INSERT INTO t1 VALUES(randomblob(10), randomblob(100));
83
84
85
86
87
88
89
90
91



92
93
94
95
96
97
98
};

static struct TserverGlobal g = {0};

typedef struct ClientSql ClientSql;
struct ClientSql {
  sqlite3_stmt *pStmt;
  int bMutex;
};




typedef struct ClientCtx ClientCtx;
struct ClientCtx {
  sqlite3 *db;                    /* Database handle for this client */
  int fd;                         /* Client fd */
  int nRepeat;                    /* Number of times to repeat SQL */
  int nSecond;                    /* Number of seconds to run for */







|

>
>
>







85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
};

static struct TserverGlobal g = {0};

typedef struct ClientSql ClientSql;
struct ClientSql {
  sqlite3_stmt *pStmt;
  int flags;
};

#define TSERVER_CLIENTSQL_MUTEX     0x0001
#define TSERVER_CLIENTSQL_INTEGRITY 0x0002

typedef struct ClientCtx ClientCtx;
struct ClientCtx {
  sqlite3 *db;                    /* Database handle for this client */
  int fd;                         /* Client fd */
  int nRepeat;                    /* Number of times to repeat SQL */
  int nSecond;                    /* Number of seconds to run for */
244
245
246
247
248
249
250
251
252
253
254
255
256
257









258
259
260
261
262
263
264
265
266
267
268
269

  for(j=0; (p->nRepeat<=0 || j<p->nRepeat) && rc==SQLITE_OK; j++){
    sqlite3_int64 t2;

    for(i=0; i<p->nPrepare && rc==SQLITE_OK; i++){
      sqlite3_stmt *pStmt = p->aPrepare[i].pStmt;

      /* If the bMutex flag is set, grab g.commit_mutex before executing
      ** the SQL statement (which is always "COMMIT" in this case). */
      if( p->aPrepare[i].bMutex ){
        sqlite3_mutex_enter(g.commit_mutex);
      }

      /* Execute the statement */









      while( sqlite3_step(pStmt)==SQLITE_ROW );
      rc = sqlite3_reset(pStmt);

      /* Relinquish the g.commit_mutex mutex if required. */
      if( p->aPrepare[i].bMutex ){
        sqlite3_mutex_leave(g.commit_mutex);
      }

      if( (rc & 0xFF)==SQLITE_BUSY ){
        if( sqlite3_get_autocommit(p->db)==0 ){
          sqlite3_exec(p->db, "ROLLBACK", 0, 0, 0);
        }







|

|




>
>
>
>
>
>
>
>
>




|







249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283

  for(j=0; (p->nRepeat<=0 || j<p->nRepeat) && rc==SQLITE_OK; j++){
    sqlite3_int64 t2;

    for(i=0; i<p->nPrepare && rc==SQLITE_OK; i++){
      sqlite3_stmt *pStmt = p->aPrepare[i].pStmt;

      /* If the MUTEX flag is set, grab g.commit_mutex before executing
      ** the SQL statement (which is always "COMMIT" in this case). */
      if( p->aPrepare[i].flags & TSERVER_CLIENTSQL_MUTEX ){
        sqlite3_mutex_enter(g.commit_mutex);
      }

      /* Execute the statement */
      if( p->aPrepare[i].flags & TSERVER_CLIENTSQL_INTEGRITY ){
        sqlite3_step(pStmt);
        if( sqlite3_stricmp("ok", sqlite3_column_text(pStmt, 0)) ){
          send_message(p, "error - integrity_check failed: %s\n", 
              sqlite3_column_text(pStmt, 0)
          );
        }
        sqlite3_reset(pStmt);
      }
      while( sqlite3_step(pStmt)==SQLITE_ROW );
      rc = sqlite3_reset(pStmt);

      /* Relinquish the g.commit_mutex mutex if required. */
      if( p->aPrepare[i].flags & TSERVER_CLIENTSQL_MUTEX ){
        sqlite3_mutex_leave(g.commit_mutex);
      }

      if( (rc & 0xFF)==SQLITE_BUSY ){
        if( sqlite3_get_autocommit(p->db)==0 ){
          sqlite3_exec(p->db, "ROLLBACK", 0, 0, 0);
        }
311
312
313
314
315
316
317

318
319
320
321
322
323
324
        }
      }
      pthread_mutex_unlock(&g.ckpt_mutex);
    }

    if( rc==SQLITE_OK && p->bClientCkptRequired ){
      rc = sqlite3_wal_checkpoint(p->db, "main");

      assert( rc==SQLITE_OK );
      p->bClientCkptRequired = 0;
    }
  }

  if( rc==SQLITE_OK ){
    int nMs = (int)(get_timer() - t0);







>







325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
        }
      }
      pthread_mutex_unlock(&g.ckpt_mutex);
    }

    if( rc==SQLITE_OK && p->bClientCkptRequired ){
      rc = sqlite3_wal_checkpoint(p->db, "main");
      if( rc==SQLITE_BUSY ) rc = SQLITE_OK;
      assert( rc==SQLITE_OK );
      p->bClientCkptRequired = 0;
    }
  }

  if( rc==SQLITE_OK ){
    int nMs = (int)(get_timer() - t0);
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
      int nSql = strlen(zSql);
      trim_string(&zSql, &nSql);
      rc = send_message(p, "%d: %.*s\n", i, nSql, zSql);
    }
  }

  else if( n>=1 && n<=4 && 0==strncmp(z, "quit", n) ){
    rc = 1;
  }

  else if( n>=2 && n<=7 && 0==strncmp(z, "repeats", n) ){
    if( nArg ){
      p->nRepeat = strtol(zArg, 0, 0);
      if( p->nRepeat>0 ) p->nSecond = 0;
    }







|







374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
      int nSql = strlen(zSql);
      trim_string(&zSql, &nSql);
      rc = send_message(p, "%d: %.*s\n", i, nSql, zSql);
    }
  }

  else if( n>=1 && n<=4 && 0==strncmp(z, "quit", n) ){
    rc = -1;
  }

  else if( n>=2 && n<=7 && 0==strncmp(z, "repeats", n) ){
    if( nArg ){
      p->nRepeat = strtol(zArg, 0, 0);
      if( p->nRepeat>0 ) p->nSecond = 0;
    }
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406







407
408
409
410
411
412
413
414
415
416
417
418
419
    }
    rc = send_message(p, "ok (seconds=%d)\n", p->nSecond);
  }

  else if( n>=1 && n<=12 && 0==strncmp(z, "mutex_commit", n) ){
    rc = handle_some_sql(p, "COMMIT;", 7);
    if( rc==SQLITE_OK ){
      p->aPrepare[p->nPrepare-1].bMutex = 1;
    }
  }

  else if( n>=1 && n<=10 && 0==strncmp(z, "checkpoint", n) ){
    if( nArg ){
      p->nClientThreshold = strtol(zArg, 0, 0);
    }
    rc = send_message(p, "ok (checkpoint=%d)\n", p->nClientThreshold);
  }

  else if( n>=2 && n<=4 && 0==strncmp(z, "stop", n) ){
    sqlite3_close(g.db);
    exit(0);
  }








  else{
    send_message(p, 
        "unrecognized dot command: %.*s\n"
        "should be \"list\", \"run\", \"repeats\", \"mutex_commit\", "
        "\"checkpoint\" or \"seconds\"\n", n, z
    );
    rc = 1;
  }

  return rc;
}








|














>
>
>
>
>
>
>





|







400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
    }
    rc = send_message(p, "ok (seconds=%d)\n", p->nSecond);
  }

  else if( n>=1 && n<=12 && 0==strncmp(z, "mutex_commit", n) ){
    rc = handle_some_sql(p, "COMMIT;", 7);
    if( rc==SQLITE_OK ){
      p->aPrepare[p->nPrepare-1].flags |= TSERVER_CLIENTSQL_MUTEX;
    }
  }

  else if( n>=1 && n<=10 && 0==strncmp(z, "checkpoint", n) ){
    if( nArg ){
      p->nClientThreshold = strtol(zArg, 0, 0);
    }
    rc = send_message(p, "ok (checkpoint=%d)\n", p->nClientThreshold);
  }

  else if( n>=2 && n<=4 && 0==strncmp(z, "stop", n) ){
    sqlite3_close(g.db);
    exit(0);
  }

  else if( n>=2 && n<=15 && 0==strncmp(z, "integrity_check", n) ){
    rc = handle_some_sql(p, "PRAGMA integrity_check;", 23);
    if( rc==SQLITE_OK ){
      p->aPrepare[p->nPrepare-1].flags |= TSERVER_CLIENTSQL_INTEGRITY;
    }
  }

  else{
    send_message(p, 
        "unrecognized dot command: %.*s\n"
        "should be \"list\", \"run\", \"repeats\", \"mutex_commit\", "
        "\"checkpoint\", \"integrity_check\" or \"seconds\"\n", n, z
    );
    rc = 1;
  }

  return rc;
}

502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
        if( nCmd>0 ){
          memmove(zCmd, &zCmd[nConsume], nCmd);
        }
      }
    }while( rc==SQLITE_OK && nConsume>0 );
  }

  fprintf(stdout, "Client %d disconnects\n", ctx.fd);
  fflush(stdout);
  close(ctx.fd);
  clear_sql(&ctx);
  sqlite3_free(ctx.aPrepare);
  sqlite3_close(ctx.db);
  return 0;
} 







|







524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
        if( nCmd>0 ){
          memmove(zCmd, &zCmd[nConsume], nCmd);
        }
      }
    }while( rc==SQLITE_OK && nConsume>0 );
  }

  fprintf(stdout, "Client %d disconnects (rc=%d)\n", ctx.fd, rc);
  fflush(stdout);
  close(ctx.fd);
  clear_sql(&ctx);
  sqlite3_free(ctx.aPrepare);
  sqlite3_close(ctx.db);
  return 0;
} 
Changes to tool/tserver_test.tcl.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17

18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36

37
38
39
40
41
42
43
#!/usr/bin/tclsh
#
# This script is used to run the performance test cases described in
# README-server-edition.html.
#


package require sqlite3

# Default values for command line switches:
set O(-database) ""
set O(-rows)     [expr 5000000]
set O(-mode)     wal2
set O(-tserver)  "./tserver"
set O(-seconds)  20
set O(-writers)  1
set O(-readers)  0

set O(-verbose)  0


proc error_out {err} {
  puts stderr $err
  exit -1
}

proc usage {} {
  puts stderr "Usage: $::argv0 ?OPTIONS?"
  puts stderr ""
  puts stderr "Where OPTIONS are:"
  puts stderr "  -database <database file>             (default: test.db)"
  puts stderr "  -mode <mode>                          (default: wal2)"
  puts stderr "  -rows <number of rows>                (default: 5000000)"
  puts stderr "  -tserver <path to tserver executable> (default: ./tserver)"
  puts stderr "  -seconds <time to run for in seconds> (default: 20)"
  puts stderr "  -writers <number of writer clients>   (default: 1)"
  puts stderr "  -readers <number of reader clients>   (default: 0)"

  puts stderr "  -verbose 0|1                          (default: 0)"
  exit -1
}

for {set i 0} {$i < [llength $argv]} {incr i} {
  set opt ""
  set arg [lindex $argv $i]










|
|
|
|
|
|
|
>
|


















>







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
#!/usr/bin/tclsh
#
# This script is used to run the performance test cases described in
# README-server-edition.html.
#


package require sqlite3

# Default values for command line switches:
set O(-database)  ""
set O(-rows)      [expr 5000000]
set O(-mode)      wal2
set O(-tserver)   "./tserver"
set O(-seconds)   20
set O(-writers)   1
set O(-readers)   0
set O(-integrity) 0
set O(-verbose)   0


proc error_out {err} {
  puts stderr $err
  exit -1
}

proc usage {} {
  puts stderr "Usage: $::argv0 ?OPTIONS?"
  puts stderr ""
  puts stderr "Where OPTIONS are:"
  puts stderr "  -database <database file>             (default: test.db)"
  puts stderr "  -mode <mode>                          (default: wal2)"
  puts stderr "  -rows <number of rows>                (default: 5000000)"
  puts stderr "  -tserver <path to tserver executable> (default: ./tserver)"
  puts stderr "  -seconds <time to run for in seconds> (default: 20)"
  puts stderr "  -writers <number of writer clients>   (default: 1)"
  puts stderr "  -readers <number of reader clients>   (default: 0)"
  puts stderr "  -integrity <number of IC clients>     (default: 0)"
  puts stderr "  -verbose 0|1                          (default: 0)"
  exit -1
}

for {set i 0} {$i < [llength $argv]} {incr i} {
  set opt ""
  set arg [lindex $argv $i]
244
245
246
247
248
249
250








251
252
253
254
255
256
257
258
259
260



261
262
263
264

265
266
267
268
269
270
271
  return [subst -nocommands {
    BEGIN;
      SELECT * FROM t1 WHERE a>abs((random()%$O(-rows))) LIMIT 10;
    END;
  }]
}










create_test_database
tserver_start

for {set i 0} {$i < $O(-writers)} {incr i} {
  client_launch w.$i [script_writer [expr {$i==0 && $O(-mode)=="wal2"}]]
}
for {set i 0} {$i < $O(-readers)} {incr i} {
  client_launch r.$i [script_reader]
}



client_wait

set name(w) "Writers"
set name(r) "Readers"

foreach r $::client_output {
  array set a $r
  set type [string range $a(name) 0 0]
  incr x($type.ok) $a(ok);
  incr x($type.busy) $a(busy);
  incr x($type.n) 1
  set t($type) 1







>
>
>
>
>
>
>
>










>
>
>




>







246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
  return [subst -nocommands {
    BEGIN;
      SELECT * FROM t1 WHERE a>abs((random()%$O(-rows))) LIMIT 10;
    END;
  }]
}

proc script_integrity {} {
  global O

  return {
    .integrity_check
  }
}


create_test_database
tserver_start

for {set i 0} {$i < $O(-writers)} {incr i} {
  client_launch w.$i [script_writer [expr {$i==0 && $O(-mode)=="wal2"}]]
}
for {set i 0} {$i < $O(-readers)} {incr i} {
  client_launch r.$i [script_reader]
}
for {set i 0} {$i < $O(-integrity)} {incr i} {
  client_launch i.$i [script_integrity]
}
client_wait

set name(w) "Writers"
set name(r) "Readers"
set name(i) "Integrity"
foreach r $::client_output {
  array set a $r
  set type [string range $a(name) 0 0]
  incr x($type.ok) $a(ok);
  incr x($type.busy) $a(busy);
  incr x($type.n) 1
  set t($type) 1