SQLite

Check-in [ae43539e62]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Enhanced "stress2" testing in the threadtest3.c test program.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: ae43539e62e76676a3daf561b629a1b9b4e2d2c9
User & Date: drh 2014-12-16 00:20:07.236
Context
2014-12-16
20:13
Experimental opimizations to speed up FK constraint CASCADE and SET NULL action processing. Requires further testing. (check-in: 35a20a5f22 user: dan tags: experimental-fk-actions)
12:46
Fix the e_walauto.test script so that it works on windows. (check-in: 7d092ebb67 user: drh tags: trunk)
01:05
Merge threading fixes from trunk into the sessions branch. (check-in: 9817a2864e user: drh tags: sessions)
00:29
Merge latest fixes and enhancements from trunk into apple-osx. (check-in: 2c1d8ddab2 user: drh tags: apple-osx)
00:20
Enhanced "stress2" testing in the threadtest3.c test program. (check-in: ae43539e62 user: drh tags: trunk)
00:08
Make sure the sqlite3BtreeCount() routine does not leave index cursors in an inconsistent state, as doing so might result in an assertion fault inside of sqlite3BtreeKey() called from saveAllCursors() if content is deleted out from under the statement that issued the sqlite3BtreeCount() call. (check-in: 5b1b697040 user: drh tags: trunk)
2014-12-15
20:49
Changes to threadtest3 so that "stress2" is more similar to the SDS stress test. (Closed-Leaf check-in: 5648af96d8 user: dan tags: threadtest3)
Changes
Unified Diff Ignore Whitespace Patch
Changes to test/threadtest3.c.
117
118
119
120
121
122
123

124


125
126
127
128
129
130
131
#  define uint32 unsigned int
#endif

struct MD5Context {
  int isInit;
  uint32 buf[4];
  uint32 bits[2];

  unsigned char in[64];


};
typedef struct MD5Context MD5Context;

/*
 * Note: this code is harmless on little-endian machines.
 */
static void byteReverse (unsigned char *buf, unsigned longs){







>
|
>
>







117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
#  define uint32 unsigned int
#endif

struct MD5Context {
  int isInit;
  uint32 buf[4];
  uint32 bits[2];
  union {
    unsigned char in[64];
    uint32 in32[16];
  } u;
};
typedef struct MD5Context MD5Context;

/*
 * Note: this code is harmless on little-endian machines.
 */
static void byteReverse (unsigned char *buf, unsigned longs){
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
  ctx->bits[1] += len >> 29;

  t = (t >> 3) & 0x3f;    /* Bytes already in shsInfo->data */

  /* Handle any leading odd-sized chunks */

  if ( t ) {
    unsigned char *p = (unsigned char *)ctx->in + t;

    t = 64-t;
    if (len < t) {
      memcpy(p, buf, len);
      return;
    }
    memcpy(p, buf, t);
    byteReverse(ctx->in, 16);
    MD5Transform(ctx->buf, (uint32 *)ctx->in);
    buf += t;
    len -= t;
  }

  /* Process data in 64-byte chunks */

  while (len >= 64) {
    memcpy(ctx->in, buf, 64);
    byteReverse(ctx->in, 16);
    MD5Transform(ctx->buf, (uint32 *)ctx->in);
    buf += 64;
    len -= 64;
  }

  /* Handle any remaining bytes of data. */

  memcpy(ctx->in, buf, len);
}

/*
 * Final wrapup - pad to 64-byte boundary with the bit pattern 
 * 1 0* (64-bit count of bits processed, MSB-first)
 */
static void MD5Final(unsigned char digest[16], MD5Context *ctx){
  unsigned count;
  unsigned char *p;

  /* Compute number of bytes mod 64 */
  count = (ctx->bits[0] >> 3) & 0x3F;

  /* Set the first char of padding to 0x80.  This is safe since there is
     always at least one byte free */
  p = ctx->in + count;
  *p++ = 0x80;

  /* Bytes of padding needed to make 64 bytes */
  count = 64 - 1 - count;

  /* Pad out to 56 mod 64 */
  if (count < 8) {
    /* Two lots of padding:  Pad the first block to 64 bytes */
    memset(p, 0, count);
    byteReverse(ctx->in, 16);
    MD5Transform(ctx->buf, (uint32 *)ctx->in);

    /* Now fill the next block with 56 bytes */
    memset(ctx->in, 0, 56);
  } else {
    /* Pad block to 56 bytes */
    memset(p, 0, count-8);
  }
  byteReverse(ctx->in, 14);

  /* Append length in bits and transform */
  ((uint32 *)ctx->in)[ 14 ] = ctx->bits[0];
  ((uint32 *)ctx->in)[ 15 ] = ctx->bits[1];

  MD5Transform(ctx->buf, (uint32 *)ctx->in);
  byteReverse((unsigned char *)ctx->buf, 4);
  memcpy(digest, ctx->buf, 16);
  memset(ctx, 0, sizeof(*ctx));    /* In case it is sensitive */
}

/*
** Convert a 128-bit MD5 digest into a 32-digit base-16 number.







|







|
|







|
|
|






|















|









|
|


|




|


|
|

|







269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
  ctx->bits[1] += len >> 29;

  t = (t >> 3) & 0x3f;    /* Bytes already in shsInfo->data */

  /* Handle any leading odd-sized chunks */

  if ( t ) {
    unsigned char *p = (unsigned char *)ctx->u.in + t;

    t = 64-t;
    if (len < t) {
      memcpy(p, buf, len);
      return;
    }
    memcpy(p, buf, t);
    byteReverse(ctx->u.in, 16);
    MD5Transform(ctx->buf, (uint32 *)ctx->u.in);
    buf += t;
    len -= t;
  }

  /* Process data in 64-byte chunks */

  while (len >= 64) {
    memcpy(ctx->u.in, buf, 64);
    byteReverse(ctx->u.in, 16);
    MD5Transform(ctx->buf, (uint32 *)ctx->u.in);
    buf += 64;
    len -= 64;
  }

  /* Handle any remaining bytes of data. */

  memcpy(ctx->u.in, buf, len);
}

/*
 * Final wrapup - pad to 64-byte boundary with the bit pattern 
 * 1 0* (64-bit count of bits processed, MSB-first)
 */
static void MD5Final(unsigned char digest[16], MD5Context *ctx){
  unsigned count;
  unsigned char *p;

  /* Compute number of bytes mod 64 */
  count = (ctx->bits[0] >> 3) & 0x3F;

  /* Set the first char of padding to 0x80.  This is safe since there is
     always at least one byte free */
  p = ctx->u.in + count;
  *p++ = 0x80;

  /* Bytes of padding needed to make 64 bytes */
  count = 64 - 1 - count;

  /* Pad out to 56 mod 64 */
  if (count < 8) {
    /* Two lots of padding:  Pad the first block to 64 bytes */
    memset(p, 0, count);
    byteReverse(ctx->u.in, 16);
    MD5Transform(ctx->buf, (uint32 *)ctx->u.in);

    /* Now fill the next block with 56 bytes */
    memset(ctx->u.in, 0, 56);
  } else {
    /* Pad block to 56 bytes */
    memset(p, 0, count-8);
  }
  byteReverse(ctx->u.in, 14);

  /* Append length in bits and transform */
  ctx->u.in32[14] = ctx->bits[0];
  ctx->u.in32[15] = ctx->bits[1];

  MD5Transform(ctx->buf, (uint32 *)ctx->u.in);
  byteReverse((unsigned char *)ctx->buf, 4);
  memcpy(digest, ctx->buf, 16);
  memset(ctx, 0, sizeof(*ctx));    /* In case it is sensitive */
}

/*
** Convert a 128-bit MD5 digest into a 32-digit base-16 number.
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
    { checkpoint_starvation_1, "checkpoint_starvation_1", 10000 },
    { checkpoint_starvation_2, "checkpoint_starvation_2", 10000 },

    { create_drop_index_1, "create_drop_index_1", 10000 },
    { lookaside1,          "lookaside1", 10000 },
    { vacuum1,             "vacuum1", 10000 },
    { stress1,             "stress1", 10000 },
    { stress2,             "stress2", 10000 },
  };

  int i;
  int bTestfound = 0;

  sqlite3_config(SQLITE_CONFIG_MULTITHREAD);
  sqlite3_config(SQLITE_CONFIG_MULTITHREAD);







|







1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
    { checkpoint_starvation_1, "checkpoint_starvation_1", 10000 },
    { checkpoint_starvation_2, "checkpoint_starvation_2", 10000 },

    { create_drop_index_1, "create_drop_index_1", 10000 },
    { lookaside1,          "lookaside1", 10000 },
    { vacuum1,             "vacuum1", 10000 },
    { stress1,             "stress1", 10000 },
    { stress2,             "stress2", 60000 },
  };

  int i;
  int bTestfound = 0;

  sqlite3_config(SQLITE_CONFIG_MULTITHREAD);
  sqlite3_config(SQLITE_CONFIG_MULTITHREAD);
Changes to test/tt3_stress.c.
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
      closedb(&err, &db);
      opendb(&err, &db, "test.db", 0);
    }
    execsql(&err, &db, "DELETE FROM t1 WHERE (rowid % 4)==:i", &i);
    i1++;
    if( err.rc ) i2++;
    clear_error(&err, SQLITE_LOCKED);
    clear_error(&err, SQLITE_ERROR);
  }
  closedb(&err, &db);
  print_and_free_err(&err);
  return sqlite3_mprintf("deleted from t1 %d/%d attempts", i2, i1);
}









<







120
121
122
123
124
125
126

127
128
129
130
131
132
133
      closedb(&err, &db);
      opendb(&err, &db, "test.db", 0);
    }
    execsql(&err, &db, "DELETE FROM t1 WHERE (rowid % 4)==:i", &i);
    i1++;
    if( err.rc ) i2++;
    clear_error(&err, SQLITE_LOCKED);

  }
  closedb(&err, &db);
  print_and_free_err(&err);
  return sqlite3_mprintf("deleted from t1 %d/%d attempts", i2, i1);
}


180
181
182
183
184
185
186
187
188



189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
** 10. Big DELETE statements.
** 11. VACUUM.
** 14. Integrity-check.
** 17. Switch the journal mode from delete to wal and back again.
** 19. Open and close database connections rapidly.
*/

#define STRESS2_TABCNT 5




static void stress2_workload1(Error *pErr, Sqlite *pDb, int i){
  int iTab = (i % STRESS2_TABCNT);
  sql_script_printf(pErr, pDb, 
      "CREATE TABLE IF NOT EXISTS t%d(x PRIMARY KEY, y, z);", iTab
  );
}

static void stress2_workload2(Error *pErr, Sqlite *pDb, int i){
  int iTab = (i % STRESS2_TABCNT);
  sql_script_printf(pErr, pDb, "DROP TABLE IF EXISTS t%d;", iTab);
}

static void stress2_workload3(Error *pErr, Sqlite *pDb, int i){
  int iTab = (i % STRESS2_TABCNT);
  sql_script_printf(pErr, pDb, "SELECT * FROM t%d WHERE z = 'small'", iTab);
}

static void stress2_workload4(Error *pErr, Sqlite *pDb, int i){
  int iTab = (i % STRESS2_TABCNT);
  sql_script_printf(pErr, pDb, "SELECT * FROM t%d WHERE z = 'big'", iTab);
}

static void stress2_workload5(Error *pErr, Sqlite *pDb, int i){
  int iTab = (i % STRESS2_TABCNT);
  sql_script_printf(pErr, pDb,
      "INSERT INTO t%d VALUES(random(), hex(randomblob(57)), 'small');", iTab
  );
}

static void stress2_workload6(Error *pErr, Sqlite *pDb, int i){
  int iTab = (i % STRESS2_TABCNT);
  sql_script_printf(pErr, pDb,
      "INSERT INTO t%d VALUES(random(), hex(randomblob(2000)), 'big');", iTab
  );
}

static void stress2_workload7(Error *pErr, Sqlite *pDb, int i){
  int iTab = (i % STRESS2_TABCNT);
  sql_script_printf(pErr, pDb,
      "UPDATE t%d SET y = hex(randomblob(57)) "
      "WHERE (x %% 5)==(%d %% 5) AND z='small';"
      ,iTab, i
  );
}
static void stress2_workload8(Error *pErr, Sqlite *pDb, int i){
  int iTab = (i % STRESS2_TABCNT);
  sql_script_printf(pErr, pDb,
      "UPDATE t%d SET y = hex(randomblob(2000)) "
      "WHERE (x %% 5)==(%d %% 5) AND z='big';"
      ,iTab, i
  );
}

static void stress2_workload9(Error *pErr, Sqlite *pDb, int i){
  int iTab = (i % STRESS2_TABCNT);
  sql_script_printf(pErr, pDb,
      "DELETE FROM t%d WHERE (x %% 5)==(%d %% 5) AND z='small';"
      ,iTab, i
  );
}
static void stress2_workload10(Error *pErr, Sqlite *pDb, int i){
  int iTab = (i % STRESS2_TABCNT);
  sql_script_printf(pErr, pDb,
      "DELETE FROM t%d WHERE (x %% 5)==(%d %% 5) AND z='big';"
      ,iTab, i
  );
}

static void stress2_workload11(Error *pErr, Sqlite *pDb, int i){
  sql_script(pErr, pDb, "VACUUM");
}








|

>
>
>

|






|




<
|



<
|



<
|
|




<
|
|




<

|
|
|



<

|
|
|




<

|
<



<

|
<







179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203

204
205
206
207

208
209
210
211

212
213
214
215
216
217

218
219
220
221
222
223

224
225
226
227
228
229
230

231
232
233
234
235
236
237
238

239
240

241
242
243

244
245

246
247
248
249
250
251
252
** 10. Big DELETE statements.
** 11. VACUUM.
** 14. Integrity-check.
** 17. Switch the journal mode from delete to wal and back again.
** 19. Open and close database connections rapidly.
*/

#define STRESS2_TABCNT 5          /* count1 in SDS test */

#define STRESS2_COUNT2 200        /* count2 in SDS test */
#define STRESS2_COUNT3  57        /* count2 in SDS test */

static void stress2_workload1(Error *pErr, Sqlite *pDb, int i){
  int iTab = (i % (STRESS2_TABCNT-1)) + 1;
  sql_script_printf(pErr, pDb, 
      "CREATE TABLE IF NOT EXISTS t%d(x PRIMARY KEY, y, z);", iTab
  );
}

static void stress2_workload2(Error *pErr, Sqlite *pDb, int i){
  int iTab = (i % (STRESS2_TABCNT-1)) + 1;
  sql_script_printf(pErr, pDb, "DROP TABLE IF EXISTS t%d;", iTab);
}

static void stress2_workload3(Error *pErr, Sqlite *pDb, int i){

  sql_script(pErr, pDb, "SELECT * FROM t0 WHERE z = 'small'");
}

static void stress2_workload4(Error *pErr, Sqlite *pDb, int i){

  sql_script(pErr, pDb, "SELECT * FROM t0 WHERE z = 'big'");
}

static void stress2_workload5(Error *pErr, Sqlite *pDb, int i){

  sql_script(pErr, pDb,
      "INSERT INTO t0 VALUES(hex(random()), hex(randomblob(200)), 'small');"
  );
}

static void stress2_workload6(Error *pErr, Sqlite *pDb, int i){

  sql_script(pErr, pDb,
      "INSERT INTO t0 VALUES(hex(random()), hex(randomblob(57)), 'big');"
  );
}

static void stress2_workload7(Error *pErr, Sqlite *pDb, int i){

  sql_script_printf(pErr, pDb,
      "UPDATE t0 SET y = hex(randomblob(200)) "
      "WHERE x LIKE hex((%d %% 5)) AND z='small';"
      ,i
  );
}
static void stress2_workload8(Error *pErr, Sqlite *pDb, int i){

  sql_script_printf(pErr, pDb,
      "UPDATE t0 SET y = hex(randomblob(57)) "
      "WHERE x LIKE hex(%d %% 5) AND z='big';"
      ,i
  );
}

static void stress2_workload9(Error *pErr, Sqlite *pDb, int i){

  sql_script_printf(pErr, pDb,
      "DELETE FROM t0 WHERE x LIKE hex(%d %% 5) AND z='small';", i

  );
}
static void stress2_workload10(Error *pErr, Sqlite *pDb, int i){

  sql_script_printf(pErr, pDb,
      "DELETE FROM t0 WHERE x LIKE hex(%d %% 5) AND z='big';", i

  );
}

static void stress2_workload11(Error *pErr, Sqlite *pDb, int i){
  sql_script(pErr, pDb, "VACUUM");
}

292
293
294
295
296
297
298
299
300



301
302
303
304
305


306
307
308
309
310
311
312
static char *stress2_thread_wrapper(int iTid, void *pArg){
  Stress2Ctx *pCtx = (Stress2Ctx*)pArg;
  Error err = {0};                /* Error code and message */
  Sqlite db = {0};                /* SQLite database connection */
  int i1 = 0;
  int i2 = 0;

  opendb(&err, &db, pCtx->zDb, 0);
  while( !timetostop(&err) ){



    pCtx->xProc(&err, &db, i1);
    i2 += (err.rc==SQLITE_OK);
    clear_error(&err, SQLITE_LOCKED);
    clear_error(&err, SQLITE_ERROR);
    i1++;


  }

  print_and_free_err(&err);
  return sqlite3_mprintf("ok %d/%d", i2, i1);
}

static void stress2_launch_thread_loop(







<

>
>
>
|
|
|
<
|
>
>







284
285
286
287
288
289
290

291
292
293
294
295
296
297

298
299
300
301
302
303
304
305
306
307
static char *stress2_thread_wrapper(int iTid, void *pArg){
  Stress2Ctx *pCtx = (Stress2Ctx*)pArg;
  Error err = {0};                /* Error code and message */
  Sqlite db = {0};                /* SQLite database connection */
  int i1 = 0;
  int i2 = 0;


  while( !timetostop(&err) ){
    int cnt;
    opendb(&err, &db, pCtx->zDb, 0);
    for(cnt=0; err.rc==SQLITE_OK && cnt<STRESS2_TABCNT; cnt++){
      pCtx->xProc(&err, &db, i1);
      i2 += (err.rc==SQLITE_OK);
      clear_error(&err, SQLITE_LOCKED);

      i1++;
    }
    closedb(&err, &db);
  }

  print_and_free_err(&err);
  return sqlite3_mprintf("ok %d/%d", i2, i1);
}

static void stress2_launch_thread_loop(
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356




357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
    { stress2_workload8 },
    { stress2_workload9 },
    { stress2_workload10 },
    { stress2_workload11 },
    { stress2_workload14 },
    { stress2_workload17 },
  };
  const char *azDb[] = {
    "test.db",
    "file::memory:?cache=shared"
  };
  int j;

  for(j=0; j<sizeof(azDb)/sizeof(azDb[0]); j++){
    int i;
    Error err = {0};
    Sqlite db = {0};
    Threadset threads = {0};
    const char *zDb = azDb[j];

    /* To make sure the db file is empty before commencing */
    opendb(&err, &db, zDb, 1);




    closedb(&err, &db);

    setstoptime(&err, nMs);
    sqlite3_enable_shared_cache(1);

    for(i=0; i<sizeof(aTask)/sizeof(aTask[0]); i++){
      stress2_launch_thread_loop(&err, &threads, zDb, aTask[i].x);
    }
    launch_thread(&err, &threads, stress2_workload19, (void*)zDb);
    launch_thread(&err, &threads, stress2_workload19, (void*)zDb);

    join_all_threads(&err, &threads);
    sqlite3_enable_shared_cache(0);
    print_and_free_err(&err);
    if( j==0 ){
      printf("Running stress2 (again, with in-memory db) for %d seconds...\n",
          nMs / 1000
      );
    }
  }
}











|
<
<
<
<

<
|
|
|
|
<

|
|
>
>
>
>
|

|
|

|
|
|
|
|

|
|
|
<
<
<
<
<
<





330
331
332
333
334
335
336
337




338

339
340
341
342

343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363






364
365
366
367
368
    { stress2_workload8 },
    { stress2_workload9 },
    { stress2_workload10 },
    { stress2_workload11 },
    { stress2_workload14 },
    { stress2_workload17 },
  };
  const char *zDb = "test.db";






  int i;
  Error err = {0};
  Sqlite db = {0};
  Threadset threads = {0};


  /* To make sure the db file is empty before commencing */
  opendb(&err, &db, zDb, 1);
  sql_script(&err, &db, 
      "CREATE TABLE IF NOT EXISTS t0(x PRIMARY KEY, y, z);"
      "CREATE INDEX IF NOT EXISTS i0 ON t0(y);"
  );
  closedb(&err, &db);

  setstoptime(&err, nMs);
  sqlite3_enable_shared_cache(1);

  for(i=0; i<sizeof(aTask)/sizeof(aTask[0]); i++){
    stress2_launch_thread_loop(&err, &threads, zDb, aTask[i].x);
  }
  launch_thread(&err, &threads, stress2_workload19, (void*)zDb);
  launch_thread(&err, &threads, stress2_workload19, (void*)zDb);

  join_all_threads(&err, &threads);
  sqlite3_enable_shared_cache(0);
  print_and_free_err(&err);






}