Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Use the mutex-free PRNG in this branch, since anybody using this branch is probably interested in high concurrency. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | begin-concurrent |
Files: | files | file ages | folders |
SHA3-256: |
e88fc6d0cd6c470394176e259001a781 |
User & Date: | drh 2019-03-26 14:04:43.408 |
Context
2019-04-15
| ||
14:59 | Merge the latest enhancements from trunk. (check-in: 7c328e8894 user: drh tags: begin-concurrent) | |
2019-03-26
| ||
14:04 | Use the mutex-free PRNG in this branch, since anybody using this branch is probably interested in high concurrency. (check-in: e88fc6d0cd user: drh tags: begin-concurrent) | |
12:07 | Merge recent enhancements from trunk. (check-in: 774d0d5288 user: drh tags: begin-concurrent) | |
2017-09-09
| ||
08:03 | Use a mutex-free PRNG for the random() and randomblob() SQL functions and for the randomness used during checkpoint operations. (Leaf check-in: e63d1a7cd3 user: drh tags: mutex-free-randomness) | |
Changes
Changes to src/func.c.
︙ | ︙ | |||
485 486 487 488 489 490 491 492 | */ static void randomFunc( sqlite3_context *context, int NotUsed, sqlite3_value **NotUsed2 ){ sqlite_int64 r; UNUSED_PARAMETER2(NotUsed, NotUsed2); | > | | 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 | */ static void randomFunc( sqlite3_context *context, int NotUsed, sqlite3_value **NotUsed2 ){ sqlite_int64 r; sqlite3 *db = sqlite3_context_db_handle(context); UNUSED_PARAMETER2(NotUsed, NotUsed2); sqlite3FastRandomness(&db->sPrng, sizeof(r), &r); if( r<0 ){ /* We need to prevent a random number of 0x8000000000000000 ** (or -9223372036854775808) since when you do abs() of that ** number of you get the same value back again. To do this ** in a way that is testable, mask the sign bit off of negative ** values, resulting in a positive value. Then take the ** 2s complement of that positive value. The end result can |
︙ | ︙ | |||
512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 | static void randomBlob( sqlite3_context *context, int argc, sqlite3_value **argv ){ sqlite3_int64 n; unsigned char *p; assert( argc==1 ); UNUSED_PARAMETER(argc); n = sqlite3_value_int64(argv[0]); if( n<1 ){ n = 1; } p = contextMalloc(context, n); if( p ){ | > | | 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 | static void randomBlob( sqlite3_context *context, int argc, sqlite3_value **argv ){ sqlite3_int64 n; unsigned char *p; sqlite3 *db = sqlite3_context_db_handle(context); assert( argc==1 ); UNUSED_PARAMETER(argc); n = sqlite3_value_int64(argv[0]); if( n<1 ){ n = 1; } p = contextMalloc(context, n); if( p ){ sqlite3FastRandomness(&db->sPrng, n, p); sqlite3_result_blob(context, (char*)p, n, sqlite3_free); } } /* ** Implementation of the last_insert_rowid() SQL function. The return ** value is the same as the sqlite3_last_insert_rowid() API function. |
︙ | ︙ |
Changes to src/main.c.
︙ | ︙ | |||
3064 3065 3066 3067 3068 3069 3070 | } sqlite3_mutex_enter(db->mutex); db->errMask = 0xff; db->nDb = 2; db->magic = SQLITE_MAGIC_BUSY; db->aDb = db->aDbStatic; db->lookaside.bDisable = 1; | | | 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075 3076 3077 3078 | } sqlite3_mutex_enter(db->mutex); db->errMask = 0xff; db->nDb = 2; db->magic = SQLITE_MAGIC_BUSY; db->aDb = db->aDbStatic; db->lookaside.bDisable = 1; sqlite3FastPrngInit(&db->sPrng); assert( sizeof(db->aLimit)==sizeof(aHardLimit) ); memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit)); db->aLimit[SQLITE_LIMIT_WORKER_THREADS] = SQLITE_DEFAULT_WORKER_THREADS; db->autoCommit = 1; db->nextAutovac = -1; db->szMmap = sqlite3GlobalConfig.szMmap; db->nextPagesize = 0; |
︙ | ︙ |
Changes to src/random.c.
︙ | ︙ | |||
101 102 103 104 105 106 107 108 109 110 111 112 113 114 | wsdPrng.s[wsdPrng.i] = wsdPrng.s[wsdPrng.j]; wsdPrng.s[wsdPrng.j] = t; t += wsdPrng.s[wsdPrng.i]; *(zBuf++) = wsdPrng.s[t]; }while( --N ); sqlite3_mutex_leave(mutex); } #ifndef SQLITE_UNTESTABLE /* ** For testing purposes, we sometimes want to preserve the state of ** PRNG and restore the PRNG to its saved state at a later time, or ** to reset the PRNG to its initial state. These routines accomplish ** those tasks. | > > > > > > > > > > > > > > > > > > > > > > | 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 | wsdPrng.s[wsdPrng.i] = wsdPrng.s[wsdPrng.j]; wsdPrng.s[wsdPrng.j] = t; t += wsdPrng.s[wsdPrng.i]; *(zBuf++) = wsdPrng.s[t]; }while( --N ); sqlite3_mutex_leave(mutex); } /* ** Initialize a fast PRNG. A Fast PRNG is called "fast" because it does ** not need a mutex to operate, though it does use a mutex to initialize. ** The quality of the randomness is not as good as the global PRNG. */ void sqlite3FastPrngInit(FastPrng *pPrng){ sqlite3_randomness(sizeof(*pPrng), pPrng); pPrng->x |= 1; } /* ** Generate N bytes of pseudo-randomness using a FastPrng */ void sqlite3FastRandomness(FastPrng *pPrng, int N, void *P){ unsigned char *pOut = (unsigned char*)P; while( N-->0 ){ pPrng->x = ((pPrng->x)>>1) ^ ((1+~((pPrng->x)&1)) & 0xd0000001); pPrng->y = (pPrng->y)*1103515245 + 12345; *(pOut++) = (pPrng->x ^ pPrng->y) & 0xff; } } #ifndef SQLITE_UNTESTABLE /* ** For testing purposes, we sometimes want to preserve the state of ** PRNG and restore the PRNG to its saved state at a later time, or ** to reset the PRNG to its initial state. These routines accomplish ** those tasks. |
︙ | ︙ |
Changes to src/select.c.
︙ | ︙ | |||
1995 1996 1997 1998 1999 2000 2001 | while( zName && sqlite3HashFind(&ht, zName)!=0 ){ nName = sqlite3Strlen30(zName); if( nName>0 ){ for(j=nName-1; j>0 && sqlite3Isdigit(zName[j]); j--){} if( zName[j]==':' ) nName = j; } zName = sqlite3MPrintf(db, "%.*z:%u", nName, zName, ++cnt); | | | 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 | while( zName && sqlite3HashFind(&ht, zName)!=0 ){ nName = sqlite3Strlen30(zName); if( nName>0 ){ for(j=nName-1; j>0 && sqlite3Isdigit(zName[j]); j--){} if( zName[j]==':' ) nName = j; } zName = sqlite3MPrintf(db, "%.*z:%u", nName, zName, ++cnt); if( cnt>3 ) sqlite3FastRandomness(&db->sPrng, sizeof(cnt), &cnt); } pCol->zName = zName; sqlite3ColumnPropertiesFromName(0, pCol); if( zName && sqlite3HashInsert(&ht, zName, pCol)==pCol ){ sqlite3OomFault(db); } } |
︙ | ︙ |
Changes to src/sqliteInt.h.
︙ | ︙ | |||
1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 | typedef struct Bitvec Bitvec; typedef struct CollSeq CollSeq; typedef struct Column Column; typedef struct Db Db; typedef struct Schema Schema; typedef struct Expr Expr; typedef struct ExprList ExprList; typedef struct FKey FKey; typedef struct FuncDestructor FuncDestructor; typedef struct FuncDef FuncDef; typedef struct FuncDefHash FuncDefHash; typedef struct IdList IdList; typedef struct Index Index; typedef struct IndexSample IndexSample; | > | 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 | typedef struct Bitvec Bitvec; typedef struct CollSeq CollSeq; typedef struct Column Column; typedef struct Db Db; typedef struct Schema Schema; typedef struct Expr Expr; typedef struct ExprList ExprList; typedef struct FastPrng FastPrng; typedef struct FKey FKey; typedef struct FuncDestructor FuncDestructor; typedef struct FuncDef FuncDef; typedef struct FuncDefHash FuncDefHash; typedef struct IdList IdList; typedef struct Index Index; typedef struct IndexSample IndexSample; |
︙ | ︙ | |||
1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 | */ #ifndef SQLITE_DEFAULT_SYNCHRONOUS # define SQLITE_DEFAULT_SYNCHRONOUS 2 #endif #ifndef SQLITE_DEFAULT_WAL_SYNCHRONOUS # define SQLITE_DEFAULT_WAL_SYNCHRONOUS SQLITE_DEFAULT_SYNCHRONOUS #endif /* ** Each database file to be accessed by the system is an instance ** of the following structure. There are normally two of these structures ** in the sqlite.aDb[] array. aDb[0] is the main database file and ** aDb[1] is the database file used to hold temporary tables. Additional ** databases may be attached. | > > > > > > > > | 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 | */ #ifndef SQLITE_DEFAULT_SYNCHRONOUS # define SQLITE_DEFAULT_SYNCHRONOUS 2 #endif #ifndef SQLITE_DEFAULT_WAL_SYNCHRONOUS # define SQLITE_DEFAULT_WAL_SYNCHRONOUS SQLITE_DEFAULT_SYNCHRONOUS #endif /* ** State of a simple PRNG used for the per-connection and per-pager ** pseudo-random number generators. */ struct FastPrng { unsigned int x, y; }; /* ** Each database file to be accessed by the system is an instance ** of the following structure. There are normally two of these structures ** in the sqlite.aDb[] array. aDb[0] is the main database file and ** aDb[1] is the database file used to hold temporary tables. Additional ** databases may be attached. |
︙ | ︙ | |||
1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 | u8 vtabOnConflict; /* Value to return for s3_vtab_on_conflict() */ u8 isTransactionSavepoint; /* True if the outermost savepoint is a TS */ u8 mTrace; /* zero or more SQLITE_TRACE flags */ u8 noSharedCache; /* True if no shared-cache backends */ u8 nSqlExec; /* Number of pending OP_SqlExec opcodes */ int nextPagesize; /* Pagesize after VACUUM if >0 */ u32 magic; /* Magic number for detect library misuse */ int nChange; /* Value returned by sqlite3_changes() */ int nTotalChange; /* Value returned by sqlite3_total_changes() */ int aLimit[SQLITE_N_LIMIT]; /* Limits */ int nMaxSorterMmap; /* Maximum size of regions mapped by sorter */ struct sqlite3InitInfo { /* Information used during initialization */ int newTnum; /* Rootpage of table being initialized */ u8 iDb; /* Which db file is being initialized */ | > | 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 | u8 vtabOnConflict; /* Value to return for s3_vtab_on_conflict() */ u8 isTransactionSavepoint; /* True if the outermost savepoint is a TS */ u8 mTrace; /* zero or more SQLITE_TRACE flags */ u8 noSharedCache; /* True if no shared-cache backends */ u8 nSqlExec; /* Number of pending OP_SqlExec opcodes */ int nextPagesize; /* Pagesize after VACUUM if >0 */ u32 magic; /* Magic number for detect library misuse */ FastPrng sPrng; /* State of the per-connection PRNG */ int nChange; /* Value returned by sqlite3_changes() */ int nTotalChange; /* Value returned by sqlite3_total_changes() */ int aLimit[SQLITE_N_LIMIT]; /* Limits */ int nMaxSorterMmap; /* Maximum size of regions mapped by sorter */ struct sqlite3InitInfo { /* Information used during initialization */ int newTnum; /* Rootpage of table being initialized */ u8 iDb; /* Which db file is being initialized */ |
︙ | ︙ | |||
4023 4024 4025 4026 4027 4028 4029 4030 4031 4032 4033 4034 4035 4036 | int sqlite3ExprCoveredByIndex(Expr*, int iCur, Index *pIdx); int sqlite3FunctionUsesThisSrc(Expr*, SrcList*); Vdbe *sqlite3GetVdbe(Parse*); #ifndef SQLITE_UNTESTABLE void sqlite3PrngSaveState(void); void sqlite3PrngRestoreState(void); #endif void sqlite3RollbackAll(sqlite3*,int); void sqlite3CodeVerifySchema(Parse*, int); void sqlite3CodeVerifyNamedSchema(Parse*, const char *zDb); void sqlite3BeginTransaction(Parse*, int); void sqlite3EndTransaction(Parse*,int); void sqlite3Savepoint(Parse*, int, Token*); void sqlite3CloseSavepoints(sqlite3 *); | > > | 4033 4034 4035 4036 4037 4038 4039 4040 4041 4042 4043 4044 4045 4046 4047 4048 | int sqlite3ExprCoveredByIndex(Expr*, int iCur, Index *pIdx); int sqlite3FunctionUsesThisSrc(Expr*, SrcList*); Vdbe *sqlite3GetVdbe(Parse*); #ifndef SQLITE_UNTESTABLE void sqlite3PrngSaveState(void); void sqlite3PrngRestoreState(void); #endif void sqlite3FastPrngInit(FastPrng*); void sqlite3FastRandomness(FastPrng*, int N, void *P); void sqlite3RollbackAll(sqlite3*,int); void sqlite3CodeVerifySchema(Parse*, int); void sqlite3CodeVerifyNamedSchema(Parse*, const char *zDb); void sqlite3BeginTransaction(Parse*, int); void sqlite3EndTransaction(Parse*,int); void sqlite3Savepoint(Parse*, int, Token*); void sqlite3CloseSavepoints(sqlite3 *); |
︙ | ︙ |
Changes to src/wal.c.
︙ | ︙ | |||
470 471 472 473 474 475 476 477 478 479 480 481 482 483 | u8 bShmUnreliable; /* SHM content is read-only and unreliable */ WalIndexHdr hdr; /* Wal-index header for current transaction */ u32 minFrame; /* Ignore wal frames before this one */ u32 iReCksum; /* On commit, recalculate checksums from here */ u32 nPriorFrame; /* For sqlite3WalInfo() */ const char *zWalName; /* Name of WAL file */ u32 nCkpt; /* Checkpoint sequence counter in the wal-header */ #ifdef SQLITE_DEBUG u8 lockError; /* True if a locking error has occurred */ #endif #ifdef SQLITE_ENABLE_SNAPSHOT WalIndexHdr *pSnapshot; /* Start transaction here if not NULL */ #endif }; | > | 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 | u8 bShmUnreliable; /* SHM content is read-only and unreliable */ WalIndexHdr hdr; /* Wal-index header for current transaction */ u32 minFrame; /* Ignore wal frames before this one */ u32 iReCksum; /* On commit, recalculate checksums from here */ u32 nPriorFrame; /* For sqlite3WalInfo() */ const char *zWalName; /* Name of WAL file */ u32 nCkpt; /* Checkpoint sequence counter in the wal-header */ FastPrng sPrng; /* Random number generator */ #ifdef SQLITE_DEBUG u8 lockError; /* True if a locking error has occurred */ #endif #ifdef SQLITE_ENABLE_SNAPSHOT WalIndexHdr *pSnapshot; /* Start transaction here if not NULL */ #endif }; |
︙ | ︙ | |||
1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 | pRet->pDbFd = pDbFd; pRet->readLock = -1; pRet->mxWalSize = mxWalSize; pRet->zWalName = zWalName; pRet->syncHeader = 1; pRet->padToSectorBoundary = 1; pRet->exclusiveMode = (bNoShm ? WAL_HEAPMEMORY_MODE: WAL_NORMAL_MODE); /* Open file handle on the write-ahead log file. */ flags = (SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_WAL); rc = sqlite3OsOpen(pVfs, zWalName, pRet->pWalFd, flags, &flags); if( rc==SQLITE_OK && flags&SQLITE_OPEN_READONLY ){ pRet->readOnly = WAL_RDONLY; } | > | 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 | pRet->pDbFd = pDbFd; pRet->readLock = -1; pRet->mxWalSize = mxWalSize; pRet->zWalName = zWalName; pRet->syncHeader = 1; pRet->padToSectorBoundary = 1; pRet->exclusiveMode = (bNoShm ? WAL_HEAPMEMORY_MODE: WAL_NORMAL_MODE); sqlite3FastPrngInit(&pRet->sPrng); /* Open file handle on the write-ahead log file. */ flags = (SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_WAL); rc = sqlite3OsOpen(pVfs, zWalName, pRet->pWalFd, flags, &flags); if( rc==SQLITE_OK && flags&SQLITE_OPEN_READONLY ){ pRet->readOnly = WAL_RDONLY; } |
︙ | ︙ | |||
1922 1923 1924 1925 1926 1927 1928 | */ if( rc==SQLITE_OK && eMode!=SQLITE_CHECKPOINT_PASSIVE ){ assert( pWal->writeLock ); if( pInfo->nBackfill<pWal->hdr.mxFrame ){ rc = SQLITE_BUSY; }else if( eMode>=SQLITE_CHECKPOINT_RESTART ){ u32 salt1; | | | 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 | */ if( rc==SQLITE_OK && eMode!=SQLITE_CHECKPOINT_PASSIVE ){ assert( pWal->writeLock ); if( pInfo->nBackfill<pWal->hdr.mxFrame ){ rc = SQLITE_BUSY; }else if( eMode>=SQLITE_CHECKPOINT_RESTART ){ u32 salt1; sqlite3FastRandomness(&pWal->sPrng, 4, &salt1); assert( pInfo->nBackfill==pWal->hdr.mxFrame ); rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(1), WAL_NREADER-1); if( rc==SQLITE_OK ){ if( eMode==SQLITE_CHECKPOINT_TRUNCATE ){ /* IMPLEMENTATION-OF: R-44699-57140 This mode works the same way as ** SQLITE_CHECKPOINT_RESTART with the addition that it also ** truncates the log file to zero bytes just prior to a |
︙ | ︙ | |||
3362 3363 3364 3365 3366 3367 3368 | int rc = SQLITE_OK; if( pWal->readLock==0 ){ volatile WalCkptInfo *pInfo = walCkptInfo(pWal); assert( pInfo->nBackfill==pWal->hdr.mxFrame ); if( pInfo->nBackfill>0 ){ u32 salt1; | | | 3364 3365 3366 3367 3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 | int rc = SQLITE_OK; if( pWal->readLock==0 ){ volatile WalCkptInfo *pInfo = walCkptInfo(pWal); assert( pInfo->nBackfill==pWal->hdr.mxFrame ); if( pInfo->nBackfill>0 ){ u32 salt1; sqlite3FastRandomness(&pWal->sPrng, 4, &salt1); rc = walLockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); if( rc==SQLITE_OK ){ /* If all readers are using WAL_READ_LOCK(0) (in other words if no ** readers are currently using the WAL), then the transactions ** frames will overwrite the start of the existing log. Update the ** wal-index header to reflect this. ** |
︙ | ︙ | |||
3578 3579 3580 3581 3582 3583 3584 | u8 aWalHdr[WAL_HDRSIZE]; /* Buffer to assemble wal-header in */ u32 aCksum[2]; /* Checksum for wal-header */ sqlite3Put4byte(&aWalHdr[0], (WAL_MAGIC | SQLITE_BIGENDIAN)); sqlite3Put4byte(&aWalHdr[4], WAL_MAX_VERSION); sqlite3Put4byte(&aWalHdr[8], szPage); sqlite3Put4byte(&aWalHdr[12], pWal->nCkpt); | | | 3580 3581 3582 3583 3584 3585 3586 3587 3588 3589 3590 3591 3592 3593 3594 | u8 aWalHdr[WAL_HDRSIZE]; /* Buffer to assemble wal-header in */ u32 aCksum[2]; /* Checksum for wal-header */ sqlite3Put4byte(&aWalHdr[0], (WAL_MAGIC | SQLITE_BIGENDIAN)); sqlite3Put4byte(&aWalHdr[4], WAL_MAX_VERSION); sqlite3Put4byte(&aWalHdr[8], szPage); sqlite3Put4byte(&aWalHdr[12], pWal->nCkpt); if( pWal->nCkpt==0 ) sqlite3FastRandomness(&pWal->sPrng, 8, pWal->hdr.aSalt); memcpy(&aWalHdr[16], pWal->hdr.aSalt, 8); walChecksumBytes(1, aWalHdr, WAL_HDRSIZE-2*4, 0, aCksum); sqlite3Put4byte(&aWalHdr[24], aCksum[0]); sqlite3Put4byte(&aWalHdr[28], aCksum[1]); pWal->szPage = szPage; pWal->hdr.bigEndCksum = SQLITE_BIGENDIAN; |
︙ | ︙ |
Changes to test/concurrent2.test.
︙ | ︙ | |||
339 340 341 342 343 344 345 | #------------------------------------------------------------------------- # Test that "PRAGMA integrity_check" works within a concurrent # transaction. Within a concurrent transaction, "PRAGMA integrity_check" # is unable to detect unused database pages, but can detect other types # of corruption. # reset_db | | > | | | | | | | | | > | > | | 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 | #------------------------------------------------------------------------- # Test that "PRAGMA integrity_check" works within a concurrent # transaction. Within a concurrent transaction, "PRAGMA integrity_check" # is unable to detect unused database pages, but can detect other types # of corruption. # reset_db do_test 8.1 { execsql { PRAGMA journal_mode = wal; CREATE TABLE kv(k INTEGER PRIMARY KEY, v UNIQUE); INSERT INTO kv VALUES(NULL, randomblob(750)); INSERT INTO kv SELECT NULL, randomblob(750) FROM kv; INSERT INTO kv SELECT NULL, randomblob(750) FROM kv; INSERT INTO kv SELECT NULL, randomblob(750) FROM kv; INSERT INTO kv SELECT NULL, randomblob(750) FROM kv; INSERT INTO kv SELECT NULL, randomblob(750) FROM kv; DELETE FROM kv WHERE rowid%2; } set v [db one {PRAGMA freelist_count}] expr $v==33 || $v==34 } {1} do_execsql_test 8.2 { PRAGMA integrity_check } ok do_execsql_test 8.3 { BEGIN CONCURRENT; PRAGMA integrity_check; } {ok} do_execsql_test 8.4 { INSERT INTO kv VALUES(1100, 1100); |
︙ | ︙ |