Overview
| Comment: | Fix an integer overflow problem in the sorter. |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | threads |
| Files: | files | file ages | folders |
| SHA1: | 9d3351b8d713232133dad149c73fb2a27c72abb1 |
| User & Date: | dan 2014-04-03 16:25:29 |
References
|
2014-04-04
| ||
| 07:52 | Add test file sort3.test, which should have been part of commit [9d3351b8d7]. check-in: dceed2c8 user: dan tags: threads | |
Context
|
2014-04-03
| ||
| 16:42 | Merge all recent changes from trunk. check-in: a0910079 user: drh tags: threads | |
| 16:25 | Fix an integer overflow problem in the sorter. check-in: 9d3351b8 user: dan tags: threads | |
| 14:29 | Fix minor errors causing compilation to fail with SQLITE_MAX_WORKER_THREADS set to a value greater than zero. check-in: 0561272a user: dan tags: threads | |
Changes
Changes to src/main.c.
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
....
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
|
assert( sizeof(db->aLimit)==sizeof(aHardLimit) );
memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit));
db->autoCommit = 1;
db->nextAutovac = -1;
db->szMmap = sqlite3GlobalConfig.szMmap;
db->nextPagesize = 0;
db->flags |= SQLITE_ShortColNames | SQLITE_EnableTrigger | SQLITE_CacheSpill
#if !defined(SQLITE_DEFAULT_AUTOMATIC_INDEX) || SQLITE_DEFAULT_AUTOMATIC_INDEX
| SQLITE_AutoIndex
#endif
#if SQLITE_DEFAULT_FILE_FORMAT<4
| SQLITE_LegacyFileFmt
#endif
................................................................................
#ifdef SQLITE_VDBE_COVERAGE
typedef void (*branch_callback)(void*,int,u8,u8);
sqlite3GlobalConfig.xVdbeBranch = va_arg(ap,branch_callback);
sqlite3GlobalConfig.pVdbeBranchArg = va_arg(ap,void*);
#endif
break;
}
}
va_end(ap);
#endif /* SQLITE_OMIT_BUILTIN_TEST */
return rc;
}
|
>
>
>
>
>
>
>
>
|
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
....
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
|
assert( sizeof(db->aLimit)==sizeof(aHardLimit) ); memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit)); db->autoCommit = 1; db->nextAutovac = -1; db->szMmap = sqlite3GlobalConfig.szMmap; db->nextPagesize = 0; db->nMaxSorterMmap = 0x7FFFFFFF; db->flags |= SQLITE_ShortColNames | SQLITE_EnableTrigger | SQLITE_CacheSpill #if !defined(SQLITE_DEFAULT_AUTOMATIC_INDEX) || SQLITE_DEFAULT_AUTOMATIC_INDEX | SQLITE_AutoIndex #endif #if SQLITE_DEFAULT_FILE_FORMAT<4 | SQLITE_LegacyFileFmt #endif ................................................................................ #ifdef SQLITE_VDBE_COVERAGE typedef void (*branch_callback)(void*,int,u8,u8); sqlite3GlobalConfig.xVdbeBranch = va_arg(ap,branch_callback); sqlite3GlobalConfig.pVdbeBranchArg = va_arg(ap,void*); #endif break; } /* sqlite3_test_control(SQLITE_TESTCTRL_SORTER_MMAP, db, nMax); */ case SQLITE_TESTCTRL_SORTER_MMAP: { sqlite3 *db = va_arg(ap, sqlite3*); db->nMaxSorterMmap = va_arg(ap, int); break; } } va_end(ap); #endif /* SQLITE_OMIT_BUILTIN_TEST */ return rc; } |
Changes to src/sqlite.h.in.
6125 6126 6127 6128 6129 6130 6131 6132 6133 6134 6135 6136 6137 6138 6139 |
#define SQLITE_TESTCTRL_OPTIMIZATIONS 15
#define SQLITE_TESTCTRL_ISKEYWORD 16
#define SQLITE_TESTCTRL_SCRATCHMALLOC 17
#define SQLITE_TESTCTRL_LOCALTIME_FAULT 18
#define SQLITE_TESTCTRL_EXPLAIN_STMT 19
#define SQLITE_TESTCTRL_NEVER_CORRUPT 20
#define SQLITE_TESTCTRL_VDBE_COVERAGE 21
#define SQLITE_TESTCTRL_LAST 21
/*
** CAPI3REF: SQLite Runtime Status
**
** ^This interface is used to retrieve runtime status information
** about the performance of SQLite, and optionally to reset various
** highwater marks. ^The first argument is an integer code for
|
> | |
6125 6126 6127 6128 6129 6130 6131 6132 6133 6134 6135 6136 6137 6138 6139 6140 |
#define SQLITE_TESTCTRL_OPTIMIZATIONS 15 #define SQLITE_TESTCTRL_ISKEYWORD 16 #define SQLITE_TESTCTRL_SCRATCHMALLOC 17 #define SQLITE_TESTCTRL_LOCALTIME_FAULT 18 #define SQLITE_TESTCTRL_EXPLAIN_STMT 19 #define SQLITE_TESTCTRL_NEVER_CORRUPT 20 #define SQLITE_TESTCTRL_VDBE_COVERAGE 21 #define SQLITE_TESTCTRL_SORTER_MMAP 22 #define SQLITE_TESTCTRL_LAST 22 /* ** CAPI3REF: SQLite Runtime Status ** ** ^This interface is used to retrieve runtime status information ** about the performance of SQLite, and optionally to reset various ** highwater marks. ^The first argument is an integer code for |
Changes to src/sqliteInt.h.
977 978 979 980 981 982 983 984 985 986 987 988 989 990 |
u8 vtabOnConflict; /* Value to return for s3_vtab_on_conflict() */
u8 isTransactionSavepoint; /* True if the outermost savepoint is a TS */
int nextPagesize; /* Pagesize after VACUUM if >0 */
u32 magic; /* Magic number for detect library misuse */
int nChange; /* Value returned by sqlite3_changes() */
int nTotalChange; /* Value returned by sqlite3_total_changes() */
int aLimit[SQLITE_N_LIMIT]; /* Limits */
struct sqlite3InitInfo { /* Information used during initialization */
int newTnum; /* Rootpage of table being initialized */
u8 iDb; /* Which db file is being initialized */
u8 busy; /* TRUE if currently initializing */
u8 orphanTrigger; /* Last statement is orphaned TEMP trigger */
} init;
int nVdbeActive; /* Number of VDBEs currently running */
|
> |
977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 |
u8 vtabOnConflict; /* Value to return for s3_vtab_on_conflict() */
u8 isTransactionSavepoint; /* True if the outermost savepoint is a TS */
int nextPagesize; /* Pagesize after VACUUM if >0 */
u32 magic; /* Magic number for detect library misuse */
int nChange; /* Value returned by sqlite3_changes() */
int nTotalChange; /* Value returned by sqlite3_total_changes() */
int aLimit[SQLITE_N_LIMIT]; /* Limits */
int nMaxSorterMmap; /* Maximum size of regions mapped by sorter */
struct sqlite3InitInfo { /* Information used during initialization */
int newTnum; /* Rootpage of table being initialized */
u8 iDb; /* Which db file is being initialized */
u8 busy; /* TRUE if currently initializing */
u8 orphanTrigger; /* Last statement is orphaned TEMP trigger */
} init;
int nVdbeActive; /* Number of VDBEs currently running */
|
Changes to src/test1.c.
5880
5881
5882
5883
5884
5885
5886
5887
5888
5889
5890
5891
5892
5893
....
5907
5908
5909
5910
5911
5912
5913
5914
5915
5916
5917
5918
5919
5920
|
Tcl_Obj *CONST objv[]
){
struct Verb {
const char *zName;
int i;
} aVerb[] = {
{ "SQLITE_TESTCTRL_LOCALTIME_FAULT", SQLITE_TESTCTRL_LOCALTIME_FAULT },
};
int iVerb;
int iFlag;
int rc;
if( objc<2 ){
Tcl_WrongNumArgs(interp, 1, objv, "VERB ARGS...");
................................................................................
Tcl_WrongNumArgs(interp, 2, objv, "ONOFF");
return TCL_ERROR;
}
if( Tcl_GetBooleanFromObj(interp, objv[2], &val) ) return TCL_ERROR;
sqlite3_test_control(SQLITE_TESTCTRL_LOCALTIME_FAULT, val);
break;
}
}
Tcl_ResetResult(interp);
return TCL_OK;
}
#if SQLITE_OS_UNIX
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|
5880
5881
5882
5883
5884
5885
5886
5887
5888
5889
5890
5891
5892
5893
5894
....
5908
5909
5910
5911
5912
5913
5914
5915
5916
5917
5918
5919
5920
5921
5922
5923
5924
5925
5926
5927
5928
5929
5930
5931
5932
5933
5934
|
Tcl_Obj *CONST objv[]
){
struct Verb {
const char *zName;
int i;
} aVerb[] = {
{ "SQLITE_TESTCTRL_LOCALTIME_FAULT", SQLITE_TESTCTRL_LOCALTIME_FAULT },
{ "SQLITE_TESTCTRL_SORTER_MMAP", SQLITE_TESTCTRL_SORTER_MMAP },
};
int iVerb;
int iFlag;
int rc;
if( objc<2 ){
Tcl_WrongNumArgs(interp, 1, objv, "VERB ARGS...");
................................................................................
Tcl_WrongNumArgs(interp, 2, objv, "ONOFF");
return TCL_ERROR;
}
if( Tcl_GetBooleanFromObj(interp, objv[2], &val) ) return TCL_ERROR;
sqlite3_test_control(SQLITE_TESTCTRL_LOCALTIME_FAULT, val);
break;
}
case SQLITE_TESTCTRL_SORTER_MMAP: {
int val;
sqlite3 *db;
if( objc!=4 ){
Tcl_WrongNumArgs(interp, 2, objv, "DB LIMIT");
return TCL_ERROR;
}
if( getDbPointer(interp, Tcl_GetString(objv[2]), &db) ) return TCL_ERROR;
if( Tcl_GetIntFromObj(interp, objv[3], &val) ) return TCL_ERROR;
sqlite3_test_control(SQLITE_TESTCTRL_SORTER_MMAP, db, val);
break;
}
}
Tcl_ResetResult(interp);
return TCL_OK;
}
#if SQLITE_OS_UNIX
|
Changes to src/vdbesort.c.
140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 ... 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 ... 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 .... 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 .... 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 .... 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 |
** Merge existing PMAs until SortSubtask.nConsolidate or fewer
** remain in temp file SortSubtask.pTemp1.
*/
struct SortSubtask {
SQLiteThread *pThread; /* Thread handle, or NULL */
int bDone; /* Set to true by pTask when finished */
sqlite3_vfs *pVfs; /* VFS used to open temporary files */
KeyInfo *pKeyInfo; /* How to compare records */
UnpackedRecord *pUnpacked; /* Space to unpack a record */
int pgsz; /* Main database page size */
u8 eWork; /* One of the SORT_SUBTASK_* constants */
int nConsolidate; /* For SORT_SUBTASK_CONS, max final PMAs */
SorterRecord *pList; /* List of records for pTask to sort */
................................................................................
pIter->pFile = pTask->pTemp1;
pIter->iReadOff = iStart;
pIter->nAlloc = 128;
pIter->aAlloc = (u8*)sqlite3Malloc(pIter->nAlloc);
if( pIter->aAlloc ){
/* Try to xFetch() a mapping of the entire temp file. If this is possible,
** the PMA will be read via the mapping. Otherwise, use xRead(). */
rc = sqlite3OsFetch(pIter->pFile, 0, pTask->iTemp1Off, &pMap);
}else{
rc = SQLITE_NOMEM;
}
if( rc==SQLITE_OK ){
if( pMap ){
pIter->aMap = (u8*)pMap;
................................................................................
if( nField && nWorker==0 ) pKeyInfo->nField = nField;
pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt);
pSorter->nTask = nWorker + 1;
for(i=0; i<pSorter->nTask; i++){
SortSubtask *pTask = &pSorter->aTask[i];
pTask->pKeyInfo = pKeyInfo;
pTask->pVfs = db->pVfs;
pTask->pgsz = pgsz;
}
if( !sqlite3TempInMemory(db) ){
pSorter->mnPmaSize = SORTER_MIN_WORKING * pgsz;
mxCache = db->aDb[0].pSchema->cache_size;
if( mxCache<SORTER_MIN_WORKING ) mxCache = SORTER_MIN_WORKING;
pSorter->mxPmaSize = mxCache * pgsz;
................................................................................
** is guaranteed to be nByte bytes or smaller in size. This function
** attempts to extend the file to nByte bytes in size and to ensure that
** the VFS has memory mapped it.
**
** Whether or not the file does end up memory mapped of course depends on
** the specific VFS implementation.
*/
static int vdbeSorterExtendFile(sqlite3_file *pFile, i64 nByte){
int rc = sqlite3OsTruncate(pFile, nByte);
if( rc==SQLITE_OK ){
void *p = 0;
sqlite3OsFetch(pFile, 0, nByte, &p);
sqlite3OsUnfetch(pFile, 0, p);
}
return rc;
}
#else
# define vdbeSorterExtendFile(x,y) SQLITE_OK
#endif
/*
** Write the current contents of the in-memory linked-list to a PMA. Return
** SQLITE_OK if successful, or an SQLite error code otherwise.
**
................................................................................
PmaWriter writer; /* Object used to write to the file */
memset(&writer, 0, sizeof(PmaWriter));
assert( pTask->nInMemory>0 );
/* If the first temporary PMA file has not been opened, open it now. */
if( pTask->pTemp1==0 ){
rc = vdbeSorterOpenTempFile(pTask->pVfs, &pTask->pTemp1);
assert( rc!=SQLITE_OK || pTask->pTemp1 );
assert( pTask->iTemp1Off==0 );
assert( pTask->nPMA==0 );
}
/* Try to get the file to memory map */
if( rc==SQLITE_OK ){
rc = vdbeSorterExtendFile(
pTask->pTemp1, pTask->iTemp1Off + pTask->nInMemory + 9
);
}
if( rc==SQLITE_OK ){
SorterRecord *p;
SorterRecord *pNext = 0;
................................................................................
pMerger = vdbeMergeEngineNew(nIter);
if( pMerger==0 ){
rc = SQLITE_NOMEM;
break;
}
/* Open a second temp file to write merged data to */
rc = vdbeSorterOpenTempFile(pTask->pVfs, &pTemp2);
if( rc==SQLITE_OK ){
rc = vdbeSorterExtendFile(pTemp2, pTask->iTemp1Off);
}
if( rc!=SQLITE_OK ){
vdbeMergeEngineFree(pMerger);
break;
}
/* This loop runs once for each output PMA. Each output PMA is made
|
| > | > < > | > > | | | | | > | | | | | |
140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 ... 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 ... 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 .... 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 .... 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 .... 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 |
** Merge existing PMAs until SortSubtask.nConsolidate or fewer
** remain in temp file SortSubtask.pTemp1.
*/
struct SortSubtask {
SQLiteThread *pThread; /* Thread handle, or NULL */
int bDone; /* Set to true by pTask when finished */
sqlite3 *db; /* Database connection */
KeyInfo *pKeyInfo; /* How to compare records */
UnpackedRecord *pUnpacked; /* Space to unpack a record */
int pgsz; /* Main database page size */
u8 eWork; /* One of the SORT_SUBTASK_* constants */
int nConsolidate; /* For SORT_SUBTASK_CONS, max final PMAs */
SorterRecord *pList; /* List of records for pTask to sort */
................................................................................
pIter->pFile = pTask->pTemp1;
pIter->iReadOff = iStart;
pIter->nAlloc = 128;
pIter->aAlloc = (u8*)sqlite3Malloc(pIter->nAlloc);
if( pIter->aAlloc ){
/* Try to xFetch() a mapping of the entire temp file. If this is possible,
** the PMA will be read via the mapping. Otherwise, use xRead(). */
if( pTask->iTemp1Off<=(i64)(pTask->db->nMaxSorterMmap) ){
rc = sqlite3OsFetch(pIter->pFile, 0, pTask->iTemp1Off, &pMap);
}
}else{
rc = SQLITE_NOMEM;
}
if( rc==SQLITE_OK ){
if( pMap ){
pIter->aMap = (u8*)pMap;
................................................................................
if( nField && nWorker==0 ) pKeyInfo->nField = nField;
pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt);
pSorter->nTask = nWorker + 1;
for(i=0; i<pSorter->nTask; i++){
SortSubtask *pTask = &pSorter->aTask[i];
pTask->pKeyInfo = pKeyInfo;
pTask->pgsz = pgsz;
pTask->db = db;
}
if( !sqlite3TempInMemory(db) ){
pSorter->mnPmaSize = SORTER_MIN_WORKING * pgsz;
mxCache = db->aDb[0].pSchema->cache_size;
if( mxCache<SORTER_MIN_WORKING ) mxCache = SORTER_MIN_WORKING;
pSorter->mxPmaSize = mxCache * pgsz;
................................................................................
** is guaranteed to be nByte bytes or smaller in size. This function
** attempts to extend the file to nByte bytes in size and to ensure that
** the VFS has memory mapped it.
**
** Whether or not the file does end up memory mapped of course depends on
** the specific VFS implementation.
*/
static int vdbeSorterExtendFile(sqlite3 *db, sqlite3_file *pFile, i64 nByte){
int rc = SQLITE_OK;
if( nByte<=(i64)(db->nMaxSorterMmap) ){
rc = sqlite3OsTruncate(pFile, nByte);
if( rc==SQLITE_OK ){
void *p = 0;
sqlite3OsFetch(pFile, 0, nByte, &p);
sqlite3OsUnfetch(pFile, 0, p);
}
}
return rc;
}
#else
# define vdbeSorterExtendFile(x,y,z) SQLITE_OK
#endif
/*
** Write the current contents of the in-memory linked-list to a PMA. Return
** SQLITE_OK if successful, or an SQLite error code otherwise.
**
................................................................................
PmaWriter writer; /* Object used to write to the file */
memset(&writer, 0, sizeof(PmaWriter));
assert( pTask->nInMemory>0 );
/* If the first temporary PMA file has not been opened, open it now. */
if( pTask->pTemp1==0 ){
rc = vdbeSorterOpenTempFile(pTask->db->pVfs, &pTask->pTemp1);
assert( rc!=SQLITE_OK || pTask->pTemp1 );
assert( pTask->iTemp1Off==0 );
assert( pTask->nPMA==0 );
}
/* Try to get the file to memory map */
if( rc==SQLITE_OK ){
rc = vdbeSorterExtendFile(pTask->db,
pTask->pTemp1, pTask->iTemp1Off + pTask->nInMemory + 9
);
}
if( rc==SQLITE_OK ){
SorterRecord *p;
SorterRecord *pNext = 0;
................................................................................
pMerger = vdbeMergeEngineNew(nIter);
if( pMerger==0 ){
rc = SQLITE_NOMEM;
break;
}
/* Open a second temp file to write merged data to */
rc = vdbeSorterOpenTempFile(pTask->db->pVfs, &pTemp2);
if( rc==SQLITE_OK ){
rc = vdbeSorterExtendFile(pTask->db, pTemp2, pTask->iTemp1Off);
}
if( rc!=SQLITE_OK ){
vdbeMergeEngineFree(pMerger);
break;
}
/* This loop runs once for each output PMA. Each output PMA is made
|
Changes to test/permutations.test.
108 109 110 111 112 113 114 115 116 117 118 119 120 121 |
savepoint4.test savepoint6.test select9.test
speed1.test speed1p.test speed2.test speed3.test speed4.test
speed4p.test sqllimits1.test tkt2686.test thread001.test thread002.test
thread003.test thread004.test thread005.test trans2.test vacuum3.test
incrvacuum_ioerr.test autovacuum_crash.test btree8.test shared_err.test
vtab_err.test walslow.test walcrash.test walcrash3.test
walthread.test rtree3.test indexfault.test securedel2.test
}]
if {[info exists ::env(QUICKTEST_INCLUDE)]} {
set allquicktests [concat $allquicktests $::env(QUICKTEST_INCLUDE)]
}
#############################################################################
# Start of tests
|
> |
108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 |
savepoint4.test savepoint6.test select9.test
speed1.test speed1p.test speed2.test speed3.test speed4.test
speed4p.test sqllimits1.test tkt2686.test thread001.test thread002.test
thread003.test thread004.test thread005.test trans2.test vacuum3.test
incrvacuum_ioerr.test autovacuum_crash.test btree8.test shared_err.test
vtab_err.test walslow.test walcrash.test walcrash3.test
walthread.test rtree3.test indexfault.test securedel2.test
sort3.test
}]
if {[info exists ::env(QUICKTEST_INCLUDE)]} {
set allquicktests [concat $allquicktests $::env(QUICKTEST_INCLUDE)]
}
#############################################################################
# Start of tests
|