Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Fix a problem with OOM handling in the sorter code. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | threads |
Files: | files | file ages | folders |
SHA1: |
59cd5229e2b5be5272cf57c7e7d09e97 |
User & Date: | dan 2014-04-01 10:19:02.635 |
Context
2014-04-01
| ||
15:38 | Even if compile time option SQLITE_MAX_WORKER_THREADS is set to one or greater, set the default number of worker threads to zero. Distribute data more evenly between threads in sqlite3VdbeSorterWrite() to improve performance when sorting large amounts of data. Add new test file sort2.test. (check-in: 643c86a056 user: dan tags: threads) | |
10:19 | Fix a problem with OOM handling in the sorter code. (check-in: 59cd5229e2 user: dan tags: threads) | |
2014-03-31
| ||
19:57 | Add the SQLITE_MAX_WORKER_THREADS compile time option. And the SQLITE_CONFIG_WORKER_THREADS sqlite3_config() switch. (check-in: 2774710df8 user: dan tags: threads) | |
Changes
Changes to src/btree.c.
︙ | ︙ | |||
4584 4585 4586 4587 4588 4589 4590 | *pRes = -1; return SQLITE_OK; } } if( pIdxKey ){ xRecordCompare = sqlite3VdbeFindCompare(pIdxKey); | | | 4584 4585 4586 4587 4588 4589 4590 4591 4592 4593 4594 4595 4596 4597 4598 | *pRes = -1; return SQLITE_OK; } } if( pIdxKey ){ xRecordCompare = sqlite3VdbeFindCompare(pIdxKey); pIdxKey->errCode = 0; assert( pIdxKey->default_rc==1 || pIdxKey->default_rc==0 || pIdxKey->default_rc==-1 ); }else{ xRecordCompare = 0; /* All keys are integers */ } |
︙ | ︙ | |||
4708 4709 4710 4711 4712 4713 4714 | if( rc ){ sqlite3_free(pCellKey); goto moveto_finish; } c = xRecordCompare(nCell, pCellKey, pIdxKey, 0); sqlite3_free(pCellKey); } | | > > > | | 4708 4709 4710 4711 4712 4713 4714 4715 4716 4717 4718 4719 4720 4721 4722 4723 4724 4725 4726 4727 4728 4729 4730 4731 4732 4733 4734 4735 | if( rc ){ sqlite3_free(pCellKey); goto moveto_finish; } c = xRecordCompare(nCell, pCellKey, pIdxKey, 0); sqlite3_free(pCellKey); } assert( (pIdxKey->errCode!=SQLITE_CORRUPT || c==0) && (pIdxKey->errCode!=SQLITE_NOMEM || !pCur->pBtree->db->mallocFailed) ); if( c<0 ){ lwr = idx+1; }else if( c>0 ){ upr = idx-1; }else{ assert( c==0 ); *pRes = 0; rc = SQLITE_OK; pCur->aiIdx[pCur->iPage] = (u16)idx; if( pIdxKey->errCode ) rc = SQLITE_CORRUPT; goto moveto_finish; } if( lwr>upr ) break; assert( lwr+upr>=0 ); idx = (lwr+upr)>>1; /* idx = (lwr+upr)/2 */ } } |
︙ | ︙ |
Changes to src/sqliteInt.h.
︙ | ︙ | |||
1637 1638 1639 1640 1641 1642 1643 | ** The r1 and r2 member variables are only used by the optimized comparison ** functions vdbeRecordCompareInt() and vdbeRecordCompareString(). */ struct UnpackedRecord { KeyInfo *pKeyInfo; /* Collation and sort-order information */ u16 nField; /* Number of entries in apMem[] */ i8 default_rc; /* Comparison result if keys are equal */ | | | 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 | ** The r1 and r2 member variables are only used by the optimized comparison ** functions vdbeRecordCompareInt() and vdbeRecordCompareString(). */ struct UnpackedRecord { KeyInfo *pKeyInfo; /* Collation and sort-order information */ u16 nField; /* Number of entries in apMem[] */ i8 default_rc; /* Comparison result if keys are equal */ u8 errCode; /* Error detected by xRecordCompare (CORRUPT or NOMEM) */ Mem *aMem; /* Values */ int r1; /* Value to return if (lhs > rhs) */ int r2; /* Value to return if (rhs < lhs) */ }; /* |
︙ | ︙ |
Changes to src/test1.c.
︙ | ︙ | |||
2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 | return TCL_OK; bad_args: Tcl_AppendResult(interp, "wrong # args: should be \"", Tcl_GetStringFromObj(objv[0], 0), " <DB> <utf8> <utf16le> <utf16be>", 0); return TCL_ERROR; } /* ** When the collation needed callback is invoked, record the name of ** the requested collating function here. The recorded name is linked ** to a TCL variable and used to make sure that the requested collation ** name is correct. */ | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 | return TCL_OK; bad_args: Tcl_AppendResult(interp, "wrong # args: should be \"", Tcl_GetStringFromObj(objv[0], 0), " <DB> <utf8> <utf16le> <utf16be>", 0); return TCL_ERROR; } /* ** Usage: add_test_utf16bin_collate <db ptr> ** ** Add a utf-16 collation sequence named "utf16bin" to the database ** handle. This collation sequence compares arguments in the same way as the ** built-in collation "binary". */ static int test_utf16bin_collate_func( void *pCtx, int nA, const void *zA, int nB, const void *zB ){ int nCmp = (nA>nB ? nB : nA); int res = memcmp(zA, zB, nCmp); if( res==0 ) res = nA - nB; return res; } static int test_utf16bin_collate( void * clientData, Tcl_Interp *interp, int objc, Tcl_Obj *CONST objv[] ){ sqlite3 *db; int rc; if( objc!=2 ) goto bad_args; if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; rc = sqlite3_create_collation(db, "utf16bin", SQLITE_UTF16, 0, test_utf16bin_collate_func ); if( sqlite3TestErrCode(interp, db, rc) ) return TCL_ERROR; return TCL_OK; bad_args: Tcl_WrongNumArgs(interp, 1, objv, "DB"); return TCL_ERROR; } /* ** When the collation needed callback is invoked, record the name of ** the requested collating function here. The recorded name is linked ** to a TCL variable and used to make sure that the requested collation ** name is correct. */ |
︙ | ︙ | |||
6477 6478 6479 6480 6481 6482 6483 6484 6485 6486 6487 6488 6489 6490 | { "sqlite3_create_function_v2", test_create_function_v2, 0 }, /* Functions from os.h */ #ifndef SQLITE_OMIT_UTF16 { "add_test_collate", test_collate, 0 }, { "add_test_collate_needed", test_collate_needed, 0 }, { "add_test_function", test_function, 0 }, #endif { "sqlite3_test_errstr", test_errstr, 0 }, { "tcl_variable_type", tcl_variable_type, 0 }, #ifndef SQLITE_OMIT_SHARED_CACHE { "sqlite3_enable_shared_cache", test_enable_shared, 0 }, { "sqlite3_shared_cache_report", sqlite3BtreeSharedCacheReport, 0}, #endif | > | 6517 6518 6519 6520 6521 6522 6523 6524 6525 6526 6527 6528 6529 6530 6531 | { "sqlite3_create_function_v2", test_create_function_v2, 0 }, /* Functions from os.h */ #ifndef SQLITE_OMIT_UTF16 { "add_test_collate", test_collate, 0 }, { "add_test_collate_needed", test_collate_needed, 0 }, { "add_test_function", test_function, 0 }, { "add_test_utf16bin_collate", test_utf16bin_collate, 0 }, #endif { "sqlite3_test_errstr", test_errstr, 0 }, { "tcl_variable_type", tcl_variable_type, 0 }, #ifndef SQLITE_OMIT_SHARED_CACHE { "sqlite3_enable_shared_cache", test_enable_shared, 0 }, { "sqlite3_shared_cache_report", sqlite3BtreeSharedCacheReport, 0}, #endif |
︙ | ︙ |
Changes to src/vdbeaux.c.
︙ | ︙ | |||
3225 3226 3227 3228 3229 3230 3231 | ** using the collation sequence pColl. As usual, return a negative , zero ** or positive value if *pMem1 is less than, equal to or greater than ** *pMem2, respectively. Similar in spirit to "rc = (*pMem1) - (*pMem2);". */ static int vdbeCompareMemString( const Mem *pMem1, const Mem *pMem2, | | > | 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 | ** using the collation sequence pColl. As usual, return a negative , zero ** or positive value if *pMem1 is less than, equal to or greater than ** *pMem2, respectively. Similar in spirit to "rc = (*pMem1) - (*pMem2);". */ static int vdbeCompareMemString( const Mem *pMem1, const Mem *pMem2, const CollSeq *pColl, u8 *prcErr /* If an OOM occurs, set to SQLITE_NOMEM */ ){ if( pMem1->enc==pColl->enc ){ /* The strings are already in the correct encoding. Call the ** comparison function directly */ return pColl->xCmp(pColl->pUser,pMem1->n,pMem1->z,pMem2->n,pMem2->z); }else{ int rc; |
︙ | ︙ | |||
3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 | v1 = sqlite3ValueText((sqlite3_value*)&c1, pColl->enc); n1 = v1==0 ? 0 : c1.n; v2 = sqlite3ValueText((sqlite3_value*)&c2, pColl->enc); n2 = v2==0 ? 0 : c2.n; rc = pColl->xCmp(pColl->pUser, n1, v1, n2, v2); sqlite3VdbeMemRelease(&c1); sqlite3VdbeMemRelease(&c2); return rc; } } /* ** Compare the values contained by the two memory cells, returning ** negative, zero or positive if pMem1 is less than, equal to, or greater | > | 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 | v1 = sqlite3ValueText((sqlite3_value*)&c1, pColl->enc); n1 = v1==0 ? 0 : c1.n; v2 = sqlite3ValueText((sqlite3_value*)&c2, pColl->enc); n2 = v2==0 ? 0 : c2.n; rc = pColl->xCmp(pColl->pUser, n1, v1, n2, v2); sqlite3VdbeMemRelease(&c1); sqlite3VdbeMemRelease(&c2); if( (v1==0 || v2==0) && prcErr ) *prcErr = SQLITE_NOMEM; return rc; } } /* ** Compare the values contained by the two memory cells, returning ** negative, zero or positive if pMem1 is less than, equal to, or greater |
︙ | ︙ | |||
3330 3331 3332 3333 3334 3335 3336 | /* The collation sequence must be defined at this point, even if ** the user deletes the collation sequence after the vdbe program is ** compiled (this was not always the case). */ assert( !pColl || pColl->xCmp ); if( pColl ){ | | | 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342 3343 3344 3345 3346 | /* The collation sequence must be defined at this point, even if ** the user deletes the collation sequence after the vdbe program is ** compiled (this was not always the case). */ assert( !pColl || pColl->xCmp ); if( pColl ){ return vdbeCompareMemString(pMem1, pMem2, pColl, 0); } /* If a NULL pointer was passed as the collate function, fall through ** to the blob case and use memcmp(). */ } /* Both values must be blobs. Compare using memcmp(). */ rc = memcmp(pMem1->z, pMem2->z, (pMem1->n>pMem2->n)?pMem2->n:pMem1->n); |
︙ | ︙ | |||
3402 3403 3404 3405 3406 3407 3408 | ** If argument bSkip is non-zero, it is assumed that the caller has already ** determined that the first fields of the keys are equal. ** ** Key1 and Key2 do not have to contain the same number of fields. If all ** fields that appear in both keys are equal, then pPKey2->default_rc is ** returned. ** | | | > > | 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421 | ** If argument bSkip is non-zero, it is assumed that the caller has already ** determined that the first fields of the keys are equal. ** ** Key1 and Key2 do not have to contain the same number of fields. If all ** fields that appear in both keys are equal, then pPKey2->default_rc is ** returned. ** ** If database corruption is discovered, set pPKey2->errCode to ** SQLITE_CORRUPT and return 0. If an OOM error is encountered, ** pPKey2->errCode is set to SQLITE_NOMEM and, if it is not NULL, the ** malloc-failed flag set on database handle (pPKey2->pKeyInfo->db). */ int sqlite3VdbeRecordCompare( int nKey1, const void *pKey1, /* Left key */ UnpackedRecord *pPKey2, /* Right key */ int bSkip /* If true, skip the first field */ ){ u32 d1; /* Offset into aKey[] of next data element */ |
︙ | ︙ | |||
3434 3435 3436 3437 3438 3439 3440 | d1 = szHdr1 + sqlite3VdbeSerialTypeLen(s1); i = 1; pRhs++; }else{ idx1 = getVarint32(aKey1, szHdr1); d1 = szHdr1; if( d1>(unsigned)nKey1 ){ | | | 3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 | d1 = szHdr1 + sqlite3VdbeSerialTypeLen(s1); i = 1; pRhs++; }else{ idx1 = getVarint32(aKey1, szHdr1); d1 = szHdr1; if( d1>(unsigned)nKey1 ){ pPKey2->errCode = (u8)SQLITE_CORRUPT_BKPT; return 0; /* Corruption */ } i = 0; } VVA_ONLY( mem1.zMalloc = 0; ) /* Only needed by assert() statements */ assert( pPKey2->pKeyInfo->nField+pPKey2->pKeyInfo->nXField>=pPKey2->nField |
︙ | ︙ | |||
3513 3514 3515 3516 3517 3518 3519 | }else if( !(serial_type & 0x01) ){ rc = +1; }else{ mem1.n = (serial_type - 12) / 2; testcase( (d1+mem1.n)==(unsigned)nKey1 ); testcase( (d1+mem1.n+1)==(unsigned)nKey1 ); if( (d1+mem1.n) > (unsigned)nKey1 ){ | | | > > | | | 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 3527 3528 3529 3530 3531 3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 | }else if( !(serial_type & 0x01) ){ rc = +1; }else{ mem1.n = (serial_type - 12) / 2; testcase( (d1+mem1.n)==(unsigned)nKey1 ); testcase( (d1+mem1.n+1)==(unsigned)nKey1 ); if( (d1+mem1.n) > (unsigned)nKey1 ){ pPKey2->errCode = (u8)SQLITE_CORRUPT_BKPT; return 0; /* Corruption */ }else if( pKeyInfo->aColl[i] ){ mem1.enc = pKeyInfo->enc; mem1.db = pKeyInfo->db; mem1.flags = MEM_Str; mem1.z = (char*)&aKey1[d1]; rc = vdbeCompareMemString( &mem1, pRhs, pKeyInfo->aColl[i], &pPKey2->errCode ); }else{ int nCmp = MIN(mem1.n, pRhs->n); rc = memcmp(&aKey1[d1], pRhs->z, nCmp); if( rc==0 ) rc = mem1.n - pRhs->n; } } } /* RHS is a blob */ else if( pRhs->flags & MEM_Blob ){ getVarint32(&aKey1[idx1], serial_type); testcase( serial_type==12 ); if( serial_type<12 || (serial_type & 0x01) ){ rc = -1; }else{ int nStr = (serial_type - 12) / 2; testcase( (d1+nStr)==(unsigned)nKey1 ); testcase( (d1+nStr+1)==(unsigned)nKey1 ); if( (d1+nStr) > (unsigned)nKey1 ){ pPKey2->errCode = (u8)SQLITE_CORRUPT_BKPT; return 0; /* Corruption */ }else{ int nCmp = MIN(nStr, pRhs->n); rc = memcmp(&aKey1[d1], pRhs->z, nCmp); if( rc==0 ) rc = nStr - pRhs->n; } } } /* RHS is null */ else{ serial_type = aKey1[idx1]; rc = (serial_type!=0); } if( rc!=0 ){ if( pKeyInfo->aSortOrder[i] ){ rc = -rc; } assert( CORRUPT_DB || pKeyInfo->db==0 || (rc<0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)<0) || (rc>0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)>0) || pKeyInfo->db->mallocFailed ); assert( mem1.zMalloc==0 ); /* See comment below */ return rc; } |
︙ | ︙ | |||
3720 3721 3722 3723 3724 3725 3726 | }else{ int nCmp; int nStr; int szHdr = aKey1[0]; nStr = (serial_type-12) / 2; if( (szHdr + nStr) > nKey1 ){ | | | 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736 3737 3738 3739 3740 | }else{ int nCmp; int nStr; int szHdr = aKey1[0]; nStr = (serial_type-12) / 2; if( (szHdr + nStr) > nKey1 ){ pPKey2->errCode = (u8)SQLITE_CORRUPT_BKPT; return 0; /* Corruption */ } nCmp = MIN( pPKey2->aMem[0].n, nStr ); res = memcmp(&aKey1[szHdr], pPKey2->aMem[0].z, nCmp); if( res==0 ){ res = nStr - pPKey2->aMem[0].n; |
︙ | ︙ |
Changes to src/vdbesort.c.
︙ | ︙ | |||
1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 | ); assert( pThread->pUnpacked==(UnpackedRecord*)pFree ); if( pFree==0 ){ rc = SQLITE_NOMEM; goto thread_out; } pThread->pUnpacked->nField = pThread->pKeyInfo->nField; } if( pThread->eWork==SORTER_THREAD_CONS ){ assert( pThread->pList==0 ); while( pThread->nPMA>pThread->nConsolidate && rc==SQLITE_OK ){ int nIter = MIN(pThread->nPMA, SORTER_MAX_MERGE_COUNT); sqlite3_file *pTemp2 = 0; /* Second temp file to use */ | > | 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 | ); assert( pThread->pUnpacked==(UnpackedRecord*)pFree ); if( pFree==0 ){ rc = SQLITE_NOMEM; goto thread_out; } pThread->pUnpacked->nField = pThread->pKeyInfo->nField; pThread->pUnpacked->errCode = 0; } if( pThread->eWork==SORTER_THREAD_CONS ){ assert( pThread->pList==0 ); while( pThread->nPMA>pThread->nConsolidate && rc==SQLITE_OK ){ int nIter = MIN(pThread->nPMA, SORTER_MAX_MERGE_COUNT); sqlite3_file *pTemp2 = 0; /* Second temp file to use */ |
︙ | ︙ | |||
1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 | rc = vdbeSorterListToPMA(pThread); assert( rc!=SQLITE_OK || (nExpect==pThread->iTemp1Off) ); } } thread_out: pThread->bDone = 1; return SQLITE_INT_TO_PTR(rc); } /* ** Run the activity scheduled by the object passed as the only argument ** in the current thread. */ | > > > > | 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 | rc = vdbeSorterListToPMA(pThread); assert( rc!=SQLITE_OK || (nExpect==pThread->iTemp1Off) ); } } thread_out: pThread->bDone = 1; if( rc==SQLITE_OK && pThread->pUnpacked->errCode ){ assert( pThread->pUnpacked->errCode==SQLITE_NOMEM ); rc = SQLITE_NOMEM; } return SQLITE_INT_TO_PTR(rc); } /* ** Run the activity scheduled by the object passed as the only argument ** in the current thread. */ |
︙ | ︙ | |||
1263 1264 1265 1266 1267 1268 1269 | } } #endif if( pThread->pThread==0 ) break; } if( rc==SQLITE_OK ){ | < < | | 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 | } } #endif if( pThread->pThread==0 ) break; } if( rc==SQLITE_OK ){ assert( pThread->pThread==0 && pThread->bDone==0 ); pThread->eWork = SORTER_THREAD_TO_PMA; pThread->pList = pSorter->pRecord; pThread->nInMemory = pSorter->nInMemory; pSorter->nInMemory = 0; pSorter->pRecord = 0; if( pSorter->aMemory ){ u8 *aMem = pThread->aListMemory; pThread->aListMemory = pSorter->aMemory; pSorter->aMemory = aMem; } #if SQLITE_MAX_WORKER_THREADS>0 if( bFg || i==(pSorter->nThread-1) ){ /* Launch a background thread for this operation */ void *pCtx = (void*)pThread; assert( pSorter->aMemory==0 || pThread->aListMemory!=0 ); if( pThread->aListMemory ){ if( pSorter->aMemory==0 ){ pSorter->aMemory = sqlite3Malloc(pSorter->nMemory); if( pSorter->aMemory==0 ) return SQLITE_NOMEM; |
︙ | ︙ |
Changes to test/malloc.test.
︙ | ︙ | |||
875 876 877 878 879 880 881 882 883 884 885 886 887 888 | do_malloc_test 39 -tclprep { sqlite3 db test.db } -sqlbody { SELECT test_auxdata('abc', 'def'); } -cleanup { db close } # Ensure that no file descriptors were leaked. do_test malloc-99.X { catch {db close} set sqlite_open_file_count } {0} | > > > > > > > > > > > > > > > > > > > > > > | 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 | do_malloc_test 39 -tclprep { sqlite3 db test.db } -sqlbody { SELECT test_auxdata('abc', 'def'); } -cleanup { db close } reset_db add_test_utf16bin_collate db do_execsql_test 40.1 { CREATE TABLE t1(a); INSERT INTO t1 VALUES('fghij'); INSERT INTO t1 VALUES('pqrst'); INSERT INTO t1 VALUES('abcde'); INSERT INTO t1 VALUES('uvwxy'); INSERT INTO t1 VALUES('klmno'); } do_execsql_test 40.2 { SELECT * FROM t1 ORDER BY 1 COLLATE utf16bin; } {abcde fghij klmno pqrst uvwxy} do_faultsim_test 40.3 -faults oom-trans* -body { execsql { SELECT * FROM t1 ORDER BY 1 COLLATE utf16bin; } } -test { faultsim_test_result {0 {abcde fghij klmno pqrst uvwxy}} faultsim_integrity_check } # Ensure that no file descriptors were leaked. do_test malloc-99.X { catch {db close} set sqlite_open_file_count } {0} |
︙ | ︙ |