Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Fix a bug in the hard_heap_limit pragma so that it returns the new value of the hard_heap_limit, not the soft_heap_limit. Change SQLITE_MAX_MEMORY so that it works by setting the default hard_heap_limit value. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | hard-heap-limit |
Files: | files | file ages | folders |
SHA3-256: |
33fd0c3abcad5555a150990a22d9c1ba |
User & Date: | drh 2019-11-14 17:46:32.327 |
Context
2019-11-14
| ||
18:07 | Add support for the sqlite3_hard_heap_limit64() interface and the hard_heap_limit pragma. (check-in: 6399c47ea8 user: drh tags: trunk) | |
17:46 | Fix a bug in the hard_heap_limit pragma so that it returns the new value of the hard_heap_limit, not the soft_heap_limit. Change SQLITE_MAX_MEMORY so that it works by setting the default hard_heap_limit value. (Closed-Leaf check-in: 33fd0c3abc user: drh tags: hard-heap-limit) | |
15:21 | Merge recent enhancements from trunk. (check-in: b8a631fd30 user: drh tags: hard-heap-limit) | |
Changes
Changes to src/malloc.c.
︙ | ︙ | |||
28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 | ** is a no-op returning zero if SQLite is not compiled with ** SQLITE_ENABLE_MEMORY_MANAGEMENT. */ UNUSED_PARAMETER(n); return 0; #endif } /* ** State information local to the memory allocation subsystem. */ static SQLITE_WSD struct Mem0Global { sqlite3_mutex *mutex; /* Mutex to serialize access */ sqlite3_int64 alarmThreshold; /* The soft heap limit */ sqlite3_int64 hardLimit; /* The hard upper bound on memory */ /* ** True if heap is nearly "full" where "full" is defined by the ** sqlite3_soft_heap_limit() setting. */ int nearlyFull; | > > > > > > > | | 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 | ** is a no-op returning zero if SQLite is not compiled with ** SQLITE_ENABLE_MEMORY_MANAGEMENT. */ UNUSED_PARAMETER(n); return 0; #endif } /* ** Default value of the hard heap limit. 0 means "no limit". */ #ifndef SQLITE_MAX_MEMORY # define SQLITE_MAX_MEMORY 0 #endif /* ** State information local to the memory allocation subsystem. */ static SQLITE_WSD struct Mem0Global { sqlite3_mutex *mutex; /* Mutex to serialize access */ sqlite3_int64 alarmThreshold; /* The soft heap limit */ sqlite3_int64 hardLimit; /* The hard upper bound on memory */ /* ** True if heap is nearly "full" where "full" is defined by the ** sqlite3_soft_heap_limit() setting. */ int nearlyFull; } mem0 = { 0, SQLITE_MAX_MEMORY, SQLITE_MAX_MEMORY, 0 }; #define mem0 GLOBAL(struct Mem0Global, mem0) /* ** Return the memory allocator mutex. sqlite3_status() needs it. */ sqlite3_mutex *sqlite3MallocMutex(void){ |
︙ | ︙ | |||
228 229 230 231 232 233 234 | /* In Firefox (circa 2017-02-08), xRoundup() is remapped to an internal ** implementation of malloc_good_size(), which must be called in debug ** mode and specifically when the DMD "Dark Matter Detector" is enabled ** or else a crash results. Hence, do not attempt to optimize out the ** following xRoundup() call. */ nFull = sqlite3GlobalConfig.m.xRoundup(n); | < < < < < < < | 235 236 237 238 239 240 241 242 243 244 245 246 247 248 | /* In Firefox (circa 2017-02-08), xRoundup() is remapped to an internal ** implementation of malloc_good_size(), which must be called in debug ** mode and specifically when the DMD "Dark Matter Detector" is enabled ** or else a crash results. Hence, do not attempt to optimize out the ** following xRoundup() call. */ nFull = sqlite3GlobalConfig.m.xRoundup(n); sqlite3StatusHighwater(SQLITE_STATUS_MALLOC_SIZE, n); if( mem0.alarmThreshold>0 ){ sqlite3_int64 nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); if( nUsed >= mem0.alarmThreshold - nFull ){ mem0.nearlyFull = 1; sqlite3MallocAlarm(nFull); if( mem0.hardLimit ){ |
︙ | ︙ |
Changes to src/pragma.c.
︙ | ︙ | |||
2102 2103 2104 2105 2106 2107 2108 | */ case PragTyp_HARD_HEAP_LIMIT: { sqlite3_int64 N; if( zRight && sqlite3DecOrHexToI64(zRight, &N)==SQLITE_OK ){ sqlite3_int64 iPrior = sqlite3_hard_heap_limit64(-1); if( N>0 && (iPrior==0 || iPrior>N) ) sqlite3_hard_heap_limit64(N); } | | | 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 | */ case PragTyp_HARD_HEAP_LIMIT: { sqlite3_int64 N; if( zRight && sqlite3DecOrHexToI64(zRight, &N)==SQLITE_OK ){ sqlite3_int64 iPrior = sqlite3_hard_heap_limit64(-1); if( N>0 && (iPrior==0 || iPrior>N) ) sqlite3_hard_heap_limit64(N); } returnSingleInt(v, sqlite3_hard_heap_limit64(-1)); break; } /* ** PRAGMA threads ** PRAGMA threads = N ** |
︙ | ︙ |
Changes to test/fuzzcheck.c.
︙ | ︙ | |||
448 449 450 451 452 453 454 455 456 457 458 459 460 461 | static int giTimeout = 10000; /* Defaults to 10 seconds */ /* Maximum number of progress handler callbacks */ static unsigned int mxProgressCb = 2000; /* Maximum string length in SQLite */ static int lengthLimit = 1000000; /* Maximum byte-code program length in SQLite */ static int vdbeOpLimit = 25000; /* Maximum size of the in-memory database */ static sqlite3_int64 maxDbSize = 104857600; | > > > | 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 | static int giTimeout = 10000; /* Defaults to 10 seconds */ /* Maximum number of progress handler callbacks */ static unsigned int mxProgressCb = 2000; /* Maximum string length in SQLite */ static int lengthLimit = 1000000; /* Limit on the amount of heap memory that can be used */ static sqlite3_int64 heapLimit = 1000000000; /* Maximum byte-code program length in SQLite */ static int vdbeOpLimit = 25000; /* Maximum size of the in-memory database */ static sqlite3_int64 maxDbSize = 104857600; |
︙ | ︙ | |||
773 774 775 776 777 778 779 780 781 782 783 784 785 786 | ** maximum length of a string or blob */ if( vdbeOpLimit>0 ){ sqlite3_limit(cx.db, SQLITE_LIMIT_VDBE_OP, vdbeOpLimit); } if( lengthLimit>0 ){ sqlite3_limit(cx.db, SQLITE_LIMIT_LENGTH, lengthLimit); } if( nDb>=20 && aDb[18]==2 && aDb[19]==2 ){ aDb[18] = aDb[19] = 1; } rc = sqlite3_deserialize(cx.db, "main", aDb, nDb, nDb, SQLITE_DESERIALIZE_RESIZEABLE | SQLITE_DESERIALIZE_FREEONCLOSE); | > | 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 | ** maximum length of a string or blob */ if( vdbeOpLimit>0 ){ sqlite3_limit(cx.db, SQLITE_LIMIT_VDBE_OP, vdbeOpLimit); } if( lengthLimit>0 ){ sqlite3_limit(cx.db, SQLITE_LIMIT_LENGTH, lengthLimit); } sqlite3_hard_heap_limit64(heapLimit); if( nDb>=20 && aDb[18]==2 && aDb[19]==2 ){ aDb[18] = aDb[19] = 1; } rc = sqlite3_deserialize(cx.db, "main", aDb, nDb, nDb, SQLITE_DESERIALIZE_RESIZEABLE | SQLITE_DESERIALIZE_FREEONCLOSE); |
︙ | ︙ | |||
1337 1338 1339 1340 1341 1342 1343 | int iSrcDb; /* Loop over all source databases */ int nTest = 0; /* Total number of tests performed */ char *zDbName = ""; /* Appreviated name of a source database */ const char *zFailCode = 0; /* Value of the TEST_FAILURE env variable */ int cellSzCkFlag = 0; /* --cell-size-check */ int sqlFuzz = 0; /* True for SQL fuzz. False for DB fuzz */ int iTimeout = 120; /* Default 120-second timeout */ | | | 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 | int iSrcDb; /* Loop over all source databases */ int nTest = 0; /* Total number of tests performed */ char *zDbName = ""; /* Appreviated name of a source database */ const char *zFailCode = 0; /* Value of the TEST_FAILURE env variable */ int cellSzCkFlag = 0; /* --cell-size-check */ int sqlFuzz = 0; /* True for SQL fuzz. False for DB fuzz */ int iTimeout = 120; /* Default 120-second timeout */ int nMem = 0; /* Memory limit override */ int nMemThisDb = 0; /* Memory limit set by the CONFIG table */ char *zExpDb = 0; /* Write Databases to files in this directory */ char *zExpSql = 0; /* Write SQL to files in this directory */ void *pHeap = 0; /* Heap for use by SQLite */ int ossFuzz = 0; /* enable OSS-FUZZ testing */ int ossFuzzThisDb = 0; /* ossFuzz value for this particular database */ int nativeMalloc = 0; /* Turn off MEMSYS3/5 and lookaside if true */ |
︙ | ︙ | |||
1387 1388 1389 1390 1391 1392 1393 | showHelp(); return 0; }else if( strcmp(z,"info")==0 ){ infoFlag = 1; }else if( strcmp(z,"limit-mem")==0 ){ | < < < < < | 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 | showHelp(); return 0; }else if( strcmp(z,"info")==0 ){ infoFlag = 1; }else if( strcmp(z,"limit-mem")==0 ){ if( i>=argc-1 ) fatalError("missing arguments on %s", argv[i]); nMem = integerValue(argv[++i]); }else if( strcmp(z,"limit-vdbe")==0 ){ vdbeLimitFlag = 1; }else if( strcmp(z,"load-sql")==0 ){ zInsSql = "INSERT INTO xsql(sqltext)VALUES(CAST(readfile(?1) AS text))"; iFirstInsArg = i+1; |
︙ | ︙ | |||
1582 1583 1584 1585 1586 1587 1588 | while( SQLITE_ROW==sqlite3_step(pStmt) ){ const char *zName = (const char *)sqlite3_column_text(pStmt,0); if( zName==0 ) continue; if( strcmp(zName, "oss-fuzz")==0 ){ ossFuzzThisDb = sqlite3_column_int(pStmt,1); if( verboseFlag ) printf("Config: oss-fuzz=%d\n", ossFuzzThisDb); } | | < < < < < | 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 | while( SQLITE_ROW==sqlite3_step(pStmt) ){ const char *zName = (const char *)sqlite3_column_text(pStmt,0); if( zName==0 ) continue; if( strcmp(zName, "oss-fuzz")==0 ){ ossFuzzThisDb = sqlite3_column_int(pStmt,1); if( verboseFlag ) printf("Config: oss-fuzz=%d\n", ossFuzzThisDb); } if( strcmp(zName, "limit-mem")==0 ){ nMemThisDb = sqlite3_column_int(pStmt,1); if( verboseFlag ) printf("Config: limit-mem=%d\n", nMemThisDb); } } sqlite3_finalize(pStmt); } if( zInsSql ){ sqlite3_create_function(db, "readfile", 1, SQLITE_UTF8, 0, |
︙ | ︙ | |||
1716 1717 1718 1719 1720 1721 1722 | sqlite3_close(db); if( sqlite3_memory_used()>0 ){ fatalError("SQLite has memory in use before the start of testing"); } /* Limit available memory, if requested */ sqlite3_shutdown(); | | > | | | | | > > > > > | 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 | sqlite3_close(db); if( sqlite3_memory_used()>0 ){ fatalError("SQLite has memory in use before the start of testing"); } /* Limit available memory, if requested */ sqlite3_shutdown(); if( nMemThisDb>0 && nMem==0 ){ if( !nativeMalloc ){ pHeap = realloc(pHeap, nMemThisDb); if( pHeap==0 ){ fatalError("failed to allocate %d bytes of heap memory", nMem); } sqlite3_config(SQLITE_CONFIG_HEAP, pHeap, nMemThisDb, 128); }else{ sqlite3_hard_heap_limit64((sqlite3_int64)nMemThisDb); } }else{ sqlite3_hard_heap_limit64(0); } /* Disable lookaside with the --native-malloc option */ if( nativeMalloc ){ sqlite3_config(SQLITE_CONFIG_LOOKASIDE, 0, 0); } |
︙ | ︙ |
Changes to test/ossfuzz.c.
︙ | ︙ | |||
150 151 152 153 154 155 156 157 158 159 160 161 162 163 | cx.iCutoffTime = cx.iLastCb + 10000; /* Now + 10 seconds */ #ifndef SQLITE_OMIT_PROGRESS_CALLBACK sqlite3_progress_handler(cx.db, 10, progress_handler, (void*)&cx); #endif /* Set a limit on the maximum size of a prepared statement */ sqlite3_limit(cx.db, SQLITE_LIMIT_VDBE_OP, 25000); /* Set a limit on the maximum length of a string or BLOB. Without this ** limit, fuzzers will invoke randomblob(N) for a large N, and the process ** will timeout trying to generate the huge blob */ sqlite3_limit(cx.db, SQLITE_LIMIT_LENGTH, 50000); /* Bit 1 of the selector enables foreign key constraints */ | > > > | 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 | cx.iCutoffTime = cx.iLastCb + 10000; /* Now + 10 seconds */ #ifndef SQLITE_OMIT_PROGRESS_CALLBACK sqlite3_progress_handler(cx.db, 10, progress_handler, (void*)&cx); #endif /* Set a limit on the maximum size of a prepared statement */ sqlite3_limit(cx.db, SQLITE_LIMIT_VDBE_OP, 25000); /* Limit total memory available to SQLite to 20MB */ sqlite3_hard_heap_limit64(20000000); /* Set a limit on the maximum length of a string or BLOB. Without this ** limit, fuzzers will invoke randomblob(N) for a large N, and the process ** will timeout trying to generate the huge blob */ sqlite3_limit(cx.db, SQLITE_LIMIT_LENGTH, 50000); /* Bit 1 of the selector enables foreign key constraints */ |
︙ | ︙ |
Changes to test/tester.tcl.
︙ | ︙ | |||
384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 | # Print a HELP message and exit # proc print_help_and_quit {} { puts {Options: --pause Wait for user input before continuing --soft-heap-limit=N Set the soft-heap-limit to N --maxerror=N Quit after N errors --verbose=(0|1) Control the amount of output. Default '1' --output=FILE set --verbose=2 and output to FILE. Implies -q -q Shorthand for --verbose=0 --help This message } exit 1 } # The following block only runs the first time this file is sourced. It # does not run in slave interpreters (since the ::cmdlinearg array is # populated before the test script is run in slave interpreters). # if {[info exists cmdlinearg]==0} { # Parse any options specified in the $argv array. This script accepts the # following options: # # --pause # --soft-heap-limit=NN # --maxerror=NN # --malloctrace=N # --backtrace=N # --binarylog=N # --soak=N # --file-retries=N # --file-retry-delay=N # --start=[$permutation:]$testfile # --match=$pattern # --verbose=$val # --output=$filename # -q Reduce output # --testdir=$dir Run tests in subdirectory $dir # --help # set cmdlinearg(soft-heap-limit) 0 set cmdlinearg(maxerror) 1000 set cmdlinearg(malloctrace) 0 set cmdlinearg(backtrace) 10 set cmdlinearg(binarylog) 0 set cmdlinearg(soak) 0 set cmdlinearg(file-retries) 0 set cmdlinearg(file-retry-delay) 0 | > > > | 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 | # Print a HELP message and exit # proc print_help_and_quit {} { puts {Options: --pause Wait for user input before continuing --soft-heap-limit=N Set the soft-heap-limit to N --hard-heap-limit=N Set the hard-heap-limit to N --maxerror=N Quit after N errors --verbose=(0|1) Control the amount of output. Default '1' --output=FILE set --verbose=2 and output to FILE. Implies -q -q Shorthand for --verbose=0 --help This message } exit 1 } # The following block only runs the first time this file is sourced. It # does not run in slave interpreters (since the ::cmdlinearg array is # populated before the test script is run in slave interpreters). # if {[info exists cmdlinearg]==0} { # Parse any options specified in the $argv array. This script accepts the # following options: # # --pause # --soft-heap-limit=NN # --hard-heap-limit=NN # --maxerror=NN # --malloctrace=N # --backtrace=N # --binarylog=N # --soak=N # --file-retries=N # --file-retry-delay=N # --start=[$permutation:]$testfile # --match=$pattern # --verbose=$val # --output=$filename # -q Reduce output # --testdir=$dir Run tests in subdirectory $dir # --help # set cmdlinearg(soft-heap-limit) 0 set cmdlinearg(hard-heap-limit) 0 set cmdlinearg(maxerror) 1000 set cmdlinearg(malloctrace) 0 set cmdlinearg(backtrace) 10 set cmdlinearg(binarylog) 0 set cmdlinearg(soak) 0 set cmdlinearg(file-retries) 0 set cmdlinearg(file-retry-delay) 0 |
︙ | ︙ | |||
446 447 448 449 450 451 452 453 454 455 456 457 458 459 | puts -nonewline "Press RETURN to begin..." flush stdout gets stdin } {^-+soft-heap-limit=.+$} { foreach {dummy cmdlinearg(soft-heap-limit)} [split $a =] break } {^-+maxerror=.+$} { foreach {dummy cmdlinearg(maxerror)} [split $a =] break } {^-+malloctrace=.+$} { foreach {dummy cmdlinearg(malloctrace)} [split $a =] break if {$cmdlinearg(malloctrace)} { if {0==$::sqlite_options(memdebug)} { | > > > | 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 | puts -nonewline "Press RETURN to begin..." flush stdout gets stdin } {^-+soft-heap-limit=.+$} { foreach {dummy cmdlinearg(soft-heap-limit)} [split $a =] break } {^-+hard-heap-limit=.+$} { foreach {dummy cmdlinearg(hard-heap-limit)} [split $a =] break } {^-+maxerror=.+$} { foreach {dummy cmdlinearg(maxerror)} [split $a =] break } {^-+malloctrace=.+$} { foreach {dummy cmdlinearg(malloctrace)} [split $a =] break if {$cmdlinearg(malloctrace)} { if {0==$::sqlite_options(memdebug)} { |
︙ | ︙ | |||
582 583 584 585 586 587 588 | } } # Update the soft-heap-limit each time this script is run. In that # way if an individual test file changes the soft-heap-limit, it # will be reset at the start of the next test file. # | | > | 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 | } } # Update the soft-heap-limit each time this script is run. In that # way if an individual test file changes the soft-heap-limit, it # will be reset at the start of the next test file. # sqlite3_soft_heap_limit64 $cmdlinearg(soft-heap-limit) sqlite3_hard_heap_limit64 $cmdlinearg(hard-heap-limit) # Create a test database # proc reset_db {} { catch {db close} forcedelete test.db forcedelete test.db-journal |
︙ | ︙ | |||
1203 1204 1205 1206 1207 1208 1209 | vfs_unlink_test sqlite3 db {} # sqlite3_clear_tsd_memdebug db close sqlite3_reset_auto_extension | | > | 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 | vfs_unlink_test sqlite3 db {} # sqlite3_clear_tsd_memdebug db close sqlite3_reset_auto_extension sqlite3_soft_heap_limit64 0 sqlite3_hard_heap_limit64 0 set nTest [incr_ntest] set nErr [set_test_counter errors] set nKnown 0 if {[file readable known-problems.txt]} { set fd [open known-problems.txt] set content [read $fd] |
︙ | ︙ |