Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Add tests so that the "coverage-sorter" test permutation covers all branches in vdbesort.c. Fix a few minor problems in the same file. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | threads |
Files: | files | file ages | folders |
SHA1: |
bde28b702dabd02269e333535cc41481 |
User & Date: | dan 2014-05-05 09:08:54.007 |
Context
2014-05-05
| ||
15:58 | Fix a race condition in the sorter code. (check-in: 2d2edfe58d user: dan tags: threads) | |
09:08 | Add tests so that the "coverage-sorter" test permutation covers all branches in vdbesort.c. Fix a few minor problems in the same file. (check-in: bde28b702d user: dan tags: threads) | |
2014-05-03
| ||
20:43 | Add an extra fault-injection test to sortfault.test. Remove an unreachable branch from vdbesort.c. (check-in: a33a366ba8 user: dan tags: threads) | |
Changes
Changes to main.mk.
︙ | ︙ | |||
308 309 310 311 312 313 314 315 316 317 318 319 320 321 | $(TOP)/src/pragma.c \ $(TOP)/src/prepare.c \ $(TOP)/src/printf.c \ $(TOP)/src/random.c \ $(TOP)/src/pcache.c \ $(TOP)/src/pcache1.c \ $(TOP)/src/select.c \ $(TOP)/src/tokenize.c \ $(TOP)/src/utf.c \ $(TOP)/src/util.c \ $(TOP)/src/vdbeapi.c \ $(TOP)/src/vdbeaux.c \ $(TOP)/src/vdbe.c \ $(TOP)/src/vdbemem.c \ | > | 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 | $(TOP)/src/pragma.c \ $(TOP)/src/prepare.c \ $(TOP)/src/printf.c \ $(TOP)/src/random.c \ $(TOP)/src/pcache.c \ $(TOP)/src/pcache1.c \ $(TOP)/src/select.c \ $(TOP)/src/threads.c \ $(TOP)/src/tokenize.c \ $(TOP)/src/utf.c \ $(TOP)/src/util.c \ $(TOP)/src/vdbeapi.c \ $(TOP)/src/vdbeaux.c \ $(TOP)/src/vdbe.c \ $(TOP)/src/vdbemem.c \ |
︙ | ︙ |
Changes to src/threads.c.
︙ | ︙ | |||
204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 | } *ppThread = p; return SQLITE_OK; } /* Get the results of the thread */ int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ assert( ppOut!=0 ); if( p==0 ) return SQLITE_NOMEM; if( p->xTask ){ *ppOut = p->xTask(p->pIn); }else{ *ppOut = p->pResult; } sqlite3_free(p); return SQLITE_OK; } #endif /* !defined(SQLITE_THREADS_IMPLEMENTED) */ /****************************** End Single-Threaded *************************/ #endif /* SQLITE_MAX_WORKER_THREADS>0 */ | > > > > > > > > > > | 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 | } *ppThread = p; return SQLITE_OK; } /* Get the results of the thread */ int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ assert( ppOut!=0 ); if( p==0 ) return SQLITE_NOMEM; if( p->xTask ){ *ppOut = p->xTask(p->pIn); }else{ *ppOut = p->pResult; } sqlite3_free(p); #if defined(SQLITE_TEST) { void *pTstAlloc = sqlite3Malloc(10); if (!pTstAlloc) return SQLITE_NOMEM; sqlite3_free(pTstAlloc); } #endif return SQLITE_OK; } #endif /* !defined(SQLITE_THREADS_IMPLEMENTED) */ /****************************** End Single-Threaded *************************/ #endif /* SQLITE_MAX_WORKER_THREADS>0 */ |
Changes to src/vdbesort.c.
︙ | ︙ | |||
1247 1248 1249 1250 1251 1252 1253 | p = 0; for(i=0; i<64; i++){ vdbeSorterMerge(pTask, p, aSlot[i], &p); } pList->pList = p; sqlite3_free(aSlot); | | | < < > | | 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 | p = 0; for(i=0; i<64; i++){ vdbeSorterMerge(pTask, p, aSlot[i], &p); } pList->pList = p; sqlite3_free(aSlot); assert( pTask->pUnpacked->errCode==SQLITE_OK || pTask->pUnpacked->errCode==SQLITE_NOMEM ); return pTask->pUnpacked->errCode; } /* ** Initialize a PMA-writer object. */ static void vdbePmaWriterInit( sqlite3_file *pFile, /* File to write to */ |
︙ | ︙ | |||
1481 1482 1483 1484 1485 1486 1487 | pMerger->aTree[i] = (int)(pIter2 - pMerger->aIter); pIter1 = &pMerger->aIter[ pMerger->aTree[i ^ 0x0001] ]; } } *pbEof = (pMerger->aIter[pMerger->aTree[1]].pFile==0); } | | | 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 | pMerger->aTree[i] = (int)(pIter2 - pMerger->aIter); pIter1 = &pMerger->aIter[ pMerger->aTree[i ^ 0x0001] ]; } } *pbEof = (pMerger->aIter[pMerger->aTree[1]].pFile==0); } return (rc==SQLITE_OK ? pTask->pUnpacked->errCode : rc); } #if SQLITE_MAX_WORKER_THREADS>0 /* ** The main routine for background threads that write level-0 PMAs. */ static void *vdbeSorterFlushThread(void *pCtx){ |
︙ | ︙ | |||
1530 1531 1532 1533 1534 1535 1536 | ** sub-task uses the main thread. */ for(i=0; i<nWorker; i++){ int iTest = (pSorter->iPrev + i + 1) % nWorker; pTask = &pSorter->aTask[iTest]; if( pTask->bDone ){ rc = vdbeSorterJoinThread(pTask); } | | | 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 | ** sub-task uses the main thread. */ for(i=0; i<nWorker; i++){ int iTest = (pSorter->iPrev + i + 1) % nWorker; pTask = &pSorter->aTask[iTest]; if( pTask->bDone ){ rc = vdbeSorterJoinThread(pTask); } if( rc!=SQLITE_OK || pTask->pThread==0 ) break; } if( rc==SQLITE_OK ){ if( i==nWorker ){ /* Use the foreground thread for this operation */ rc = vdbeSorterListToPMA(&pSorter->aTask[nWorker], &pSorter->list); }else{ |
︙ | ︙ | |||
1860 1861 1862 1863 1864 1865 1866 | } } for(i=pMerger->nTree-1; rc==SQLITE_OK && i>0; i--){ rc = vdbeSorterDoCompare(pTask, pMerger, i); } | | | 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 | } } for(i=pMerger->nTree-1; rc==SQLITE_OK && i>0; i--){ rc = vdbeSorterDoCompare(pTask, pMerger, i); } return (rc==SQLITE_OK ? pTask->pUnpacked->errCode : rc); } /* ** If the PmaReader passed as the first argument is not an incremental-reader ** (if pIter->pIncr==0), then this function is a no-op. Otherwise, it serves ** to open and/or initialize the temp file related fields of the IncrMerge ** object at (pIter->pIncr). |
︙ | ︙ | |||
2070 2071 2072 2073 2074 2075 2076 | MergeEngine *pNew = vdbeMergeEngineNew(SORTER_MAX_MERGE_COUNT); if( pNew==0 ){ rc = SQLITE_NOMEM; }else{ rc = vdbeIncrNew(pTask, pNew, &pIter->pIncr); } } | | | | > | 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 | MergeEngine *pNew = vdbeMergeEngineNew(SORTER_MAX_MERGE_COUNT); if( pNew==0 ){ rc = SQLITE_NOMEM; }else{ rc = vdbeIncrNew(pTask, pNew, &pIter->pIncr); } } if( rc==SQLITE_OK ){ p = pIter->pIncr->pMerger; nDiv = nDiv / SORTER_MAX_MERGE_COUNT; } } if( rc==SQLITE_OK ){ p->aIter[iSeq % SORTER_MAX_MERGE_COUNT].pIncr = pIncr; }else{ vdbeIncrFree(pIncr); } |
︙ | ︙ | |||
2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 | #if SQLITE_MAX_WORKER_THREADS sqlite3 *db = pTask0->pSorter->db; #endif rc = vdbeSorterMergeTreeBuild(pSorter, &pMain); if( rc==SQLITE_OK ){ #if SQLITE_MAX_WORKER_THREADS if( pSorter->bUseThreads ){ int iTask; PmaReader *pIter; SortSubtask *pLast = &pSorter->aTask[pSorter->nTask-1]; rc = vdbeSortAllocUnpacked(pLast); if( rc==SQLITE_OK ){ pIter = (PmaReader*)sqlite3DbMallocZero(db, sizeof(PmaReader)); | > | 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 | #if SQLITE_MAX_WORKER_THREADS sqlite3 *db = pTask0->pSorter->db; #endif rc = vdbeSorterMergeTreeBuild(pSorter, &pMain); if( rc==SQLITE_OK ){ #if SQLITE_MAX_WORKER_THREADS assert( pSorter->bUseThreads==0 || pSorter->nTask>1 ); if( pSorter->bUseThreads ){ int iTask; PmaReader *pIter; SortSubtask *pLast = &pSorter->aTask[pSorter->nTask-1]; rc = vdbeSortAllocUnpacked(pLast); if( rc==SQLITE_OK ){ pIter = (PmaReader*)sqlite3DbMallocZero(db, sizeof(PmaReader)); |
︙ | ︙ | |||
2195 2196 2197 2198 2199 2200 2201 | for(iTask=0; iTask<(pSorter->nTask-1); iTask++){ IncrMerger *pIncr; if( (pIncr = pMain->aIter[iTask].pIncr) ){ vdbeIncrSetThreads(pIncr); assert( pIncr->pTask!=pLast ); } } | < | | | | | | | | < < | | 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 | for(iTask=0; iTask<(pSorter->nTask-1); iTask++){ IncrMerger *pIncr; if( (pIncr = pMain->aIter[iTask].pIncr) ){ vdbeIncrSetThreads(pIncr); assert( pIncr->pTask!=pLast ); } } for(iTask=0; rc==SQLITE_OK && iTask<pSorter->nTask; iTask++){ PmaReader *p = &pMain->aIter[iTask]; assert( p->pIncr==0 || p->pIncr->pTask==&pSorter->aTask[iTask] ); if( p->pIncr ){ if( iTask==pSorter->nTask-1 ){ rc = vdbePmaReaderIncrInit(p, INCRINIT_TASK); }else{ rc = vdbePmaReaderBgIncrInit(p); } } } } pMain = 0; } if( rc==SQLITE_OK ){ rc = vdbePmaReaderIncrInit(pIter, INCRINIT_ROOT); } }else #endif { rc = vdbeIncrInitMerger(pTask0, pMain, INCRINIT_NORMAL); pSorter->pMerger = pMain; pMain = 0; |
︙ | ︙ | |||
2255 2256 2257 2258 2259 2260 2261 | rc = vdbeSorterSort(&pSorter->aTask[0], &pSorter->list); }else{ *pbEof = 1; } return rc; } | | > > > | | < | 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 | rc = vdbeSorterSort(&pSorter->aTask[0], &pSorter->list); }else{ *pbEof = 1; } return rc; } /* Write the current in-memory list to a PMA. When the VdbeSorterWrite() ** function flushes the contents of memory to disk, it immediately always ** creates a new list consisting of a single key immediately afterwards. ** So the list is never empty at this point. */ assert( pSorter->list.pList ); rc = vdbeSorterFlushPMA(pSorter); /* Join all threads */ rc = vdbeSorterJoinAll(pSorter, rc); vdbeSorterRewindDebug("rewind"); /* Assuming no errors have occurred, set up a merger structure to |
︙ | ︙ |
Changes to test/sort.test.
︙ | ︙ | |||
529 530 531 532 533 534 535 | } set {} {} } {} } #------------------------------------------------------------------------- # | | | | | | | > > > > > > > > > < | 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 | } set {} {} } {} } #------------------------------------------------------------------------- # foreach {tn mmap_limit nWorker tmpstore coremutex fakeheap softheaplimit} { 1 0 3 file true false 0 2 0 3 file true true 0 3 0 0 file true false 0 4 1000000 3 file true false 0 5 0 0 memory false true 0 6 0 0 file false true 1000000 7 0 0 file false true 10000 } { db close sqlite3_shutdown sqlite3_config_worker_threads $nWorker if {$coremutex} { sqlite3_config multithread } else { sqlite3_config singlethread } sqlite3_initialize sorter_test_fakeheap $fakeheap sqlite3_soft_heap_limit $softheaplimit reset_db sqlite3_test_control SQLITE_TESTCTRL_SORTER_MMAP db $mmap_limit execsql "PRAGMA temp_store = $tmpstore" set ten [string repeat X 10300] set one [string repeat y 200] if {$softheaplimit} { execsql { PRAGMA cache_size = 20 }; } else { execsql { PRAGMA cache_size = 5 }; } do_execsql_test 15.$tn.1 { WITH rr AS ( SELECT 4, $ten UNION ALL SELECT 2, $one UNION ALL SELECT 1, $ten UNION ALL SELECT 3, $one ) SELECT * FROM rr ORDER BY 1; |
︙ | ︙ | |||
578 579 580 581 582 583 584 | INSERT INTO t1 VALUES(6); INSERT INTO t1 VALUES(1); CREATE INDEX i1 ON t1(a); SELECT * FROM t1 ORDER BY a; } {1 2 3 4 5 6} do_execsql_test 15.$tn.3 { | < > > > > > > > > > > > > > > > > > > > > > > > > > > > | 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 | INSERT INTO t1 VALUES(6); INSERT INTO t1 VALUES(1); CREATE INDEX i1 ON t1(a); SELECT * FROM t1 ORDER BY a; } {1 2 3 4 5 6} do_execsql_test 15.$tn.3 { WITH rr AS ( SELECT 4, $ten UNION ALL SELECT 2, $one ) SELECT * FROM rr ORDER BY 1; } [list 2 $one 4 $ten] sorter_test_fakeheap 0 } db close sqlite3_shutdown sqlite3_config_worker_threads 0 set t(0) singlethread set t(1) multithread set t(2) serialized sqlite3_config $t($sqlite_options(threadsafe)) sqlite3_initialize sqlite3_soft_heap_limit 0 reset_db do_catchsql_test 16.1 { CREATE TABLE t1(a, b, c); INSERT INTO t1 VALUES(1, 2, 3); INSERT INTO t1 VALUES(1, NULL, 3); INSERT INTO t1 VALUES(NULL, 2, 3); INSERT INTO t1 VALUES(1, 2, NULL); INSERT INTO t1 VALUES(4, 5, 6); CREATE UNIQUE INDEX i1 ON t1(b, a, c); } {0 {}} reset_db do_catchsql_test 16.2 { CREATE TABLE t1(a, b, c); INSERT INTO t1 VALUES(1, 2, 3); INSERT INTO t1 VALUES(1, NULL, 3); INSERT INTO t1 VALUES(1, 2, 3); INSERT INTO t1 VALUES(1, 2, NULL); INSERT INTO t1 VALUES(4, 5, 6); CREATE UNIQUE INDEX i1 ON t1(b, a, c); } {1 {UNIQUE constraint failed: t1.b, t1.a, t1.c}} reset_db do_execsql_test 17.1 { SELECT * FROM sqlite_master ORDER BY sql; } {} finish_test |
Changes to test/sortfault.test.
︙ | ︙ | |||
14 15 16 17 18 19 20 | # module (code in vdbesort.c). # set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix sortfault | < | | | | > > > > > > > > > < | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 | # module (code in vdbesort.c). # set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix sortfault do_execsql_test 1.0 { PRAGMA cache_size = 5; } foreach {tn mmap_limit nWorker tmpstore threadsmode fakeheap lookaside} { 1 0 0 file multithread false false 2 100000 0 file multithread false false 3 100000 1 file multithread false false 4 2000000 0 file singlethread false true } { if {$sqlite_options(threadsafe)} { set threadsmode singlethread } catch { db close } sqlite3_shutdown sqlite3_config_worker_threads $nWorker sqlite3_config $threadsmode if { $lookaside } { sqlite3_config_lookaside 100 500 } else { sqlite3_config_lookaside 0 0 } sqlite3_initialize sorter_test_fakeheap $fakeheap set str [string repeat a 1000] puts $threadsmode do_faultsim_test 1.$tn -prep { sqlite3 db test.db sqlite3_test_control SQLITE_TESTCTRL_SORTER_MMAP db $::mmap_limit execsql { PRAGMA cache_size = 5 } } -body { execsql { WITH r(x,y) AS ( SELECT 1, $::str UNION ALL SELECT x+1, $::str FROM r LIMIT 200 ) SELECT count(x), length(y) FROM r GROUP BY (x%5) } } -test { faultsim_test_result {0 {40 1000 40 1000 40 1000 40 1000 40 1000}} } do_faultsim_test 2.$tn -faults oom* -prep { sqlite3 db test.db sqlite3_test_control SQLITE_TESTCTRL_SORTER_MMAP db $::mmap_limit add_test_utf16bin_collate db execsql { PRAGMA cache_size = 5 } } -body { execsql { WITH r(x,y) AS ( SELECT 100, $::str UNION ALL SELECT x-1, $::str FROM r LIMIT 100 ) SELECT count(x), length(y) FROM r GROUP BY y COLLATE utf16bin, (x%5) } } -test { faultsim_test_result {0 {20 1000 20 1000 20 1000 20 1000 20 1000}} } if {$mmap_limit > 1000000} { set str2 [string repeat $str 10] sqlite3_memdebug_vfs_oom_test 0 sqlite3 db test.db sqlite3_test_control SQLITE_TESTCTRL_SORTER_MMAP db $::mmap_limit execsql { PRAGMA cache_size = 5 } do_faultsim_test 3.$tn -faults oom-trans* -body { execsql { WITH r(x,y) AS ( SELECT 300, $::str2 UNION ALL SELECT x-1, $::str2 FROM r LIMIT 300 ) SELECT count(x), length(y) FROM r GROUP BY y, (x%5) } } -test { faultsim_test_result {0 {60 10000 60 10000 60 10000 60 10000 60 10000}} } sqlite3_memdebug_vfs_oom_test 1 } } catch { db close } sqlite3_shutdown sqlite3_config_worker_threads 0 set t(0) singlethread set t(1) multithread set t(2) serialized sqlite3_config $t($sqlite_options(threadsafe)) sqlite3_config_lookaside 100 500 sqlite3_initialize #------------------------------------------------------------------------- # reset_db do_execsql_test 4.0 { CREATE TABLE t1(a, b, c); INSERT INTO t1 VALUES(1, 2, 3); } do_test 4.1 { for {set i 0} {$i < 256} {incr i} { execsql { INSERT INTO t1 SELECT ((a<<3) + b) & 2147483647, ((b<<3) + c) & 2147483647, ((c<<3) + a) & 2147483647 FROM t1 ORDER BY rowid DESC LIMIT 1; } } } {} faultsim_save_and_close do_faultsim_test 4.2 -faults oom* -prep { faultsim_restore_and_reopen } -body { execsql { CREATE UNIQUE INDEX i1 ON t1(a,b,c) } } -test { faultsim_test_result {0 {}} } #------------------------------------------------------------------------- # reset_db set a [string repeat a 500] set b [string repeat b 500] set c [string repeat c 500] do_execsql_test 5.0 { CREATE TABLE t1(a, b, c); INSERT INTO t1 VALUES($a, $b, $c); INSERT INTO t1 VALUES($c, $b, $a); } do_faultsim_test 5.1 -faults oom* -body { execsql { SELECT * FROM t1 ORDER BY a } } -test { faultsim_test_result [list 0 [list $::a $::b $::c $::c $::b $::a]] } finish_test |