/ Check-in [32ccf3ae]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix a race condition in the sorter.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | threads
Files: files | file ages | folders
SHA1: 32ccf3ae18531682dfd039fa8df6ad9a907ac455
User & Date: dan 2014-05-03 19:33:00
Context
2014-05-03
20:43
Add an extra fault-injection test to sortfault.test. Remove an unreachable branch from vdbesort.c. check-in: a33a366b user: dan tags: threads
19:33
Fix a race condition in the sorter. check-in: 32ccf3ae user: dan tags: threads
14:28
Fix a problem in the sorter causing it to return spurious SQLITE_NOMEM errors when configured to use memsys3 or memsys5. check-in: 3a66c4e1 user: dan tags: threads
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/vdbesort.c.

646
647
648
649
650
651
652

653
654
655
656
657
658
659
....
1840
1841
1842
1843
1844
1845
1846

1847
1848
1849







1850
1851
1852
1853
1854
1855
1856
1857
....
2192
2193
2194
2195
2196
2197
2198




2199


2200
2201
2202
2203
2204
2205
2206
/*
** Advance iterator pIter to the next key in its PMA. Return SQLITE_OK if
** no error occurs, or an SQLite error code if one does.
*/
static int vdbePmaReaderNext(PmaReader *pIter){
  int rc = SQLITE_OK;             /* Return Code */
  u64 nRec = 0;                   /* Size of record in bytes */


  if( pIter->iReadOff>=pIter->iEof ){
    IncrMerger *pIncr = pIter->pIncr;
    int bEof = 1;
    if( pIncr ){
      rc = vdbeIncrSwap(pIncr);
      if( rc==SQLITE_OK && pIncr->bEof==0 ){
................................................................................
static int vdbeIncrInitMerger(
  SortSubtask *pTask, 
  MergeEngine *pMerger, 
  int eMode                       /* One of the INCRINIT_XXX constants */
){
  int rc = SQLITE_OK;             /* Return code */
  int i;                          /* For iterating through PmaReader objects */


  for(i=0; rc==SQLITE_OK && i<pMerger->nTree; i++){
    if( eMode==INCRINIT_ROOT ){







      rc = vdbePmaReaderNext(&pMerger->aIter[i]);
    }else{
      rc = vdbePmaReaderIncrInit(&pMerger->aIter[i], INCRINIT_NORMAL);
    }
  }

  for(i=pMerger->nTree-1; rc==SQLITE_OK && i>0; i--){
    rc = vdbeSorterDoCompare(pTask, pMerger, i);
................................................................................
              assert( pIncr->pTask!=pLast );
            }
          }
          if( pSorter->nTask>1 ){
            for(iTask=0; rc==SQLITE_OK && iTask<pSorter->nTask; iTask++){
              PmaReader *p = &pMain->aIter[iTask];
              assert( p->pIncr==0 || p->pIncr->pTask==&pSorter->aTask[iTask] );




              if( p->pIncr ){ rc = vdbePmaReaderBgIncrInit(p); }


            }
          }
        }
        pMain = 0;
      }
      if( rc==SQLITE_OK ){
        int eMode = (pSorter->nTask>1 ? INCRINIT_ROOT : INCRINIT_NORMAL);







>







 







>

|

>
>
>
>
>
>
>
|







 







>
>
>
>
|
>
>







646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
....
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
....
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
/*
** Advance iterator pIter to the next key in its PMA. Return SQLITE_OK if
** no error occurs, or an SQLite error code if one does.
*/
static int vdbePmaReaderNext(PmaReader *pIter){
  int rc = SQLITE_OK;             /* Return Code */
  u64 nRec = 0;                   /* Size of record in bytes */


  if( pIter->iReadOff>=pIter->iEof ){
    IncrMerger *pIncr = pIter->pIncr;
    int bEof = 1;
    if( pIncr ){
      rc = vdbeIncrSwap(pIncr);
      if( rc==SQLITE_OK && pIncr->bEof==0 ){
................................................................................
static int vdbeIncrInitMerger(
  SortSubtask *pTask, 
  MergeEngine *pMerger, 
  int eMode                       /* One of the INCRINIT_XXX constants */
){
  int rc = SQLITE_OK;             /* Return code */
  int i;                          /* For iterating through PmaReader objects */
  int nTree = pMerger->nTree;

  for(i=0; rc==SQLITE_OK && i<nTree; i++){
    if( eMode==INCRINIT_ROOT ){
      /* Iterators should be normally initialized in order, as if they are
      ** reading from the same temp file this makes for more linear file IO.
      ** However, in the INCRINIT_ROOT case, if iterator aIter[nTask-1] is
      ** in use it will block the vdbePmaReaderNext() call while it uses
      ** the main thread to fill its buffer. So calling PmaReaderNext()
      ** on this iterator before any of the multi-threaded iterators takes
      ** better advantage of multi-processor hardware. */
      rc = vdbePmaReaderNext(&pMerger->aIter[nTree-i-1]);
    }else{
      rc = vdbePmaReaderIncrInit(&pMerger->aIter[i], INCRINIT_NORMAL);
    }
  }

  for(i=pMerger->nTree-1; rc==SQLITE_OK && i>0; i--){
    rc = vdbeSorterDoCompare(pTask, pMerger, i);
................................................................................
              assert( pIncr->pTask!=pLast );
            }
          }
          if( pSorter->nTask>1 ){
            for(iTask=0; rc==SQLITE_OK && iTask<pSorter->nTask; iTask++){
              PmaReader *p = &pMain->aIter[iTask];
              assert( p->pIncr==0 || p->pIncr->pTask==&pSorter->aTask[iTask] );
              if( p->pIncr ){ 
                if( iTask==pSorter->nTask-1 ){
                  rc = vdbePmaReaderIncrInit(p, INCRINIT_TASK);
                }else{
                  rc = vdbePmaReaderBgIncrInit(p);
                }
              }
            }
          }
        }
        pMain = 0;
      }
      if( rc==SQLITE_OK ){
        int eMode = (pSorter->nTask>1 ? INCRINIT_ROOT : INCRINIT_NORMAL);

Changes to test/sort.test.

537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
...
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
          1          0       3     file      true    false
          2          0       3     file      true     true
          3          0       0     file      true    false
          4    1000000       3     file      true    false
          5          0       0   memory     false     true
} {
  db close

  sqlite3_shutdown
  sqlite3_config_worker_threads $nWorker
  if {$coremutex} {
    sqlite3_config multithread
  } else {
    sqlite3_config singlethread
  }
  sqlite3_initialize

  sorter_test_fakeheap $fakeheap

  reset_db
  sqlite3_test_control SQLITE_TESTCTRL_SORTER_MMAP db $mmap_limit
  execsql "PRAGMA temp_store = $tmpstore"
  
  set ten [string repeat X 10300]
................................................................................
  } [list 2 $one 4 $ten]

  sorter_test_fakeheap 0
}

db close
sqlite3_shutdown
#sqlite3_config_worker_threads $sqlite_options(worker_threads)
sqlite3_config_worker_threads 0
set t(0) singlethread
set t(1) multithread
set t(2) serialized
sqlite3_config $t($sqlite_options(threadsafe))
sqlite3_initialize

finish_test








<








<







 







<









537
538
539
540
541
542
543

544
545
546
547
548
549
550
551

552
553
554
555
556
557
558
...
591
592
593
594
595
596
597

598
599
600
601
602
603
604
605
606
          1          0       3     file      true    false
          2          0       3     file      true     true
          3          0       0     file      true    false
          4    1000000       3     file      true    false
          5          0       0   memory     false     true
} {
  db close

  sqlite3_shutdown
  sqlite3_config_worker_threads $nWorker
  if {$coremutex} {
    sqlite3_config multithread
  } else {
    sqlite3_config singlethread
  }
  sqlite3_initialize

  sorter_test_fakeheap $fakeheap

  reset_db
  sqlite3_test_control SQLITE_TESTCTRL_SORTER_MMAP db $mmap_limit
  execsql "PRAGMA temp_store = $tmpstore"
  
  set ten [string repeat X 10300]
................................................................................
  } [list 2 $one 4 $ten]

  sorter_test_fakeheap 0
}

db close
sqlite3_shutdown

sqlite3_config_worker_threads 0
set t(0) singlethread
set t(1) multithread
set t(2) serialized
sqlite3_config $t($sqlite_options(threadsafe))
sqlite3_initialize

finish_test

Changes to test/sortfault.test.

19
20
21
22
23
24
25
26
27
28
29









30
31
32

33
34
35
36
37
38
39
40
41
42
43
44
45
46
47








48
49
set testprefix sortfault


do_execsql_test 1.0 {
  PRAGMA cache_size = 5;
}

foreach {tn mmap_limit} {
  1 0
  2 100000
} {









  do_faultsim_test 1.$tn -prep {
    sqlite3 db test.db
    sqlite3_test_control SQLITE_TESTCTRL_SORTER_MMAP db $::mmap_limit

  } -body {
    execsql { 
      WITH r(x,y) AS (
          SELECT 1, randomblob(1000)
          UNION ALL
          SELECT x+1, randomblob(1000) FROM r
          LIMIT 500
          )
        SELECT count(x), length(y) FROM r GROUP BY (x%5)
    } 
  } -test {
    faultsim_test_result {0 {100 1000 100 1000 100 1000 100 1000 100 1000}}
  }
}









finish_test








|
|
|

>
>
>
>
>
>
>
>
>



>



|

|
|
|
|
|

|



>
>
>
>
>
>
>
>


19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
set testprefix sortfault


do_execsql_test 1.0 {
  PRAGMA cache_size = 5;
}

foreach {tn mmap_limit nWorker tmpstore threadsmode fakeheap} {
          1          0       0     file multithread    false
          2     100000       0     file multithread    false
} {
  catch { db close }
  sqlite3_shutdown
  sqlite3_config_worker_threads $nWorker
  sqlite3_config $threadsmode
  sqlite3_initialize
  sorter_test_fakeheap $fakeheap

  set str [string repeat a 1000]

  do_faultsim_test 1.$tn -prep {
    sqlite3 db test.db
    sqlite3_test_control SQLITE_TESTCTRL_SORTER_MMAP db $::mmap_limit
    execsql { PRAGMA cache_size = 5 }
  } -body {
    execsql { 
      WITH r(x,y) AS (
          SELECT 1, $::str
          UNION ALL
          SELECT x+1, $::str FROM r
          LIMIT 200
      )
      SELECT count(x), length(y) FROM r GROUP BY (x%5)
    }
  } -test {
    faultsim_test_result {0 {40 1000 40 1000 40 1000 40 1000 40 1000}}
  }
}

catch { db close }
sqlite3_shutdown
sqlite3_config_worker_threads 0
set t(0) singlethread
set t(1) multithread
set t(2) serialized
sqlite3_config $t($sqlite_options(threadsafe))
sqlite3_initialize
finish_test