/ Check-in [3a66c4e1]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix a problem in the sorter causing it to return spurious SQLITE_NOMEM errors when configured to use memsys3 or memsys5.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | threads
Files: files | file ages | folders
SHA1: 3a66c4e1bf311d38668dfcdcd77867feff6db7bd
User & Date: dan 2014-05-03 14:28:14
Context
2014-05-03
19:33
Fix a race condition in the sorter. check-in: 32ccf3ae user: dan tags: threads
14:28
Fix a problem in the sorter causing it to return spurious SQLITE_NOMEM errors when configured to use memsys3 or memsys5. check-in: 3a66c4e1 user: dan tags: threads
13:53
Get SQLITE_MUTEX_STATIC_APP1 and _APP2 working for the debugMutex implementation. check-in: f49ba1c9 user: drh tags: threads
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/test1.c.

6332
6333
6334
6335
6336
6337
6338


































6339
6340
6341
6342
6343
6344
6345
....
6565
6566
6567
6568
6569
6570
6571

6572
6573
6574
6575
6576
6577
6578
      sqlite3_free(zErrMsg);
      return TCL_ERROR;
    }
  }
  return TCL_OK;
}




































/*
** Register commands with the TCL interpreter.
*/
int Sqlitetest1_Init(Tcl_Interp *interp){
  extern int sqlite3_search_count;
  extern int sqlite3_found_count;
................................................................................
     { "print_explain_query_plan", test_print_eqp, 0  },
#endif
     { "sqlite3_test_control", test_test_control },
#if SQLITE_OS_UNIX
     { "getrusage", test_getrusage },
#endif
     { "load_static_extension", tclLoadStaticExtensionCmd },

  };
  static int bitmask_size = sizeof(Bitmask)*8;
  int i;
  extern int sqlite3_sync_count, sqlite3_fullsync_count;
  extern int sqlite3_opentemp_count;
  extern int sqlite3_like_count;
  extern int sqlite3_xferopt_count;







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







>







6332
6333
6334
6335
6336
6337
6338
6339
6340
6341
6342
6343
6344
6345
6346
6347
6348
6349
6350
6351
6352
6353
6354
6355
6356
6357
6358
6359
6360
6361
6362
6363
6364
6365
6366
6367
6368
6369
6370
6371
6372
6373
6374
6375
6376
6377
6378
6379
....
6599
6600
6601
6602
6603
6604
6605
6606
6607
6608
6609
6610
6611
6612
6613
      sqlite3_free(zErrMsg);
      return TCL_ERROR;
    }
  }
  return TCL_OK;
}

/*
**     sorter_test_fakeheap BOOL
**
*/
static int sorter_test_fakeheap(
  void * clientData,
  Tcl_Interp *interp,
  int objc,
  Tcl_Obj *CONST objv[]
){
  int bArg;
  if( objc!=2 ){
    Tcl_WrongNumArgs(interp, 1, objv, "BOOL");
    return TCL_ERROR;
  }

  if( Tcl_GetBooleanFromObj(interp, objv[1], &bArg) ){
    return TCL_ERROR;
  }

  if( bArg ){
    if( sqlite3GlobalConfig.pHeap==0 ){
      sqlite3GlobalConfig.pHeap = SQLITE_INT_TO_PTR(-1);
    }
  }else{
    if( sqlite3GlobalConfig.pHeap==SQLITE_INT_TO_PTR(-1) ){
      sqlite3GlobalConfig.pHeap = 0;
    }
  }

  Tcl_ResetResult(interp);
  return TCL_OK;
}


/*
** Register commands with the TCL interpreter.
*/
int Sqlitetest1_Init(Tcl_Interp *interp){
  extern int sqlite3_search_count;
  extern int sqlite3_found_count;
................................................................................
     { "print_explain_query_plan", test_print_eqp, 0  },
#endif
     { "sqlite3_test_control", test_test_control },
#if SQLITE_OS_UNIX
     { "getrusage", test_getrusage },
#endif
     { "load_static_extension", tclLoadStaticExtensionCmd },
     { "sorter_test_fakeheap", sorter_test_fakeheap },
  };
  static int bitmask_size = sizeof(Bitmask)*8;
  int i;
  extern int sqlite3_sync_count, sqlite3_fullsync_count;
  extern int sqlite3_opentemp_count;
  extern int sqlite3_like_count;
  extern int sqlite3_xferopt_count;

Changes to src/test_config.c.

95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113

#if SQLITE_MAX_MMAP_SIZE>0
  Tcl_SetVar2(interp, "sqlite_options", "mmap", "1", TCL_GLOBAL_ONLY);
#else
  Tcl_SetVar2(interp, "sqlite_options", "mmap", "0", TCL_GLOBAL_ONLY);
#endif

#if SQLITE_MAX_WORKER_THREADS>0
  Tcl_SetVar2(interp, "sqlite_options", "worker_threads", "1", TCL_GLOBAL_ONLY);
#else
  Tcl_SetVar2(interp, "sqlite_options", "worker_threads", "0", TCL_GLOBAL_ONLY);
#endif

#if 1 /* def SQLITE_MEMDEBUG */
  Tcl_SetVar2(interp, "sqlite_options", "memdebug", "1", TCL_GLOBAL_ONLY);
#else
  Tcl_SetVar2(interp, "sqlite_options", "memdebug", "0", TCL_GLOBAL_ONLY);
#endif








|
|
|
<
<







95
96
97
98
99
100
101
102
103
104


105
106
107
108
109
110
111

#if SQLITE_MAX_MMAP_SIZE>0
  Tcl_SetVar2(interp, "sqlite_options", "mmap", "1", TCL_GLOBAL_ONLY);
#else
  Tcl_SetVar2(interp, "sqlite_options", "mmap", "0", TCL_GLOBAL_ONLY);
#endif

  Tcl_SetVar2(interp, "sqlite_options", "worker_threads", 
      STRINGVALUE(SQLITE_MAX_WORKER_THREADS), TCL_GLOBAL_ONLY
  );



#if 1 /* def SQLITE_MEMDEBUG */
  Tcl_SetVar2(interp, "sqlite_options", "memdebug", "1", TCL_GLOBAL_ONLY);
#else
  Tcl_SetVar2(interp, "sqlite_options", "memdebug", "0", TCL_GLOBAL_ONLY);
#endif

Changes to src/vdbesort.c.

1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
      pSorter->iPrev = (pTask - pSorter->aTask);
      pTask->list = pSorter->list;
      pSorter->list.pList = 0;
      pSorter->list.szPMA = 0;
      if( aMem ){
        pSorter->list.aMemory = aMem;
        pSorter->nMemory = sqlite3MallocSize(aMem);
      }else{
        pSorter->list.aMemory = sqlite3Malloc(pSorter->nMemory);
        if( !pSorter->list.aMemory ) return SQLITE_NOMEM;
      }

      rc = vdbeSorterCreateThread(pTask, vdbeSorterFlushThread, pCtx);
    }
  }







|







1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
      pSorter->iPrev = (pTask - pSorter->aTask);
      pTask->list = pSorter->list;
      pSorter->list.pList = 0;
      pSorter->list.szPMA = 0;
      if( aMem ){
        pSorter->list.aMemory = aMem;
        pSorter->nMemory = sqlite3MallocSize(aMem);
      }else if( pSorter->list.aMemory ){
        pSorter->list.aMemory = sqlite3Malloc(pSorter->nMemory);
        if( !pSorter->list.aMemory ) return SQLITE_NOMEM;
      }

      rc = vdbeSorterCreateThread(pTask, vdbeSorterFlushThread, pCtx);
    }
  }

Changes to test/permutations.test.

327
328
329
330
331
332
333






334
335
336
337
338
339
340
test_suite "coverage-analyze" -description {
  Coverage tests for file analyze.c.
} -files {
  analyze3.test analyze4.test analyze5.test analyze6.test
  analyze7.test analyze8.test analyze9.test analyzeA.test
  analyze.test analyzeB.test mallocA.test
} 








lappend ::testsuitelist xxx
#-------------------------------------------------------------------------
# Define the permutation test suites:
#








>
>
>
>
>
>







327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
test_suite "coverage-analyze" -description {
  Coverage tests for file analyze.c.
} -files {
  analyze3.test analyze4.test analyze5.test analyze6.test
  analyze7.test analyze8.test analyze9.test analyzeA.test
  analyze.test analyzeB.test mallocA.test
} 

test_suite "coverage-sorter" -description {
  Coverage tests for file vdbesort.c.
} -files {
  sort.test sortfault.test
} 


lappend ::testsuitelist xxx
#-------------------------------------------------------------------------
# Define the permutation test suites:
#

Changes to test/sort.test.

4
5
6
7
8
9
10

11
12
13
14
15
16
17
18
19
20
21
...
481
482
483
484
485
486
487


488





















































































































489

# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************

# This file implements regression tests for SQLite library.  The
# focus of this file is testing the CREATE TABLE statement.
#
# $Id: sort.test,v 1.25 2005/11/14 22:29:06 drh Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# Create a bunch of data to sort against
#
do_test sort-1.0 {
................................................................................
  SELECT a, b FROM t10 ORDER BY a;
} [db eval {SELECT a, b FROM t10 ORDER BY a, b}]
do_execsql_test sort-13.3 {
  PRAGMA cache_size = 5;
  SELECT a, b FROM t10 ORDER BY a;
} [db eval {SELECT a, b FROM t10 ORDER BY a, b}]

























































































































finish_test








>

|

<







 







>
>
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>

>
4
5
6
7
8
9
10
11
12
13
14

15
16
17
18
19
20
21
...
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#
# This file implements regression tests for SQLite library.  The
# focus of this file is testing the sorter (code in vdbesort.c).
#


set testdir [file dirname $argv0]
source $testdir/tester.tcl

# Create a bunch of data to sort against
#
do_test sort-1.0 {
................................................................................
  SELECT a, b FROM t10 ORDER BY a;
} [db eval {SELECT a, b FROM t10 ORDER BY a, b}]
do_execsql_test sort-13.3 {
  PRAGMA cache_size = 5;
  SELECT a, b FROM t10 ORDER BY a;
} [db eval {SELECT a, b FROM t10 ORDER BY a, b}]

#-------------------------------------------------------------------------
# Sort some large ( > 4KiB) records.
#
proc cksum {x} {
  set i1 1
  set i2 2
  binary scan $x c* L
  foreach {a b} $L {
    set i1 [expr (($i2<<3) + $a) & 0x7FFFFFFF]
    set i2 [expr (($i1<<3) + $b) & 0x7FFFFFFF]
  }
  list $i1 $i2
}
db func cksum cksum

do_execsql_test sort-14.0 {
  PRAGMA cache_size = 5;
  CREATE TABLE t11(a, b);
  INSERT INTO t11 VALUES(randomblob(5000), NULL);
  INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --2
  INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --3
  INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --4
  INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --5
  INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --6
  INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --7
  INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --8
  INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --9
  UPDATE t11 SET b = cksum(a);
}

foreach {tn mmap_limit} {
  1 0
  2 1000000
} {
  do_test sort-14.$tn {
    sqlite3_test_control SQLITE_TESTCTRL_SORTER_MMAP db $mmap_limit
    set prev ""
    db eval { SELECT * FROM t11 ORDER BY b } {
      if {$b != [cksum $a]} {error "checksum failed"}
      if {[string compare $b $prev] < 0} {error "sort failed"}
      set prev $b
    }
    set {} {}
  } {}
}

#-------------------------------------------------------------------------
#
foreach {tn mmap_limit nWorker tmpstore coremutex fakeheap} {
          1          0       3     file      true    false
          2          0       3     file      true     true
          3          0       0     file      true    false
          4    1000000       3     file      true    false
          5          0       0   memory     false     true
} {
  db close

  sqlite3_shutdown
  sqlite3_config_worker_threads $nWorker
  if {$coremutex} {
    sqlite3_config multithread
  } else {
    sqlite3_config singlethread
  }
  sqlite3_initialize

  sorter_test_fakeheap $fakeheap

  reset_db
  sqlite3_test_control SQLITE_TESTCTRL_SORTER_MMAP db $mmap_limit
  execsql "PRAGMA temp_store = $tmpstore"
  
  set ten [string repeat X 10300]
  set one [string repeat y   200]

  do_execsql_test 15.$tn.1 {
    PRAGMA cache_size = 5;
    WITH rr AS (
      SELECT 4, $ten UNION ALL
      SELECT 2, $one UNION ALL
      SELECT 1, $ten UNION ALL
      SELECT 3, $one
    )
    SELECT * FROM rr ORDER BY 1;
  } [list 1 $ten 2 $one 3 $one 4 $ten]

  do_execsql_test 15.$tn.2 {
    CREATE TABLE t1(a);
    INSERT INTO t1 VALUES(4);
    INSERT INTO t1 VALUES(5);
    INSERT INTO t1 VALUES(3);
    INSERT INTO t1 VALUES(2);
    INSERT INTO t1 VALUES(6);
    INSERT INTO t1 VALUES(1);
    CREATE INDEX i1 ON t1(a);
    SELECT * FROM t1 ORDER BY a;
  } {1 2 3 4 5 6}

  do_execsql_test 15.$tn.3 {
    PRAGMA cache_size = 5;
    WITH rr AS (
      SELECT 4, $ten UNION ALL
      SELECT 2, $one
    )
    SELECT * FROM rr ORDER BY 1;
  } [list 2 $one 4 $ten]

  sorter_test_fakeheap 0
}

db close
sqlite3_shutdown
#sqlite3_config_worker_threads $sqlite_options(worker_threads)
sqlite3_config_worker_threads 0
set t(0) singlethread
set t(1) multithread
set t(2) serialized
sqlite3_config $t($sqlite_options(threadsafe))
sqlite3_initialize

finish_test

Changes to test/sort2.test.

5
6
7
8
9
10
11



12
13
14
15
16
17
18
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file implements regression tests for SQLite library. 



#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix sort2

foreach {tn script} {







>
>
>







5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file implements regression tests for SQLite library. 
#
# Specifically, the tests in this file attempt to verify that 
# multi-threaded sorting works.
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix sort2

foreach {tn script} {

Changes to test/sort3.test.

5
6
7
8
9
10
11




12
13
14
15
16
17
18
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file implements regression tests for SQLite library. 




#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix sort3

# Sort roughly 20MB of data. Once with a mmap limit of 5MB and once without.







>
>
>
>







5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file implements regression tests for SQLite library. 
#
# The tests in this file verify that sorting works when the library is
# configured to use mmap(), but the temporary files generated by the
# sorter are too large to be completely mapped.
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix sort3

# Sort roughly 20MB of data. Once with a mmap limit of 5MB and once without.

Changes to test/sortfault.test.

6
7
8
9
10
11
12



13
14
15
16
17
18
19
20
21
22




23
24

25
26
27
28
29
30
31
32
33
34
35
36
37
38

39
40
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file implements regression tests for SQLite library. 
#




set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix sortfault


do_execsql_test 1.0 {
  PRAGMA cache_size = 5;
}





do_faultsim_test 1 -prep {
  sqlite3 db test.db

} -body {
  execsql { 
    WITH r(x,y) AS (
      SELECT 1, randomblob(1000)
      UNION ALL
      SELECT x+1, randomblob(1000) FROM r
      LIMIT 500
    )
    SELECT count(x), length(y) FROM r GROUP BY (x%5)
  } 
} -test {
  faultsim_test_result {0 {100 1000 100 1000 100 1000 100 1000 100 1000}}
}


finish_test








>
>
>










>
>
>
>
|
|
>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
>


6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file implements regression tests for SQLite library. 
#
# Specifically, it tests the effects of fault injection on the sorter
# module (code in vdbesort.c).
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix sortfault


do_execsql_test 1.0 {
  PRAGMA cache_size = 5;
}

foreach {tn mmap_limit} {
  1 0
  2 100000
} {
  do_faultsim_test 1.$tn -prep {
    sqlite3 db test.db
    sqlite3_test_control SQLITE_TESTCTRL_SORTER_MMAP db $::mmap_limit
  } -body {
    execsql { 
      WITH r(x,y) AS (
          SELECT 1, randomblob(1000)
          UNION ALL
          SELECT x+1, randomblob(1000) FROM r
          LIMIT 500
          )
        SELECT count(x), length(y) FROM r GROUP BY (x%5)
    } 
  } -test {
    faultsim_test_result {0 {100 1000 100 1000 100 1000 100 1000 100 1000}}
  }
}

finish_test