/ Check-in [0e6defa6]
Login
Home Timeline Branches Tags Tickets Wiki More...
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add fault-injection and other tests (and fixes) to improve coverage of vdbesort.c.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | experimental
Files: files | file ages | folders
SHA1: 0e6defa6aa540b413ea3f4bb6dcd86364d547067
User & Date: dan 2011-08-08 16:44:25
Context
2011-08-08
19:26
Remove redundant parameter from vdbeSorterInitMerge() in vdbesort.c. check-in: eec8c0df user: dan tags: experimental
16:44
Add fault-injection and other tests (and fixes) to improve coverage of vdbesort.c. check-in: 0e6defa6 user: dan tags: experimental
2011-08-06
15:09
Fix a problem with building large indexes introduced by the previous commit. check-in: 038ec9ea user: dan tags: experimental
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/vdbesort.c. 

392
393
394
395
396
397
398
399
400


401
402
403
404
405
406
407
408
409
410
411
412

413
414
415
416
417
418
419
420
421
422
...
644
645
646
647
648
649
650







651
652
653
654
655
656
657
658
659
660

      /* Write the size of the record in bytes to the output file */
      (void)sqlite3BtreeKeySize(pCsr->pCursor, &nKey);
      rc = vdbeSorterWriteVarint(pSorter->pTemp1, nKey, &iWriteOff);

      /* Make sure the aMalloc[] buffer is large enough for the record */
      if( rc==SQLITE_OK && nKey>nMalloc ){
        aMalloc = sqlite3DbReallocOrFree(db, aMalloc, nKey);
        if( !aMalloc ){
          rc = SQLITE_NOMEM;


        }
      }

      /* Write the record itself to the output file */
      if( rc==SQLITE_OK ){
        rc = sqlite3BtreeKey(pCsr->pCursor, 0, nKey, aMalloc);
        if( rc==SQLITE_OK ){
          rc = sqlite3OsWrite(pSorter->pTemp1, aMalloc, nKey, iWriteOff);
          iWriteOff += nKey;
        }
      }


    }

    assert( pSorter->nBtree==(
          iWriteOff-pSorter->iWriteOff-sqlite3VarintLen(pSorter->nBtree)
    ));
    pSorter->iWriteOff = iWriteOff;
    sqlite3DbFree(db, aMalloc);
  }

  pSorter->nBtree = 0;
................................................................................
** Copy the current sorter key into the memory cell pOut.
*/
int sqlite3VdbeSorterRowkey(sqlite3 *db, VdbeCursor *pCsr, Mem *pOut){
  VdbeSorter *pSorter = pCsr->pSorter;
  VdbeSorterIter *pIter;

  pIter = &pSorter->aIter[ pSorter->aTree[1] ];







  if( sqlite3VdbeMemGrow(pOut, pIter->nKey, 0) ){
    return SQLITE_NOMEM;
  }
  pOut->n = pIter->nKey;
  MemSetTypeFlag(pOut, MEM_Blob);
  memcpy(pOut->z, pIter->aKey, pIter->nKey);

  return SQLITE_OK;
}









|
|
>
>












>


|







 







>
>
>
>
>
>
>











392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
...
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670

      /* Write the size of the record in bytes to the output file */
      (void)sqlite3BtreeKeySize(pCsr->pCursor, &nKey);
      rc = vdbeSorterWriteVarint(pSorter->pTemp1, nKey, &iWriteOff);

      /* Make sure the aMalloc[] buffer is large enough for the record */
      if( rc==SQLITE_OK && nKey>nMalloc ){
        aMalloc = sqlite3DbReallocOrFree(db, aMalloc, nKey);
        if( !aMalloc ){ 
          rc = SQLITE_NOMEM; 
        }else{
          nMalloc = nKey;
        }
      }

      /* Write the record itself to the output file */
      if( rc==SQLITE_OK ){
        rc = sqlite3BtreeKey(pCsr->pCursor, 0, nKey, aMalloc);
        if( rc==SQLITE_OK ){
          rc = sqlite3OsWrite(pSorter->pTemp1, aMalloc, nKey, iWriteOff);
          iWriteOff += nKey;
        }
      }

      if( rc!=SQLITE_OK ) break;
    }

    assert( rc!=SQLITE_OK || pSorter->nBtree==(
          iWriteOff-pSorter->iWriteOff-sqlite3VarintLen(pSorter->nBtree)
    ));
    pSorter->iWriteOff = iWriteOff;
    sqlite3DbFree(db, aMalloc);
  }

  pSorter->nBtree = 0;
................................................................................
** Copy the current sorter key into the memory cell pOut.
*/
int sqlite3VdbeSorterRowkey(sqlite3 *db, VdbeCursor *pCsr, Mem *pOut){
  VdbeSorter *pSorter = pCsr->pSorter;
  VdbeSorterIter *pIter;

  pIter = &pSorter->aIter[ pSorter->aTree[1] ];

  /* Coverage testing note: As things are currently, this call will always
  ** succeed. This is because the memory cell passed by the VDBE layer 
  ** happens to be the same one as was used to assemble the keys before they
  ** were passed to the sorter - meaning it is always large enough for the
  ** largest key. But this could change very easily, so we leave the call
  ** to sqlite3VdbeMemGrow() in. */
  if( sqlite3VdbeMemGrow(pOut, pIter->nKey, 0) ){
    return SQLITE_NOMEM;
  }
  pOut->n = pIter->nKey;
  MemSetTypeFlag(pOut, MEM_Blob);
  memcpy(pOut->z, pIter->aKey, pIter->nKey);

  return SQLITE_OK;
}

Changes to test/index4.test. 

62
63
64
65
66
67
68
69










































70

  do_execsql_test 1.5 {
    PRAGMA integrity_check 
  } {ok}

  sqlite3_soft_heap_limit $soft_limit
}












































finish_test









>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>


62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112

  do_execsql_test 1.5 {
    PRAGMA integrity_check 
  } {ok}

  sqlite3_soft_heap_limit $soft_limit
}


do_execsql_test 1.6 {
  BEGIN;
    DROP TABLE t1;
    CREATE TABLE t1(x);
    INSERT INTO t1 VALUES('a');
    INSERT INTO t1 VALUES('b');
    INSERT INTO t1 VALUES('c');
    INSERT INTO t1 VALUES('d');
    INSERT INTO t1 VALUES('e');
    INSERT INTO t1 VALUES('f');
    INSERT INTO t1 VALUES('g');
    INSERT INTO t1 VALUES(NULL);
    INSERT INTO t1 SELECT randomblob(1202) FROM t1;     --    16
    INSERT INTO t1 SELECT randomblob(2202) FROM t1;     --    32
    INSERT INTO t1 SELECT randomblob(3202) FROM t1;     --    64
    INSERT INTO t1 SELECT randomblob(4202) FROM t1;     --   128
    INSERT INTO t1 SELECT randomblob(5202) FROM t1;     --   256
  COMMIT;
  CREATE INDEX i1 ON t1(x); 
  PRAGMA integrity_check
} {ok}

do_execsql_test 1.7 {
  BEGIN;
    DROP TABLE t1;
    CREATE TABLE t1(x);
    INSERT INTO t1 VALUES('a');
  COMMIT;
  CREATE INDEX i1 ON t1(x); 
  PRAGMA integrity_check
} {ok}

do_execsql_test 1.8 {
  BEGIN;
    DROP TABLE t1;
    CREATE TABLE t1(x);
  COMMIT;
  CREATE INDEX i1 ON t1(x); 
  PRAGMA integrity_check
} {ok}


finish_test

Added test/indexfault.test. 



















































































































































































































































































































































































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248

# 2011 August 08
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
source $testdir/lock_common.tcl
source $testdir/malloc_common.tcl

set testprefix indexfault

#-------------------------------------------------------------------------
# These tests - indexfault-1.* - Build an index on a smallish table with
# all different kinds of fault-injection. The CREATE INDEX is run once
# with default options and once with a 50KB soft-heap-limit.
#
do_execsql_test 1.0 {
  BEGIN;
    CREATE TABLE t1(x);
    INSERT INTO t1 VALUES(randomblob(202));
    INSERT INTO t1 SELECT randomblob(202) FROM t1;     --     2
    INSERT INTO t1 SELECT randomblob(202) FROM t1;     --     4
    INSERT INTO t1 SELECT randomblob(202) FROM t1;     --     8
    INSERT INTO t1 SELECT randomblob(202) FROM t1;     --    16
    INSERT INTO t1 SELECT randomblob(202) FROM t1;     --    32
    INSERT INTO t1 SELECT randomblob(202) FROM t1;     --    64
    INSERT INTO t1 SELECT randomblob(202) FROM t1;     --   128
    INSERT INTO t1 SELECT randomblob(202) FROM t1;     --   256
  COMMIT;
}
faultsim_save_and_close

do_faultsim_test 1.1 -prep {
  faultsim_restore_and_reopen
} -body {
  execsql { CREATE INDEX i1 ON t1(x) }
  faultsim_test_result {0 {}} 
  faultsim_integrity_check
}
ifcapable memorymanage {
  set soft_limit [sqlite3_soft_heap_limit 50000]
  do_faultsim_test 2.1 -prep {
    faultsim_restore_and_reopen
  } -body {
    execsql { CREATE INDEX i1 ON t1(x) }
    faultsim_test_result {0 {}} 
  }
  sqlite3_soft_heap_limit $soft_limit
}

#-------------------------------------------------------------------------
# These are similar to the indexfault-1.* tests, except they create an
# index with more than one column.
#
sqlite3 db test.db
do_execsql_test 2.0 {
  BEGIN;
    DROP TABLE IF EXISTS t1;
    CREATE TABLE t1(t,u,v,w,x,y,z);
    INSERT INTO t1 VALUES(
      randomblob(30), randomblob(30), randomblob(30), randomblob(30),
      randomblob(30), randomblob(30), randomblob(30)
    );
    INSERT INTO t1 SELECT 
      randomblob(30), randomblob(30), randomblob(30), randomblob(30),
      randomblob(30), randomblob(30), randomblob(30) FROM t1;         -- 2
    INSERT INTO t1 SELECT 
      randomblob(30), randomblob(30), randomblob(30), randomblob(30),
      randomblob(30), randomblob(30), randomblob(30) FROM t1;         -- 4
    INSERT INTO t1 SELECT 
      randomblob(30), randomblob(30), randomblob(30), randomblob(30),
      randomblob(30), randomblob(30), randomblob(30) FROM t1;         -- 8
    INSERT INTO t1 SELECT 
      randomblob(30), randomblob(30), randomblob(30), randomblob(30),
      randomblob(30), randomblob(30), randomblob(30) FROM t1;         -- 16
    INSERT INTO t1 SELECT 
      randomblob(30), randomblob(30), randomblob(30), randomblob(30),
      randomblob(30), randomblob(30), randomblob(30) FROM t1;         -- 32
    INSERT INTO t1 SELECT 
      randomblob(30), randomblob(30), randomblob(30), randomblob(30),
      randomblob(30), randomblob(30), randomblob(30) FROM t1;         -- 64
    INSERT INTO t1 SELECT 
      randomblob(30), randomblob(30), randomblob(30), randomblob(30),
      randomblob(30), randomblob(30), randomblob(30) FROM t1;         -- 128
  COMMIT;
}
faultsim_save_and_close

do_faultsim_test 2.1 -prep {
  faultsim_restore_and_reopen
} -body {
  execsql { CREATE INDEX i1 ON t1(t,u,v,w,x,y,z) }
  faultsim_test_result {0 {}} 
  faultsim_integrity_check
}
ifcapable memorymanage {
  set soft_limit [sqlite3_soft_heap_limit 50000]
  do_faultsim_test 2.2 -prep {
    faultsim_restore_and_reopen
  } -body {
    execsql { CREATE INDEX i1 ON t1(t,u,v,w,x,y,z) }
    faultsim_test_result {0 {}} 
  }
  sqlite3_soft_heap_limit $soft_limit
}

#-------------------------------------------------------------------------
# The following tests - indexfault-2.* - all attempt to build a index
# on table t1 in the main database with injected IO errors. Individual
# test cases work as follows:
#
#   3.1: IO errors injected into xOpen() calls.
#   3.2: As 7.1, but with a low (50KB) soft-heap-limit.
#
#   3.3: IO errors injected into the first 200 write() calls made on the
#        second temporary file.
#   3.4: As 7.3, but with a low (50KB) soft-heap-limit.
#
#

# Set up the custom fault-injector. This is further configured by using
# different values for $::custom_filter and different implementations
# of Tcl proc [xCustom] for each test case.
#
set FAULTSIM(custom)            [list      \
  -injectinstall   custom_injectinstall    \
  -injectstart     custom_injectstart      \
  -injectstop      custom_injectstop       \
  -injecterrlist   {{1 {disk I/O error}}}  \
  -injectuninstall custom_injectuninstall  \
]
proc custom_injectinstall {} {
  testvfs shmfault -default true
  shmfault filter $::custom_filter
  shmfault script xCustom
}
proc custom_injectuninstall {} {
  catch {db  close}
  catch {db2 close}
  shmfault delete
}
set ::custom_ifail -1
set ::custom_nfail -1
proc custom_injectstart {iFail} {
  set ::custom_ifail $iFail
  set ::custom_nfail 0
}
proc custom_injectstop {} {
  set ::custom_ifail -1
  return $::custom_nfail
}

# Set up a table to build indexes on. Save the setup using the 
# [faultsim_save_and_close] mechanism.
# 
sqlite3 db test.db
do_execsql_test 3.0 {
  BEGIN;
    DROP TABLE IF EXISTS t1;
    CREATE TABLE t1(x);
    INSERT INTO t1 VALUES(randomblob(11000));
    INSERT INTO t1 SELECT randomblob(11001) FROM t1;     --     2
    INSERT INTO t1 SELECT randomblob(11002) FROM t1;     --     4
    INSERT INTO t1 SELECT randomblob(11003) FROM t1;     --     8
    INSERT INTO t1 SELECT randomblob(11004) FROM t1;     --    16
    INSERT INTO t1 SELECT randomblob(11005) FROM t1;     --    32
    INSERT INTO t1 SELECT randomblob(11006) FROM t1;     --    64
    INSERT INTO t1 SELECT randomblob(11007) FROM t1;     --   128
    INSERT INTO t1 SELECT randomblob(11008) FROM t1;     --   256
    INSERT INTO t1 SELECT randomblob(11009) FROM t1;     --   512
  COMMIT;
}
faultsim_save_and_close

set ::custom_filter xOpen
proc xCustom {args} {
  incr ::custom_ifail -1
  if {$::custom_ifail==0} {
    incr ::custom_nfail
    return "SQLITE_IOERR"
  }
  return "SQLITE_OK"
}
do_faultsim_test 3.1 -faults custom -prep {
  faultsim_restore_and_reopen
} -body {
  execsql { CREATE INDEX i1 ON t1(x) }
  faultsim_test_result {0 {}} 
}
ifcapable memorymanage {
  set soft_limit [sqlite3_soft_heap_limit 50000]
  do_faultsim_test 3.2 -faults custom -prep {
    faultsim_restore_and_reopen
  } -body {
    execsql { CREATE INDEX i1 ON t1(x) }
    faultsim_test_result {0 {}} 
  }
  sqlite3_soft_heap_limit $soft_limit
}

set ::custom_filter {xOpen xWrite}
proc xCustom {method args} {
  if {$method == "xOpen"} {
    if {[lindex $args 0] == ""} {
      incr ::nTmpOpen 1
      if {$::nTmpOpen == 3} { return "failme" }
    }
    return "SQLITE_OK"
  }
  if {$::custom_ifail<200 && [lindex $args 1] == "failme"} {
    incr ::custom_ifail -1
    if {$::custom_ifail==0} {
      incr ::custom_nfail
      return "SQLITE_IOERR"
    }
  }
  return "SQLITE_OK"
}

do_faultsim_test 3.3 -faults custom -prep {
  faultsim_restore_and_reopen
  set ::nTmpOpen 0
} -body {
  execsql { CREATE INDEX i1 ON t1(x) }
  faultsim_test_result {0 {}} 
}

ifcapable memorymanage {
  set soft_limit [sqlite3_soft_heap_limit 50000]
  do_faultsim_test 3.4 -faults custom -prep {
    faultsim_restore_and_reopen
    set ::nTmpOpen 0
  } -body {
    execsql { CREATE INDEX i1 ON t1(x) }
    faultsim_test_result {0 {}} 
  }
  sqlite3_soft_heap_limit $soft_limit
}

finish_test

Changes to test/permutations.test. 

107
108
109
110
111
112
113
114
115
116
117
118
119
120
121

  misc7.test mutex2.test notify2.test onefile.test pagerfault2.test 
  savepoint4.test savepoint6.test select9.test 
  speed1.test speed1p.test speed2.test speed3.test speed4.test 
  speed4p.test sqllimits1.test tkt2686.test thread001.test thread002.test
  thread003.test thread004.test thread005.test trans2.test vacuum3.test 
  incrvacuum_ioerr.test autovacuum_crash.test btree8.test shared_err.test
  vtab_err.test walslow.test walcrash.test 
  walthread.test rtree3.test
}]
if {[info exists ::env(QUICKTEST_INCLUDE)]} {
  set allquicktests [concat $allquicktests $::env(QUICKTEST_INCLUDE)]
}

#############################################################################
# Start of tests








|








107
108
109
110
111
112
113
114
115
116
117
118
119
120
121

  misc7.test mutex2.test notify2.test onefile.test pagerfault2.test 
  savepoint4.test savepoint6.test select9.test 
  speed1.test speed1p.test speed2.test speed3.test speed4.test 
  speed4p.test sqllimits1.test tkt2686.test thread001.test thread002.test
  thread003.test thread004.test thread005.test trans2.test vacuum3.test 
  incrvacuum_ioerr.test autovacuum_crash.test btree8.test shared_err.test
  vtab_err.test walslow.test walcrash.test 
  walthread.test rtree3.test indexfault.test
}]
if {[info exists ::env(QUICKTEST_INCLUDE)]} {
  set allquicktests [concat $allquicktests $::env(QUICKTEST_INCLUDE)]
}

#############################################################################
# Start of tests

This page was generated in about 0.014s by Fossil 2.11 [a92d5a514d] 2019-10-23 00:46:53