/ Check-in [d3cae986]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fixes to fts3 integrity check code.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts3-refactor
Files: files | file ages | folders
SHA1: d3cae986ee1a176b1b015c3cebcd58ff0c3bdf92
User & Date: dan 2009-11-15 06:50:11
Original User & Date: dan 2009-11-14 23:50:11
Context
2009-11-16
16:36
Add a few extra coverage test cases for fts3. check-in: f29c8fca user: dan tags: fts3-refactor
2009-11-15
06:50
Fixes to fts3 integrity check code. check-in: d3cae986 user: dan tags: fts3-refactor
2009-11-14
18:41
Further OOM testing for fts3 code. Add Tcl code implementing an integrity-check for fts3. check-in: c27d46b3 user: dan tags: fts3-refactor
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/test_hexio.c.

311
312
313
314
315
316
317













318
319
320
321
322
323
324
...
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
  nOut = sqlite3Utf8To8(z);
  sqlite3TestBinToHex(z,nOut);
  Tcl_AppendResult(interp, (char*)z, 0);
  sqlite3_free(z);
#endif
  return TCL_OK;
}














/*
** USAGE:  read_varint BLOB VARNAME
**
** Read a varint from the start of BLOB. Set variable VARNAME to contain
** the interpreted value. Return the number of bytes of BLOB consumed.
*/
................................................................................

  if( objc!=3 ){
    Tcl_WrongNumArgs(interp, 1, objv, "BLOB VARNAME");
    return TCL_ERROR;
  }
  zBlob = Tcl_GetByteArrayFromObj(objv[1], &nBlob);

  nVal = sqlite3GetVarint(zBlob, (sqlite3_uint64 *)(&iVal));
  Tcl_ObjSetVar2(interp, objv[2], 0, Tcl_NewWideIntObj(iVal), 0);
  Tcl_SetObjResult(interp, Tcl_NewIntObj(nVal));
  return TCL_OK;
}


/*







>
>
>
>
>
>
>
>
>
>
>
>
>







 







|







311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
...
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
  nOut = sqlite3Utf8To8(z);
  sqlite3TestBinToHex(z,nOut);
  Tcl_AppendResult(interp, (char*)z, 0);
  sqlite3_free(z);
#endif
  return TCL_OK;
}

static int getFts3Varint(const char *p, sqlite_int64 *v){
  const unsigned char *q = (const unsigned char *) p;
  sqlite_uint64 x = 0, y = 1;
  while( (*q & 0x80) == 0x80 ){
    x += y * (*q++ & 0x7f);
    y <<= 7;
  }
  x += y * (*q++);
  *v = (sqlite_int64) x;
  return (int) (q - (unsigned char *)p);
}


/*
** USAGE:  read_varint BLOB VARNAME
**
** Read a varint from the start of BLOB. Set variable VARNAME to contain
** the interpreted value. Return the number of bytes of BLOB consumed.
*/
................................................................................

  if( objc!=3 ){
    Tcl_WrongNumArgs(interp, 1, objv, "BLOB VARNAME");
    return TCL_ERROR;
  }
  zBlob = Tcl_GetByteArrayFromObj(objv[1], &nBlob);

  nVal = getFts3Varint(zBlob, (sqlite3_uint64 *)(&iVal));
  Tcl_ObjSetVar2(interp, objv[2], 0, Tcl_NewWideIntObj(iVal), 0);
  Tcl_SetObjResult(interp, Tcl_NewIntObj(nVal));
  return TCL_OK;
}


/*

Changes to test/fts3_common.tcl.

76
77
78
79
80
81
82




83
84
85
86
87
88
89
90
91




92
93
94
95
96
97

98
99
100
101
102
103
104
...
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
            set C($docid,$iCol,$iPos) $zTerm
          }
        }
      }
    }
  }






  db eval "SELECT * FROM ${tbl}_content" E {
    set iCol 0
    set iDoc $E(docid)
    foreach col [lrange $E(*) 1 end] {
      set c $E($col)
      set sql {SELECT fts3_tokenizer_test('simple', $c)}

      foreach {pos term dummy} [db one $sql] {




        if {$C($iDoc,$iCol,$pos) != "$term"} {
          set    es "Error at docid=$iDoc col=$iCol pos=$pos. "
          append es "Index has \"$C($iDoc,$iCol,$pos)\", document has \"$term\""
          lappend errors $es
        }
        unset C($iDoc,$iCol,$pos)

      }
      incr iCol
    }
  }

  foreach c [array names C] {
    lappend errors "Bad index entry: $c -> $C($c)"
................................................................................
            ORDER BY level ASC, idx DESC
  " {
    if {$start_block == 0} {
      foreach {t d} [fts3_readleaf $root] { lappend a($t) $d }
    } else {
      db eval " SELECT block 
                FROM ${tbl}_segments 
                WHERE blockid>=$start_block AND blockid<$leaves_end_block
                ORDER BY blockid
      " {
        foreach {t d} [fts3_readleaf $block] { lappend a($t) $d }

      }
    }
  }







>
>
>
>









>
>
>
>
|
|
|
|
|
|
>







 







|







76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
...
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
            set C($docid,$iCol,$iPos) $zTerm
          }
        }
      }
    }
  }

  foreach key [array names C] {
    #puts "$key -> $C($key)"
  }


  db eval "SELECT * FROM ${tbl}_content" E {
    set iCol 0
    set iDoc $E(docid)
    foreach col [lrange $E(*) 1 end] {
      set c $E($col)
      set sql {SELECT fts3_tokenizer_test('simple', $c)}

      foreach {pos term dummy} [db one $sql] {
        if {![info exists C($iDoc,$iCol,$pos)]} {
          set es "Error at docid=$iDoc col=$iCol pos=$pos. Index is missing"
          lappend errors $es
        } else {
          if {$C($iDoc,$iCol,$pos) != "$term"} {
            set    es "Error at docid=$iDoc col=$iCol pos=$pos. Index "
            append es "has \"$C($iDoc,$iCol,$pos)\", document has \"$term\""
            lappend errors $es
          }
          unset C($iDoc,$iCol,$pos)
        }
      }
      incr iCol
    }
  }

  foreach c [array names C] {
    lappend errors "Bad index entry: $c -> $C($c)"
................................................................................
            ORDER BY level ASC, idx DESC
  " {
    if {$start_block == 0} {
      foreach {t d} [fts3_readleaf $root] { lappend a($t) $d }
    } else {
      db eval " SELECT block 
                FROM ${tbl}_segments 
                WHERE blockid>=$start_block AND blockid<=$leaves_end_block
                ORDER BY blockid
      " {
        foreach {t d} [fts3_readleaf $block] { lappend a($t) $d }

      }
    }
  }