SQLite

Check-in [6b67eba54e]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Improvements to detection of corruption in the %_stat shadow table of FTS4.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256: 6b67eba54e7b4887a8cdab7537b12a95e9f17bcdaa0b423af5ed5bd91f69f2c6
User & Date: drh 2019-11-18 14:04:21.649
Context
2019-11-18
17:46
Add support for SQLITE_OPEN_NOFOLLOW. (check-in: cb79c82849 user: drh tags: trunk)
14:04
Improvements to detection of corruption in the %_stat shadow table of FTS4. (check-in: 6b67eba54e user: drh tags: trunk)
12:04
Further improvements to shadow table corruption detection in FTS3. (check-in: e35d8c76aa user: drh tags: trunk)
Changes
Unified Diff Ignore Whitespace Patch
Changes to ext/fts3/fts3.c.
︙︙
4891
4892
4893
4894
4895
4896
4897

4898
4899
4900
4901
4902
4903

4904
4905
4906
4907
4908
4909
4910

    sqlite3_int64 nByte = 0;
    const char *pEnd;
    const char *a;

    rc = sqlite3Fts3SelectDoctotal(p, &pStmt);
    if( rc!=SQLITE_OK ) return rc;
    a = sqlite3_column_blob(pStmt, 0);

    assert( a );

    pEnd = &a[sqlite3_column_bytes(pStmt, 0)];
    a += sqlite3Fts3GetVarint(a, &nDoc);
    while( a<pEnd ){
      a += sqlite3Fts3GetVarint(a, &nByte);

    }
    if( nDoc==0 || nByte==0 ){
      sqlite3_reset(pStmt);
      return FTS_CORRUPT_VTAB;
    }

    pCsr->nDoc = nDoc;








>
|
<
|
|
|
|
>








4891
4892
4893
4894
4895
4896
4897
4898
4899

4900
4901
4902
4903
4904
4905
4906
4907
4908
4909
4910
4911

    sqlite3_int64 nByte = 0;
    const char *pEnd;
    const char *a;

    rc = sqlite3Fts3SelectDoctotal(p, &pStmt);
    if( rc!=SQLITE_OK ) return rc;
    a = sqlite3_column_blob(pStmt, 0);
    testcase( a==0 );  /* If %_stat.value set to X'' */
    if( a ){

      pEnd = &a[sqlite3_column_bytes(pStmt, 0)];
      a += sqlite3Fts3GetVarintBounded(a, pEnd, &nDoc);
      while( a<pEnd ){
        a += sqlite3Fts3GetVarintBounded(a, pEnd, &nByte);
      }
    }
    if( nDoc==0 || nByte==0 ){
      sqlite3_reset(pStmt);
      return FTS_CORRUPT_VTAB;
    }

    pCsr->nDoc = nDoc;
︙︙
Changes to ext/fts3/fts3_snippet.c.
︙︙
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071

    int rc = sqlite3Fts3SelectDoctotal(pTab, ppStmt);
    if( rc!=SQLITE_OK ) return rc;
  }
  pStmt = *ppStmt;
  assert( sqlite3_data_count(pStmt)==1 );

  n = sqlite3_column_bytes(pStmt, 0);
  if( n==0 ){
    return FTS_CORRUPT_VTAB;
  }
  a = sqlite3_column_blob(pStmt, 0);
  if( a==0 ){
    return SQLITE_NOMEM;
  }
  pEnd = a + n;
  a += sqlite3Fts3GetVarintBounded(a, pEnd, &nDoc);
  if( nDoc<=0 || a>pEnd ){
    return FTS_CORRUPT_VTAB;
  }
  *pnDoc = nDoc;








<
<
<


|








1052
1053
1054
1055
1056
1057
1058



1059
1060
1061
1062
1063
1064
1065
1066
1067
1068

    int rc = sqlite3Fts3SelectDoctotal(pTab, ppStmt);
    if( rc!=SQLITE_OK ) return rc;
  }
  pStmt = *ppStmt;
  assert( sqlite3_data_count(pStmt)==1 );

  n = sqlite3_column_bytes(pStmt, 0);



  a = sqlite3_column_blob(pStmt, 0);
  if( a==0 ){
    return FTS_CORRUPT_VTAB;
  }
  pEnd = a + n;
  a += sqlite3Fts3GetVarintBounded(a, pEnd, &nDoc);
  if( nDoc<=0 || a>pEnd ){
    return FTS_CORRUPT_VTAB;
  }
  *pnDoc = nDoc;
︙︙
Changes to ext/fts3/fts3_tokenizer.c.
︙︙
386
387
388
389
390
391
392
393


394
395
396
397
398
399
400

  rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
  if( rc!=SQLITE_OK ){
    return rc;
  }

  sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
  if( SQLITE_ROW==sqlite3_step(pStmt) ){
    if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){


      memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
    }
  }

  return sqlite3_finalize(pStmt);
}









|
>
>








386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402

  rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
  if( rc!=SQLITE_OK ){
    return rc;
  }

  sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
  if( SQLITE_ROW==sqlite3_step(pStmt) ){
    if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB
     && sqlite3_column_bytes(pStmt, 0)==sizeof(*pp)
    ){
      memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
    }
  }

  return sqlite3_finalize(pStmt);
}
︙︙
Changes to test/fts3auto.test.
︙︙
566
567
568
569
570
571
572





573
574
575
576
577
578
579

  do_fts3query_test 4.$tn.3.5 -deferred five t1 {one NEAR/3 five}

  do_fts3query_test 4.$tn.4.1 -deferred fi* t1 {on* AND fi*}
  do_fts3query_test 4.$tn.4.2 -deferred fi* t1 {on* NEAR fi*}
  do_fts3query_test 4.$tn.4.3 -deferred fi* t1 {on* NEAR/1 fi*}
  do_fts3query_test 4.$tn.4.4 -deferred fi* t1 {on* NEAR/2 fi*}
  do_fts3query_test 4.$tn.4.5 -deferred fi* t1 {on* NEAR/3 fi*}





}

#--------------------------------------------------------------------------
# The following test cases - fts3auto-5.* - focus on using prefix indexes.
#
set chunkconfig [fts3_configure_incr_load 1 1]
foreach {tn create pending} {








>
>
>
>
>








566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584

  do_fts3query_test 4.$tn.3.5 -deferred five t1 {one NEAR/3 five}

  do_fts3query_test 4.$tn.4.1 -deferred fi* t1 {on* AND fi*}
  do_fts3query_test 4.$tn.4.2 -deferred fi* t1 {on* NEAR fi*}
  do_fts3query_test 4.$tn.4.3 -deferred fi* t1 {on* NEAR/1 fi*}
  do_fts3query_test 4.$tn.4.4 -deferred fi* t1 {on* NEAR/2 fi*}
  do_fts3query_test 4.$tn.4.5 -deferred fi* t1 {on* NEAR/3 fi*}

  db eval {UPDATE t1_stat SET value=x'' WHERE id=0}
  do_catchsql_test 4.$tn.4.6 {
    SELECT docid FROM t1 WHERE t1 MATCH 'on* NEAR/3 fi*'
  } {1 {database disk image is malformed}}
}

#--------------------------------------------------------------------------
# The following test cases - fts3auto-5.* - focus on using prefix indexes.
#
set chunkconfig [fts3_configure_incr_load 1 1]
foreach {tn create pending} {
︙︙