SQLite

Check-in [b79f19ed]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Catch fts5 index corruption caused by issuing 'delete' commands with incorrect data earlier in some cases. Also fix a couple of test script problems.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256: b79f19edfd33c2a75f936c352668e14e81f35acf4f07edc27a21f941a7304b38
User & Date: dan 2020-09-11 15:01:49
Context
2020-09-15
12:29
Do not invoke usleep() for more than 999999 microseconds. (check-in: 1f5ed852 user: drh tags: trunk)
2020-09-11
15:01
Catch fts5 index corruption caused by issuing 'delete' commands with incorrect data earlier in some cases. Also fix a couple of test script problems. (check-in: b79f19ed user: dan tags: trunk)
2020-09-10
15:09
Try again to fix the typo in the sqlite3_txn_state() documentation. (check-in: 6d1ab040 user: drh tags: trunk)
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5_expr.c.

︙︙
2405
2406
2407
2408
2409
2410
2411

2412

2413
2414
2415
2416
2417
2418
2419

      Fts5Colset *pColset = pNear->pColset;
      if( pColset->nCol>1 ) zRet = fts5PrintfAppend(zRet, "{");
      for(ii=0; ii<pColset->nCol; ii++){
        zRet = fts5PrintfAppend(zRet, "%s%s", 
            pConfig->azCol[pColset->aiCol[ii]], ii==pColset->nCol-1 ? "" : " "
        );
      }

      zRet = fts5PrintfAppend(zRet, "%s : ", pColset->nCol>1 ? "}" : "");

      if( zRet==0 ) return 0;
    }

    if( pNear->nPhrase>1 ){
      zRet = fts5PrintfAppend(zRet, "NEAR(");
      if( zRet==0 ) return 0;
    }








>
|
>








2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421

      Fts5Colset *pColset = pNear->pColset;
      if( pColset->nCol>1 ) zRet = fts5PrintfAppend(zRet, "{");
      for(ii=0; ii<pColset->nCol; ii++){
        zRet = fts5PrintfAppend(zRet, "%s%s", 
            pConfig->azCol[pColset->aiCol[ii]], ii==pColset->nCol-1 ? "" : " "
        );
      }
      if( zRet ){
        zRet = fts5PrintfAppend(zRet, "%s : ", pColset->nCol>1 ? "}" : "");
      }
      if( zRet==0 ) return 0;
    }

    if( pNear->nPhrase>1 ){
      zRet = fts5PrintfAppend(zRet, "NEAR(");
      if( zRet==0 ) return 0;
    }
︙︙

Changes to ext/fts5/fts5_storage.c.

︙︙
425
426
427
428
429
430
431


432
433




434

435
436
437
438
439
440
441

        nText = sqlite3_value_bytes(apVal[iCol-1]);
      }
      ctx.szCol = 0;
      rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, 
          zText, nText, (void*)&ctx, fts5StorageInsertCallback
      );
      p->aTotalSize[iCol-1] -= (i64)ctx.szCol;


    }
  }




  p->nTotalRow--;


  rc2 = sqlite3_reset(pSeek);
  if( rc==SQLITE_OK ) rc = rc2;
  return rc;
}










>
>
|
|
>
>
>
>
|
>








425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448

        nText = sqlite3_value_bytes(apVal[iCol-1]);
      }
      ctx.szCol = 0;
      rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, 
          zText, nText, (void*)&ctx, fts5StorageInsertCallback
      );
      p->aTotalSize[iCol-1] -= (i64)ctx.szCol;
      if( p->aTotalSize[iCol-1]<0 ){
        rc = FTS5_CORRUPT;
      }
    }
  }
  if( rc==SQLITE_OK && p->nTotalRow<1 ){
    rc = FTS5_CORRUPT;
  }else{
    p->nTotalRow--;
  }

  rc2 = sqlite3_reset(pSeek);
  if( rc==SQLITE_OK ) rc = rc2;
  return rc;
}
︙︙

Changes to ext/fts5/test/fts5corrupt3.test.

︙︙
9687
9688
9689
9690
9691
9692
9693
9694
9695
9696
9697
9698
9699
9700
9701

|      0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00   ................
|   4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04   ........version.
| end crash-3aef66940ace0c.db
}]} {}

do_catchsql_test 65.1 {
  SELECT ( MATCH (t1,591)) FROM t1 WHERE t1 MATCH 'e*eŸ'
} {1 {database disk image is malformed}}

#-------------------------------------------------------------------------
#
reset_db
do_test 66.0 {
  sqlite3 db {}
  db deserialize [decode_hexdb {








|








9687
9688
9689
9690
9691
9692
9693
9694
9695
9696
9697
9698
9699
9700
9701

|      0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00   ................
|   4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04   ........version.
| end crash-3aef66940ace0c.db
}]} {}

do_catchsql_test 65.1 {
  SELECT ( MATCH (t1,591)) FROM t1 WHERE t1 MATCH 'e*eŸ'
} {1 {malformed database schema (t2) - invalid rootpage}}

#-------------------------------------------------------------------------
#
reset_db
do_test 66.0 {
  sqlite3 db {}
  db deserialize [decode_hexdb {
︙︙

Changes to ext/fts5/test/fts5delete.test.

︙︙
45
46
47
48
49
50
51











































52
53

do_test 1.2 {
  execsql { INSERT INTO t1(t1, rank) VALUES('usermerge', 2); }
  for {set i 0} {$i < 5} {incr i} {
    execsql { INSERT INTO t1(t1, rank) VALUES('merge', 1) }
    execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
  }
} {}












































finish_test








>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>



45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96

do_test 1.2 {
  execsql { INSERT INTO t1(t1, rank) VALUES('usermerge', 2); }
  for {set i 0} {$i < 5} {incr i} {
    execsql { INSERT INTO t1(t1, rank) VALUES('merge', 1) }
    execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
  }
} {}

#-------------------------------------------------------------------------
reset_db
do_execsql_test 2.0 {
  CREATE TABLE test (
      id INTEGER PRIMARY KEY,
      name TEXT,
      value TEXT
  );
  CREATE VIRTUAL TABLE test_idx USING fts5(
      name, content=test, content_rowid=id
  );
}

do_catchsql_test 2.1 {
  INSERT INTO test_idx (test_idx, rowid, name) VALUES('delete', 1, 'quick');
} {1 {database disk image is malformed}}

do_catchsql_test 2.2 {
  INSERT INTO test_idx(rowid, name) VALUES(123, 'one one one');
  INSERT INTO test_idx (test_idx, rowid, name) VALUES('delete', 123, 'one');
  INSERT INTO test_idx (test_idx, rowid, name) VALUES('delete', 123, 'one');
} {1 {database disk image is malformed}}

do_execsql_test 2.3 {
  DROP TABLE test_idx;
  CREATE VIRTUAL TABLE test_idx USING fts5(
      name, content=test, content_rowid=id
  );

  INSERT INTO test_idx(rowid, name) VALUES(123, 'one one one');
  INSERT INTO test_idx(rowid, name) VALUES(124, 'two two two');
  INSERT INTO test_idx(rowid, name) VALUES(125, 'two two two');
  INSERT INTO test_idx (test_idx, rowid, name) VALUES('delete', 123, 'one');
  INSERT INTO test_idx (test_idx, rowid, name) VALUES('delete', 123, 'one');
  INSERT INTO test_idx (test_idx, rowid, name) VALUES('delete', 123, 'one');
}

do_catchsql_test 2.4 {
  SELECT rowid FROM test_idx WHERE test_idx MATCH 'two' ORDER BY rank;
} {1 {database disk image is malformed}}



finish_test