SQLite

Check-in [e79b97369f]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Ensure that the xIntegrity methods of fts3 and fts5 work on read-only databases.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | branch-3.45
Files: files | file ages | folders
SHA3-256: e79b97369fa740f62f695057d4a2cf8dae48a683982ec879f04a19039c9cb418
User & Date: dan 2024-01-23 10:47:04.969
Context
2024-01-23
13:53
When a JSON input is a blob, but it looks like valid JSON when cast to text, then accept it as valid JSON. This replicates a long-standing bug in the behavior of JSON routines, and thus avoids breaking legacy apps. (check-in: 4c2c1b97dc user: drh tags: branch-3.45)
10:47
Ensure that the xIntegrity methods of fts3 and fts5 work on read-only databases. (check-in: e79b97369f user: dan tags: branch-3.45)
2024-01-20
12:19
When backing out a character in a constructed string in JSON, first make sure the string has not been reset by on OOM. (check-in: 950bf9fe78 user: drh tags: branch-3.45)
2024-01-16
16:05
Ensure that the xIntegrity methods of fts3 and fts5 work on read-only databases. (check-in: b855886c4c user: dan tags: trunk)
Changes
Unified Diff Ignore Whitespace Patch
Changes to ext/fts3/fts3.c.
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038



4039
4040
4041
4042
4043
4044
4045
4046
4047
  return 0;
}

/*
** Implementation of the xIntegrity() method on the FTS3/FTS4 virtual
** table.
*/
static int fts3Integrity(
  sqlite3_vtab *pVtab,      /* The virtual table to be checked */
  const char *zSchema,      /* Name of schema in which pVtab lives */
  const char *zTabname,     /* Name of the pVTab table */
  int isQuick,              /* True if this is a quick_check */
  char **pzErr              /* Write error message here */
){
  Fts3Table *p = (Fts3Table*)pVtab;
  char *zSql;
  int rc;
  char *zErr = 0;

  assert( pzErr!=0 );
  assert( *pzErr==0 );
  UNUSED_PARAMETER(isQuick);
  zSql = sqlite3_mprintf(
            "INSERT INTO \"%w\".\"%w\"(\"%w\") VALUES('integrity-check');",
            zSchema, zTabname, zTabname);
  if( zSql==0 ){
    return SQLITE_NOMEM;
  }
  rc = sqlite3_exec(p->db, zSql, 0, 0, &zErr);
  sqlite3_free(zSql);
  if( (rc&0xff)==SQLITE_CORRUPT ){
    *pzErr = sqlite3_mprintf("malformed inverted index for FTS%d table %s.%s",
                p->bFts4 ? 4 : 3, zSchema, zTabname);
  }else if( rc!=SQLITE_OK ){
    *pzErr = sqlite3_mprintf("unable to validate the inverted index for"
                             " FTS%d table %s.%s: %s",
                p->bFts4 ? 4 : 3, zSchema, zTabname, zErr);



  }
  sqlite3_free(zErr);
  return SQLITE_OK;
}



static const sqlite3_module fts3Module = {
  /* iVersion      */ 4,







|







<

|

<
<
<
|
<
<
<
<
<
<
<
|
<
<
|


|
>
>
>

|







4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016

4017
4018
4019



4020







4021


4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
  return 0;
}

/*
** Implementation of the xIntegrity() method on the FTS3/FTS4 virtual
** table.
*/
static int fts3IntegrityMethod(
  sqlite3_vtab *pVtab,      /* The virtual table to be checked */
  const char *zSchema,      /* Name of schema in which pVtab lives */
  const char *zTabname,     /* Name of the pVTab table */
  int isQuick,              /* True if this is a quick_check */
  char **pzErr              /* Write error message here */
){
  Fts3Table *p = (Fts3Table*)pVtab;

  int rc;
  int bOk = 0;




  rc = sqlite3Fts3IntegrityCheck(p, &bOk);







  assert( rc!=SQLITE_CORRUPT_VTAB || bOk==0 );


  if( rc!=SQLITE_OK && rc!=SQLITE_CORRUPT_VTAB ){
    *pzErr = sqlite3_mprintf("unable to validate the inverted index for"
                             " FTS%d table %s.%s: %s",
                p->bFts4 ? 4 : 3, zSchema, zTabname, sqlite3_errstr(rc));
  }else if( bOk==0 ){
    *pzErr = sqlite3_mprintf("malformed inverted index for FTS%d table %s.%s",
                p->bFts4 ? 4 : 3, zSchema, zTabname);
  }
  sqlite3Fts3SegmentsClose(p);
  return SQLITE_OK;
}



static const sqlite3_module fts3Module = {
  /* iVersion      */ 4,
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
  /* xRollback     */ fts3RollbackMethod,
  /* xFindFunction */ fts3FindFunctionMethod,
  /* xRename */       fts3RenameMethod,
  /* xSavepoint    */ fts3SavepointMethod,
  /* xRelease      */ fts3ReleaseMethod,
  /* xRollbackTo   */ fts3RollbackToMethod,
  /* xShadowName   */ fts3ShadowName,
  /* xIntegrity    */ fts3Integrity,
};

/*
** This function is registered as the module destructor (called when an
** FTS3 enabled database connection is closed). It frees the memory
** allocated for the tokenizer hash table.
*/







|







4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
  /* xRollback     */ fts3RollbackMethod,
  /* xFindFunction */ fts3FindFunctionMethod,
  /* xRename */       fts3RenameMethod,
  /* xSavepoint    */ fts3SavepointMethod,
  /* xRelease      */ fts3ReleaseMethod,
  /* xRollbackTo   */ fts3RollbackToMethod,
  /* xShadowName   */ fts3ShadowName,
  /* xIntegrity    */ fts3IntegrityMethod,
};

/*
** This function is registered as the module destructor (called when an
** FTS3 enabled database connection is closed). It frees the memory
** allocated for the tokenizer hash table.
*/
Changes to ext/fts3/fts3Int.h.
648
649
650
651
652
653
654


655
656
657
#ifndef SQLITE_DISABLE_FTS3_UNICODE
int sqlite3FtsUnicodeFold(int, int);
int sqlite3FtsUnicodeIsalnum(int);
int sqlite3FtsUnicodeIsdiacritic(int);
#endif

int sqlite3Fts3ExprIterate(Fts3Expr*, int (*x)(Fts3Expr*,int,void*), void*);



#endif /* !SQLITE_CORE || SQLITE_ENABLE_FTS3 */
#endif /* _FTSINT_H */







>
>



648
649
650
651
652
653
654
655
656
657
658
659
#ifndef SQLITE_DISABLE_FTS3_UNICODE
int sqlite3FtsUnicodeFold(int, int);
int sqlite3FtsUnicodeIsalnum(int);
int sqlite3FtsUnicodeIsdiacritic(int);
#endif

int sqlite3Fts3ExprIterate(Fts3Expr*, int (*x)(Fts3Expr*,int,void*), void*);

int sqlite3Fts3IntegrityCheck(Fts3Table *p, int *pbOk);

#endif /* !SQLITE_CORE || SQLITE_ENABLE_FTS3 */
#endif /* _FTSINT_H */
Changes to ext/fts3/fts3_write.c.
5290
5291
5292
5293
5294
5295
5296
5297
5298
5299
5300
5301
5302
5303
5304
** content table. If no error occurs and the contents do match, set *pbOk
** to true and return SQLITE_OK. Or if the contents do not match, set *pbOk
** to false before returning.
**
** If an error occurs (e.g. an OOM or IO error), return an SQLite error 
** code. The final value of *pbOk is undefined in this case.
*/
static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){
  int rc = SQLITE_OK;             /* Return code */
  u64 cksum1 = 0;                 /* Checksum based on FTS index contents */
  u64 cksum2 = 0;                 /* Checksum based on %_content contents */
  sqlite3_stmt *pAllLangid = 0;   /* Statement to return all language-ids */

  /* This block calculates the checksum according to the FTS index. */
  rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0);







|







5290
5291
5292
5293
5294
5295
5296
5297
5298
5299
5300
5301
5302
5303
5304
** content table. If no error occurs and the contents do match, set *pbOk
** to true and return SQLITE_OK. Or if the contents do not match, set *pbOk
** to false before returning.
**
** If an error occurs (e.g. an OOM or IO error), return an SQLite error 
** code. The final value of *pbOk is undefined in this case.
*/
int sqlite3Fts3IntegrityCheck(Fts3Table *p, int *pbOk){
  int rc = SQLITE_OK;             /* Return code */
  u64 cksum1 = 0;                 /* Checksum based on FTS index contents */
  u64 cksum2 = 0;                 /* Checksum based on %_content contents */
  sqlite3_stmt *pAllLangid = 0;   /* Statement to return all language-ids */

  /* This block calculates the checksum according to the FTS index. */
  rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0);
5368
5369
5370
5371
5372
5373
5374
5375
5376
5377
5378
5379
5380
5381
5382
        }
      }
    }

    sqlite3_finalize(pStmt);
  }

  *pbOk = (cksum1==cksum2);
  return rc;
}

/*
** Run the integrity-check. If no error occurs and the current contents of
** the FTS index are correct, return SQLITE_OK. Or, if the contents of the
** FTS index are incorrect, return SQLITE_CORRUPT_VTAB.







|







5368
5369
5370
5371
5372
5373
5374
5375
5376
5377
5378
5379
5380
5381
5382
        }
      }
    }

    sqlite3_finalize(pStmt);
  }

  *pbOk = (rc==SQLITE_OK && cksum1==cksum2);
  return rc;
}

/*
** Run the integrity-check. If no error occurs and the current contents of
** the FTS index are correct, return SQLITE_OK. Or, if the contents of the
** FTS index are incorrect, return SQLITE_CORRUPT_VTAB.
5408
5409
5410
5411
5412
5413
5414
5415
5416
5417
5418
5419
5420
5421
5422
** passed.
*/
static int fts3DoIntegrityCheck(
  Fts3Table *p                    /* FTS3 table handle */
){
  int rc;
  int bOk = 0;
  rc = fts3IntegrityCheck(p, &bOk);
  if( rc==SQLITE_OK && bOk==0 ) rc = FTS_CORRUPT_VTAB;
  return rc;
}

/*
** Handle a 'special' INSERT of the form:
**







|







5408
5409
5410
5411
5412
5413
5414
5415
5416
5417
5418
5419
5420
5421
5422
** passed.
*/
static int fts3DoIntegrityCheck(
  Fts3Table *p                    /* FTS3 table handle */
){
  int rc;
  int bOk = 0;
  rc = sqlite3Fts3IntegrityCheck(p, &bOk);
  if( rc==SQLITE_OK && bOk==0 ) rc = FTS_CORRUPT_VTAB;
  return rc;
}

/*
** Handle a 'special' INSERT of the form:
**
Changes to ext/fts5/fts5_main.c.
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977

2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993

2994
2995
2996
2997
2998
2999
3000
3001
  sqlite3_vtab *pVtab,    /* the FTS5 virtual table to check */
  const char *zSchema,    /* Name of schema in which this table lives */
  const char *zTabname,   /* Name of the table itself */
  int isQuick,            /* True if this is a quick-check */
  char **pzErr            /* Write error message here */
){
  Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
  Fts5Config *pConfig = pTab->p.pConfig;
  char *zSql;
  char *zErr = 0;
  int rc;

  assert( pzErr!=0 && *pzErr==0 );
  UNUSED_PARAM(isQuick);
  zSql = sqlite3_mprintf(
            "INSERT INTO \"%w\".\"%w\"(\"%w\") VALUES('integrity-check');",
            zSchema, zTabname, pConfig->zName);
  if( zSql==0 ) return SQLITE_NOMEM;
  rc = sqlite3_exec(pConfig->db, zSql, 0, 0, &zErr);
  sqlite3_free(zSql);
  if( (rc&0xff)==SQLITE_CORRUPT ){
    *pzErr = sqlite3_mprintf("malformed inverted index for FTS5 table %s.%s",
                zSchema, zTabname);
  }else if( rc!=SQLITE_OK ){
    *pzErr = sqlite3_mprintf("unable to validate the inverted index for"
                             " FTS5 table %s.%s: %s",
                zSchema, zTabname, zErr);
  }

  sqlite3_free(zErr);
  return SQLITE_OK;
}

static int fts5Init(sqlite3 *db){
  static const sqlite3_module fts5Mod = {
    /* iVersion      */ 4,
    /* xCreate       */ fts5CreateMethod,







<
<
<

>


|
<
<
<
<
<






|

>
|







2967
2968
2969
2970
2971
2972
2973



2974
2975
2976
2977
2978





2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
  sqlite3_vtab *pVtab,    /* the FTS5 virtual table to check */
  const char *zSchema,    /* Name of schema in which this table lives */
  const char *zTabname,   /* Name of the table itself */
  int isQuick,            /* True if this is a quick-check */
  char **pzErr            /* Write error message here */
){
  Fts5FullTable *pTab = (Fts5FullTable*)pVtab;



  int rc;

  assert( pzErr!=0 && *pzErr==0 );
  UNUSED_PARAM(isQuick);
  rc = sqlite3Fts5StorageIntegrity(pTab->pStorage, 0);





  if( (rc&0xff)==SQLITE_CORRUPT ){
    *pzErr = sqlite3_mprintf("malformed inverted index for FTS5 table %s.%s",
                zSchema, zTabname);
  }else if( rc!=SQLITE_OK ){
    *pzErr = sqlite3_mprintf("unable to validate the inverted index for"
                             " FTS5 table %s.%s: %s",
                zSchema, zTabname, sqlite3_errstr(rc));
  }
  sqlite3Fts5IndexCloseReader(pTab->p.pIndex);

  return SQLITE_OK;
}

static int fts5Init(sqlite3 *db){
  static const sqlite3_module fts5Mod = {
    /* iVersion      */ 4,
    /* xCreate       */ fts5CreateMethod,
Changes to ext/fts5/test/fts5integrity.test.
350
351
352
353
354
355
356


























357
358
do_execsql_test 11.3 {
  PRAGMA integrity_check(t2);
} {ok}
do_execsql_test 11.4 {
  DROP TABLE t1;
  PRAGMA integrity_check(t2);
} {ok}



























finish_test







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>


350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
do_execsql_test 11.3 {
  PRAGMA integrity_check(t2);
} {ok}
do_execsql_test 11.4 {
  DROP TABLE t1;
  PRAGMA integrity_check(t2);
} {ok}

#-------------------------------------------------------------------
reset_db

do_execsql_test 12.1 {
  CREATE VIRTUAL TABLE x1 USING fts5(a, b);
  INSERT INTO x1 VALUES('one', 'two');
  INSERT INTO x1 VALUES('three', 'four');
  INSERT INTO x1 VALUES('five', 'six');
}

do_execsql_test 12.2 {
  PRAGMA integrity_check
} {ok}

db close
sqlite3 db test.db -readonly 1

explain_i {
  PRAGMA integrity_check
  }
do_execsql_test 12.3 {
  PRAGMA integrity_check
} {ok}



finish_test
Changes to test/fts4intck1.test.
49
50
51
52
53
54
55

















56
57
58
  PRAGMA integrity_check(t2);
} {ok}

proc slang {in} {return $in}
do_execsql_test 2.3 {
  PRAGMA integrity_check(t2);
} {{malformed inverted index for FTS4 table main.t2}}



















finish_test







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>



49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
  PRAGMA integrity_check(t2);
} {ok}

proc slang {in} {return $in}
do_execsql_test 2.3 {
  PRAGMA integrity_check(t2);
} {{malformed inverted index for FTS4 table main.t2}}

#-------------------------------------------------------------------------
# Test that integrity-check works on a read-only database.
#
reset_db
do_execsql_test 3.0 {
  CREATE VIRTUAL TABLE x1 USING fts4(a, b);
  INSERT INTO x1 VALUES('one', 'two');
  INSERT INTO x1 VALUES('three', 'four');
}
db close
sqlite3 db test.db -readonly 1

do_execsql_test 3.1 {
  PRAGMA integrity_check;
} {ok}



finish_test