Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
| Comment: | Fix the fts5 integrity-check code so that it works with detail=none tables. |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | fts5-offsets |
| Files: | files | file ages | folders |
| SHA1: |
3a1df69e58e5830da1dff158eedbe381 |
| User & Date: | dan 2015-12-31 18:39:14 |
Context
|
2016-01-02
| ||
| 19:01 | Changes to run many fts5 tests with detail=none and detail=col tables as well as the default detail=full. Also fixes for the bugs uncovered by running said tests. check-in: 6322a1d9 user: dan tags: fts5-offsets | |
|
2015-12-31
| ||
| 18:39 | Fix the fts5 integrity-check code so that it works with detail=none tables. check-in: 3a1df69e user: dan tags: fts5-offsets | |
| 17:36 | Fix some problems with fts5 detail=none tables. Some still remain. check-in: 6a6f7bc4 user: dan tags: fts5-offsets | |
Changes
Changes to ext/fts5/fts5_index.c.
5195 5196 5197 5198 5199 5200 5201 5202 5203 5204 5205 5206 5207 5208 5209 5210 5211 5212 5213 5214 5215 5216 5217 5218 5219 5220 5221 5222 5223 5224 5225 5226 .... 5519 5520 5521 5522 5523 5524 5525 5526 5527 5528 5529 5530 5531 5532 .... 5570 5571 5572 5573 5574 5575 5576 5577 5578 5579 5580 5581 5582 5583 5584 5585 5586 5587 5588 5589 |
Fts5Index *p, /* Fts5 index object */
int iIdx,
const char *z, /* Index key to query for */
int n, /* Size of index key in bytes */
int flags, /* Flags for Fts5IndexQuery */
u64 *pCksum /* IN/OUT: Checksum value */
){
u64 cksum = *pCksum;
Fts5IndexIter *pIdxIter = 0;
Fts5Buffer buf = {0, 0, 0};
int rc = sqlite3Fts5IndexQuery(p, z, n, flags, 0, &pIdxIter);
while( rc==SQLITE_OK && 0==sqlite3Fts5IterEof(pIdxIter) ){
i64 rowid = sqlite3Fts5IterRowid(pIdxIter);
rc = sqlite3Fts5IterPoslistBuffer(pIdxIter, &buf);
if( rc==SQLITE_OK ){
Fts5PoslistReader sReader;
for(sqlite3Fts5PoslistReaderInit(buf.p, buf.n, &sReader);
sReader.bEof==0;
sqlite3Fts5PoslistReaderNext(&sReader)
){
int iCol = FTS5_POS2COLUMN(sReader.iPos);
int iOff = FTS5_POS2OFFSET(sReader.iPos);
cksum ^= sqlite3Fts5IndexEntryCksum(rowid, iCol, iOff, iIdx, z, n);
}
rc = sqlite3Fts5IterNext(pIdxIter);
}
}
sqlite3Fts5IterClose(pIdxIter);
fts5BufferFree(&buf);
*pCksum = cksum;
................................................................................
**
** Return SQLITE_CORRUPT if any of the internal checks fail, or if the
** checksum does not match. Return SQLITE_OK if all checks pass without
** error, or some other SQLite error code if another error (e.g. OOM)
** occurs.
*/
int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){
u64 cksum2 = 0; /* Checksum based on contents of indexes */
Fts5Buffer poslist = {0,0,0}; /* Buffer used to hold a poslist */
Fts5IndexIter *pIter; /* Used to iterate through entire index */
Fts5Structure *pStruct; /* Index structure */
#ifdef SQLITE_DEBUG
/* Used by extra internal tests only run if NDEBUG is not defined */
................................................................................
int iOff = 0; /* Offset within poslist */
i64 iRowid = fts5MultiIterRowid(pIter);
char *z = (char*)fts5MultiIterTerm(pIter, &n);
/* If this is a new term, query for it. Update cksum3 with the results. */
fts5TestTerm(p, &term, z, n, cksum2, &cksum3);
poslist.n = 0;
fts5SegiterPoslist(p, &pIter->aSeg[pIter->aFirst[1].iFirst] , 0, &poslist);
while( 0==sqlite3Fts5PoslistNext64(poslist.p, poslist.n, &iOff, &iPos) ){
int iCol = FTS5_POS2COLUMN(iPos);
int iTokOff = FTS5_POS2OFFSET(iPos);
cksum2 ^= sqlite3Fts5IndexEntryCksum(iRowid, iCol, iTokOff, -1, z, n);
}
}
fts5TestTerm(p, &term, 0, 0, cksum2, &cksum3);
fts5MultiIterFree(p, pIter);
if( p->rc==SQLITE_OK && cksum!=cksum2 ) p->rc = FTS5_CORRUPT;
|
> > > > > | | | | | | | | | | | > > > > > > > | | | | | | > |
5195 5196 5197 5198 5199 5200 5201 5202 5203 5204 5205 5206 5207 5208 5209 5210 5211 5212 5213 5214 5215 5216 5217 5218 5219 5220 5221 5222 5223 5224 5225 5226 5227 5228 5229 5230 5231 5232 5233 5234 .... 5527 5528 5529 5530 5531 5532 5533 5534 5535 5536 5537 5538 5539 5540 5541 .... 5579 5580 5581 5582 5583 5584 5585 5586 5587 5588 5589 5590 5591 5592 5593 5594 5595 5596 5597 5598 5599 5600 5601 5602 |
Fts5Index *p, /* Fts5 index object */
int iIdx,
const char *z, /* Index key to query for */
int n, /* Size of index key in bytes */
int flags, /* Flags for Fts5IndexQuery */
u64 *pCksum /* IN/OUT: Checksum value */
){
int eDetail = p->pConfig->eDetail;
u64 cksum = *pCksum;
Fts5IndexIter *pIdxIter = 0;
Fts5Buffer buf = {0, 0, 0};
int rc = sqlite3Fts5IndexQuery(p, z, n, flags, 0, &pIdxIter);
while( rc==SQLITE_OK && 0==sqlite3Fts5IterEof(pIdxIter) ){
i64 rowid = sqlite3Fts5IterRowid(pIdxIter);
if( eDetail==FTS5_DETAIL_NONE ){
cksum ^= sqlite3Fts5IndexEntryCksum(rowid, 0, 0, iIdx, z, n);
}else{
rc = sqlite3Fts5IterPoslistBuffer(pIdxIter, &buf);
if( rc==SQLITE_OK ){
Fts5PoslistReader sReader;
for(sqlite3Fts5PoslistReaderInit(buf.p, buf.n, &sReader);
sReader.bEof==0;
sqlite3Fts5PoslistReaderNext(&sReader)
){
int iCol = FTS5_POS2COLUMN(sReader.iPos);
int iOff = FTS5_POS2OFFSET(sReader.iPos);
cksum ^= sqlite3Fts5IndexEntryCksum(rowid, iCol, iOff, iIdx, z, n);
}
}
}
if( rc==SQLITE_OK ){
rc = sqlite3Fts5IterNext(pIdxIter);
}
}
sqlite3Fts5IterClose(pIdxIter);
fts5BufferFree(&buf);
*pCksum = cksum;
................................................................................
**
** Return SQLITE_CORRUPT if any of the internal checks fail, or if the
** checksum does not match. Return SQLITE_OK if all checks pass without
** error, or some other SQLite error code if another error (e.g. OOM)
** occurs.
*/
int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){
int eDetail = p->pConfig->eDetail;
u64 cksum2 = 0; /* Checksum based on contents of indexes */
Fts5Buffer poslist = {0,0,0}; /* Buffer used to hold a poslist */
Fts5IndexIter *pIter; /* Used to iterate through entire index */
Fts5Structure *pStruct; /* Index structure */
#ifdef SQLITE_DEBUG
/* Used by extra internal tests only run if NDEBUG is not defined */
................................................................................
int iOff = 0; /* Offset within poslist */
i64 iRowid = fts5MultiIterRowid(pIter);
char *z = (char*)fts5MultiIterTerm(pIter, &n);
/* If this is a new term, query for it. Update cksum3 with the results. */
fts5TestTerm(p, &term, z, n, cksum2, &cksum3);
if( eDetail==FTS5_DETAIL_NONE ){
cksum2 ^= sqlite3Fts5IndexEntryCksum(iRowid, 0, 0, -1, z, n);
}else{
poslist.n = 0;
fts5SegiterPoslist(p, &pIter->aSeg[pIter->aFirst[1].iFirst], 0, &poslist);
while( 0==sqlite3Fts5PoslistNext64(poslist.p, poslist.n, &iOff, &iPos) ){
int iCol = FTS5_POS2COLUMN(iPos);
int iTokOff = FTS5_POS2OFFSET(iPos);
cksum2 ^= sqlite3Fts5IndexEntryCksum(iRowid, iCol, iTokOff, -1, z, n);
}
}
}
fts5TestTerm(p, &term, 0, 0, cksum2, &cksum3);
fts5MultiIterFree(p, pIter);
if( p->rc==SQLITE_OK && cksum!=cksum2 ) p->rc = FTS5_CORRUPT;
|
Changes to ext/fts5/fts5_storage.c.
849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 ... 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 ... 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 |
int iPos;
int iCol;
if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){
pCtx->szCol++;
}
iPos = pTermset ? pCtx->iCol : pCtx->szCol-1;
iCol = pTermset ? 0 : pCtx->iCol;
rc = sqlite3Fts5TermsetAdd(pTermset, 0, pToken, nToken, &bPresent);
if( rc==SQLITE_OK && bPresent==0 ){
pCtx->cksum ^= sqlite3Fts5IndexEntryCksum(
pCtx->iRowid, iCol, iPos, 0, pToken, nToken
);
}
................................................................................
int rc2;
while( SQLITE_ROW==sqlite3_step(pScan) ){
int i;
ctx.iRowid = sqlite3_column_int64(pScan, 0);
ctx.szCol = 0;
if( pConfig->bColumnsize ){
rc = sqlite3Fts5StorageDocsize(p, ctx.iRowid, aColSize);
}
for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){
if( pConfig->abUnindexed[i] ) continue;
ctx.iCol = i;
ctx.szCol = 0;
if( pConfig->eDetail!=FTS5_DETAIL_FULL ){
rc = sqlite3Fts5TermsetNew(&ctx.pTermset);
}
if( rc==SQLITE_OK ){
rc = sqlite3Fts5Tokenize(pConfig,
FTS5_TOKENIZE_DOCUMENT,
(const char*)sqlite3_column_text(pScan, i+1),
sqlite3_column_bytes(pScan, i+1),
................................................................................
fts5StorageIntegrityCallback
);
}
if( rc==SQLITE_OK && pConfig->bColumnsize && ctx.szCol!=aColSize[i] ){
rc = FTS5_CORRUPT;
}
aTotalSize[i] += ctx.szCol;
sqlite3Fts5TermsetFree(ctx.pTermset);
ctx.pTermset = 0;
}
if( rc!=SQLITE_OK ) break;
}
rc2 = sqlite3_reset(pScan);
if( rc==SQLITE_OK ) rc = rc2;
}
/* Test that the "totals" (sometimes called "averages") record looks Ok */
|
> > | | > > > > > > > > > > > > > > > > | > | | | > > > > |
849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 ... 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 ... 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 |
int iPos;
int iCol;
if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){
pCtx->szCol++;
}
switch( pCtx->pConfig->eDetail ){
case FTS5_DETAIL_FULL:
iPos = pCtx->szCol-1;
iCol = pCtx->iCol;
break;
case FTS5_DETAIL_COLUMNS:
iPos = pCtx->iCol;
iCol = 0;
break;
default:
assert( pCtx->pConfig->eDetail==FTS5_DETAIL_NONE );
iPos = 0;
iCol = 0;
break;
}
rc = sqlite3Fts5TermsetAdd(pTermset, 0, pToken, nToken, &bPresent);
if( rc==SQLITE_OK && bPresent==0 ){
pCtx->cksum ^= sqlite3Fts5IndexEntryCksum(
pCtx->iRowid, iCol, iPos, 0, pToken, nToken
);
}
................................................................................
int rc2;
while( SQLITE_ROW==sqlite3_step(pScan) ){
int i;
ctx.iRowid = sqlite3_column_int64(pScan, 0);
ctx.szCol = 0;
if( pConfig->bColumnsize ){
rc = sqlite3Fts5StorageDocsize(p, ctx.iRowid, aColSize);
}
if( rc==SQLITE_OK && pConfig->eDetail==FTS5_DETAIL_NONE ){
rc = sqlite3Fts5TermsetNew(&ctx.pTermset);
}
for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){
if( pConfig->abUnindexed[i] ) continue;
ctx.iCol = i;
ctx.szCol = 0;
if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
rc = sqlite3Fts5TermsetNew(&ctx.pTermset);
}
if( rc==SQLITE_OK ){
rc = sqlite3Fts5Tokenize(pConfig,
FTS5_TOKENIZE_DOCUMENT,
(const char*)sqlite3_column_text(pScan, i+1),
sqlite3_column_bytes(pScan, i+1),
................................................................................
fts5StorageIntegrityCallback
);
}
if( rc==SQLITE_OK && pConfig->bColumnsize && ctx.szCol!=aColSize[i] ){
rc = FTS5_CORRUPT;
}
aTotalSize[i] += ctx.szCol;
if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
sqlite3Fts5TermsetFree(ctx.pTermset);
ctx.pTermset = 0;
}
}
sqlite3Fts5TermsetFree(ctx.pTermset);
ctx.pTermset = 0;
if( rc!=SQLITE_OK ) break;
}
rc2 = sqlite3_reset(pScan);
if( rc==SQLITE_OK ) rc = rc2;
}
/* Test that the "totals" (sometimes called "averages") record looks Ok */
|
Changes to ext/fts5/test/fts5simple2.test.
105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 |
do_execsql_test 7.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
BEGIN;
INSERT INTO t1 VALUES('a1 b1');
INSERT INTO t1 VALUES('a1 b2');
COMMIT;
}
do_execsql_test 7.0.4 { SELECT rowid FROM t1('b*') ORDER BY rowid DESC } {2 1}
do_execsql_test 7.0.5 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC } {2 1}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 7.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
INSERT INTO t1 VALUES('a1 b1 c1');
INSERT INTO t1 VALUES('a2 b2 c2');
INSERT INTO t1 VALUES('a1 b1 c1');
}
do_execsql_test 7.0.1 { SELECT rowid FROM t1('b*') } {1 2 3}
do_execsql_test 7.0.2 { SELECT rowid FROM t1('a1') } {1 3}
do_execsql_test 7.0.3 { SELECT rowid FROM t1('c2') } {2}
do_execsql_test 7.0.4 { SELECT rowid FROM t1('b*') ORDER BY rowid DESC } {3 2 1}
do_execsql_test 7.0.5 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC } {3 1}
do_execsql_test 7.0.7 { SELECT rowid FROM t1('c2') ORDER BY rowid DESC } {2}
do_execsql_test 7.1.0 { INSERT INTO t1(t1) VALUES('optimize') }
do_execsql_test 7.1.1 { SELECT rowid FROM t1('b*') } {1 2 3}
do_execsql_test 7.1.2 { SELECT rowid FROM t1('a1') } {1 3}
do_execsql_test 7.1.3 { SELECT rowid FROM t1('c2') } {2}
do_execsql_test 7.2.1 { SELECT rowid FROM t1('b*') ORDER BY rowid DESC} {3 2 1}
do_execsql_test 7.2.2 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC} {3 1}
do_execsql_test 7.2.3 { SELECT rowid FROM t1('c2') ORDER BY rowid DESC} {2}
finish_test
|
| | | > > > > > > > > > > > > > > > > > > > > > > > > > > > | | | | | | < > | > | | < | < < < < > |
105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 |
do_execsql_test 7.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
BEGIN;
INSERT INTO t1 VALUES('a1 b1');
INSERT INTO t1 VALUES('a1 b2');
COMMIT;
}
do_execsql_test 7.1 { SELECT rowid FROM t1('b*') ORDER BY rowid DESC } {2 1}
do_execsql_test 7.2 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC } {2 1}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 8.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
INSERT INTO t1 VALUES('a1 b1 c1');
INSERT INTO t1 VALUES('a2 b2 c2');
INSERT INTO t1 VALUES('a1 b1 c1');
}
do_execsql_test 8.0.1 { SELECT rowid FROM t1('b*') } {1 2 3}
do_execsql_test 8.0.2 { SELECT rowid FROM t1('a1') } {1 3}
do_execsql_test 8.0.3 { SELECT rowid FROM t1('c2') } {2}
do_execsql_test 8.0.4 { SELECT rowid FROM t1('b*') ORDER BY rowid DESC } {3 2 1}
do_execsql_test 8.0.5 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC } {3 1}
do_execsql_test 8.0.8 { SELECT rowid FROM t1('c2') ORDER BY rowid DESC } {2}
do_execsql_test 8.1.0 { INSERT INTO t1(t1) VALUES('optimize') }
do_execsql_test 8.1.1 { SELECT rowid FROM t1('b*') } {1 2 3}
do_execsql_test 8.1.2 { SELECT rowid FROM t1('a1') } {1 3}
do_execsql_test 8.1.3 { SELECT rowid FROM t1('c2') } {2}
do_execsql_test 8.2.1 { SELECT rowid FROM t1('b*') ORDER BY rowid DESC} {3 2 1}
do_execsql_test 8.2.2 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC} {3 1}
do_execsql_test 8.2.3 { SELECT rowid FROM t1('c2') ORDER BY rowid DESC} {2}
#--------------------------------------------------------------------------
#
reset_db
do_execsql_test 9.0.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
INSERT INTO t1 VALUES('a1 b1 c1');
INSERT INTO t1 VALUES('a2 b2 c2');
INSERT INTO t1 VALUES('a1 b1 c1');
}
do_execsql_test 9.0.1 {
INSERT INTO t1(t1) VALUES('integrity-check');
} {}
reset_db
do_execsql_test 9.1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=none);
INSERT INTO t1 VALUES('a1 b1 c1', 'x y z');
INSERT INTO t1 VALUES('a2 b2 c2', '1 2 3');
INSERT INTO t1 VALUES('a1 b1 c1', 'x 2 z');
}
do_execsql_test 9.2.1 {
INSERT INTO t1(t1) VALUES('integrity-check');
} {}
finish_test
|