Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Add the languageid_bits= option to fts. Still some problems to work out. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | fts-languageid-bits |
Files: | files | file ages | folders |
SHA1: |
d36d7e68334c0685d1941dd0323b1a9c |
User & Date: | dan 2013-06-19 20:13:28.631 |
Context
2013-06-20
| ||
11:01 | Fix fts handling of the case where both a rowid and docid are specified as part of an insert statement. (check-in: 610e7e9612 user: dan tags: fts-languageid-bits) | |
2013-06-19
| ||
20:13 | Add the languageid_bits= option to fts. Still some problems to work out. (check-in: d36d7e6833 user: dan tags: fts-languageid-bits) | |
14:49 | Only default HAVE_POSIX_FALLOCATE on for linux, and then only if it is not previously defined. (check-in: 2b2ade9278 user: drh tags: trunk) | |
Changes
Changes to ext/fts3/fts3.c.
︙ | ︙ | |||
1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 | int bNoDocsize = 0; /* True to omit %_docsize table */ int bDescIdx = 0; /* True to store descending indexes */ char *zPrefix = 0; /* Prefix parameter value (or NULL) */ char *zCompress = 0; /* compress=? parameter (or NULL) */ char *zUncompress = 0; /* uncompress=? parameter (or NULL) */ char *zContent = 0; /* content=? parameter (or NULL) */ char *zLanguageid = 0; /* languageid=? parameter (or NULL) */ assert( strlen(argv[0])==4 ); assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4) || (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4) ); nDb = (int)strlen(argv[1]) + 1; | > | 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 | int bNoDocsize = 0; /* True to omit %_docsize table */ int bDescIdx = 0; /* True to store descending indexes */ char *zPrefix = 0; /* Prefix parameter value (or NULL) */ char *zCompress = 0; /* compress=? parameter (or NULL) */ char *zUncompress = 0; /* uncompress=? parameter (or NULL) */ char *zContent = 0; /* content=? parameter (or NULL) */ char *zLanguageid = 0; /* languageid=? parameter (or NULL) */ char *zLanguageidBits = 0; /* languageid_bits=? parameter (or NULL) */ assert( strlen(argv[0])==4 ); assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4) || (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4) ); nDb = (int)strlen(argv[1]) + 1; |
︙ | ︙ | |||
1121 1122 1123 1124 1125 1126 1127 | /* Check if it is an FTS4 special argument. */ else if( isFts4 && fts3IsSpecialColumn(z, &nKey, &zVal) ){ struct Fts4Option { const char *zOpt; int nOpt; } aFts4Opt[] = { | | | | | | | | > | 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 | /* Check if it is an FTS4 special argument. */ else if( isFts4 && fts3IsSpecialColumn(z, &nKey, &zVal) ){ struct Fts4Option { const char *zOpt; int nOpt; } aFts4Opt[] = { { "matchinfo", 9 }, /* 0 -> MATCHINFO */ { "prefix", 6 }, /* 1 -> PREFIX */ { "compress", 8 }, /* 2 -> COMPRESS */ { "uncompress", 10 }, /* 3 -> UNCOMPRESS */ { "order", 5 }, /* 4 -> ORDER */ { "content", 7 }, /* 5 -> CONTENT */ { "languageid", 10 }, /* 6 -> LANGUAGEID */ { "languageid_bits", 15 } /* 7 -> LANGUAGEID_BITS */ }; int iOpt; if( !zVal ){ rc = SQLITE_NOMEM; }else{ for(iOpt=0; iOpt<SizeofArray(aFts4Opt); iOpt++){ |
︙ | ︙ | |||
1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 | case 6: /* LANGUAGEID */ assert( iOpt==6 ); sqlite3_free(zLanguageid); zLanguageid = zVal; zVal = 0; break; } } sqlite3_free(zVal); } } /* Otherwise, the argument is a column name. */ | > > > > > > > | 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 | case 6: /* LANGUAGEID */ assert( iOpt==6 ); sqlite3_free(zLanguageid); zLanguageid = zVal; zVal = 0; break; case 7: /* LANGUAGEID_BITS */ assert( iOpt==7 ); sqlite3_free(zLanguageidBits); zLanguageidBits = zVal; zVal = 0; break; } } sqlite3_free(zVal); } } /* Otherwise, the argument is a column name. */ |
︙ | ︙ | |||
1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 | p->bFts4 = isFts4; p->bDescIdx = bDescIdx; p->bAutoincrmerge = 0xff; /* 0xff means setting unknown */ p->zContentTbl = zContent; p->zLanguageid = zLanguageid; zContent = 0; zLanguageid = 0; TESTONLY( p->inTransaction = -1 ); TESTONLY( p->mxSavepoint = -1 ); p->aIndex = (struct Fts3Index *)&p->azColumn[nCol]; memcpy(p->aIndex, aIndex, sizeof(struct Fts3Index) * nIndex); p->nIndex = nIndex; for(i=0; i<nIndex; i++){ | > > > > > > > > > | 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 | p->bFts4 = isFts4; p->bDescIdx = bDescIdx; p->bAutoincrmerge = 0xff; /* 0xff means setting unknown */ p->zContentTbl = zContent; p->zLanguageid = zLanguageid; zContent = 0; zLanguageid = 0; if( zLanguageidBits && p->zLanguageid && p->zContentTbl==0 ){ p->nLanguageidBits = atoi(zLanguageidBits); if( p->nLanguageidBits>32 || p->nLanguageidBits<0 ){ rc = SQLITE_ERROR; *pzErr = sqlite3_mprintf("languageid_bits parameter out of range"); goto fts3_init_out; } } TESTONLY( p->inTransaction = -1 ); TESTONLY( p->mxSavepoint = -1 ); p->aIndex = (struct Fts3Index *)&p->azColumn[nCol]; memcpy(p->aIndex, aIndex, sizeof(struct Fts3Index) * nIndex); p->nIndex = nIndex; for(i=0; i<nIndex; i++){ |
︙ | ︙ | |||
1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 | fts3_init_out: sqlite3_free(zPrefix); sqlite3_free(aIndex); sqlite3_free(zCompress); sqlite3_free(zUncompress); sqlite3_free(zContent); sqlite3_free(zLanguageid); sqlite3_free((void *)aCol); if( rc!=SQLITE_OK ){ if( p ){ fts3DisconnectMethod((sqlite3_vtab *)p); }else if( pTokenizer ){ pTokenizer->pModule->xDestroy(pTokenizer); } | > | 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 | fts3_init_out: sqlite3_free(zPrefix); sqlite3_free(aIndex); sqlite3_free(zCompress); sqlite3_free(zUncompress); sqlite3_free(zContent); sqlite3_free(zLanguageid); sqlite3_free(zLanguageidBits); sqlite3_free((void *)aCol); if( rc!=SQLITE_OK ){ if( p ){ fts3DisconnectMethod((sqlite3_vtab *)p); }else if( pTokenizer ){ pTokenizer->pModule->xDestroy(pTokenizer); } |
︙ | ︙ | |||
3058 3059 3060 3061 3062 3063 3064 | Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; Fts3Table *p = (Fts3Table *)pCursor->pVtab; /* The column value supplied by SQLite must be in range. */ assert( iCol>=0 && iCol<=p->nColumn+2 ); if( iCol==p->nColumn+1 ){ | | | | | | 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 | Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; Fts3Table *p = (Fts3Table *)pCursor->pVtab; /* The column value supplied by SQLite must be in range. */ assert( iCol>=0 && iCol<=p->nColumn+2 ); if( iCol==p->nColumn+1 ){ /* This call is a request for the "docid" column. The value currently ** stored in pCsr->iPrevId is a rowid. Transform this to a docid and ** return it. */ sqlite3_result_int64(pCtx, sqlite3Fts3RowidToDocid(p, pCsr->iPrevId)); }else if( iCol==p->nColumn ){ /* The extra column whose name is the same as the table. ** Return a blob which is a pointer to the cursor. */ sqlite3_result_blob(pCtx, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT); }else if( iCol==p->nColumn+2 && pCsr->pExpr ){ sqlite3_result_int64(pCtx, pCsr->iLangid); }else{ |
︙ | ︙ |
Changes to ext/fts3/fts3Int.h.
︙ | ︙ | |||
205 206 207 208 209 210 211 212 213 214 215 216 217 218 | const char *zDb; /* logical database name */ const char *zName; /* virtual table name */ int nColumn; /* number of named columns in virtual table */ char **azColumn; /* column names. malloced */ sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ char *zContentTbl; /* content=xxx option, or NULL */ char *zLanguageid; /* languageid=xxx option, or NULL */ u8 bAutoincrmerge; /* True if automerge=1 */ u32 nLeafAdd; /* Number of leaf blocks added this trans */ /* Precompiled statements used by the implementation. Each of these ** statements is run and reset within a single virtual table API call. */ sqlite3_stmt *aStmt[37]; | > | 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 | const char *zDb; /* logical database name */ const char *zName; /* virtual table name */ int nColumn; /* number of named columns in virtual table */ char **azColumn; /* column names. malloced */ sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ char *zContentTbl; /* content=xxx option, or NULL */ char *zLanguageid; /* languageid=xxx option, or NULL */ int nLanguageidBits; /* languageid_bits=N option, or 0 */ u8 bAutoincrmerge; /* True if automerge=1 */ u32 nLeafAdd; /* Number of leaf blocks added this trans */ /* Precompiled statements used by the implementation. Each of these ** statements is run and reset within a single virtual table API call. */ sqlite3_stmt *aStmt[37]; |
︙ | ︙ | |||
417 418 419 420 421 422 423 | */ #define FTSQUERY_NEAR 1 #define FTSQUERY_NOT 2 #define FTSQUERY_AND 3 #define FTSQUERY_OR 4 #define FTSQUERY_PHRASE 5 | < > > | 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 | */ #define FTSQUERY_NEAR 1 #define FTSQUERY_NOT 2 #define FTSQUERY_AND 3 #define FTSQUERY_OR 4 #define FTSQUERY_PHRASE 5 /* fts3_write.c */ i64 sqlite3Fts3DocidToRowid(Fts3Table *p, i64 iDocid, int iLangid); i64 sqlite3Fts3RowidToDocid(Fts3Table *p, i64 iRowid); int sqlite3Fts3UpdateMethod(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*); int sqlite3Fts3PendingTermsFlush(Fts3Table *); void sqlite3Fts3PendingTermsClear(Fts3Table *); int sqlite3Fts3Optimize(Fts3Table *); int sqlite3Fts3SegReaderNew(int, int, sqlite3_int64, sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**); int sqlite3Fts3SegReaderPending( |
︙ | ︙ |
Changes to ext/fts3/fts3_write.c.
︙ | ︙ | |||
483 484 485 486 487 488 489 490 491 492 493 494 495 496 | if( rc==SQLITE_OK ){ sqlite3_step(pStmt); rc = sqlite3_reset(pStmt); } *pRC = rc; } /* ** This function ensures that the caller has obtained an exclusive ** shared-cache table-lock on the %_segdir table. This is required before ** writing data to the fts3 table. If this lock is not acquired first, then ** the caller may end up attempting to take this lock as part of committing ** a transaction, causing SQLite to return SQLITE_LOCKED or | > > > > > > > > > > > > > > > > > > | 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 | if( rc==SQLITE_OK ){ sqlite3_step(pStmt); rc = sqlite3_reset(pStmt); } *pRC = rc; } static void fts3SqlExecI64( int *pRC, /* Result code */ Fts3Table *p, /* The FTS3 table */ int eStmt, /* Index of statement to evaluate */ i64 iVal ){ sqlite3_stmt *pStmt; int rc; if( *pRC ) return; rc = fts3SqlStmt(p, eStmt, &pStmt, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int64(pStmt, 1, iVal); sqlite3_step(pStmt); rc = sqlite3_reset(pStmt); } *pRC = rc; } /* ** This function ensures that the caller has obtained an exclusive ** shared-cache table-lock on the %_segdir table. This is required before ** writing data to the fts3 table. If this lock is not acquired first, then ** the caller may end up attempting to take this lock as part of committing ** a transaction, causing SQLite to return SQLITE_LOCKED or |
︙ | ︙ | |||
923 924 925 926 927 928 929 | ** apVal[p->nColumn+2] Hidden column with same name as table ** apVal[p->nColumn+3] Hidden "docid" column (alias for rowid) ** apVal[p->nColumn+4] Hidden languageid column */ static int fts3InsertData( Fts3Table *p, /* Full-text table */ sqlite3_value **apVal, /* Array of values to insert */ | | > | > > | | > > > > | | | | > > > > > | > | > | > | 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 | ** apVal[p->nColumn+2] Hidden column with same name as table ** apVal[p->nColumn+3] Hidden "docid" column (alias for rowid) ** apVal[p->nColumn+4] Hidden languageid column */ static int fts3InsertData( Fts3Table *p, /* Full-text table */ sqlite3_value **apVal, /* Array of values to insert */ sqlite3_int64 *piRowid, /* OUT: Rowid for row just inserted */ i64 iRowid /* Explicit rowid, if piRowid==NULL */ ){ int rc; /* Return code */ sqlite3_stmt *pContentInsert; /* INSERT INTO %_content VALUES(...) */ if( p->zContentTbl ){ sqlite3_value *pRowid; assert( p->nLanguageidBits==0 && piRowid ); pRowid = apVal[p->nColumn+3]; if( sqlite3_value_type(pRowid)==SQLITE_NULL ){ pRowid = apVal[1]; } if( sqlite3_value_type(pRowid)!=SQLITE_INTEGER ){ return SQLITE_CONSTRAINT; } *piRowid = sqlite3_value_int64(pRowid); return SQLITE_OK; } /* Locate the statement handle used to insert data into the %_content ** table. The SQL for this statement is: ** ** INSERT INTO %_content VALUES(?, ?, ?, ...) ** ** The statement features N '?' variables, where N is the number of user ** defined columns in the FTS3 table, plus one for the docid field. */ rc = fts3SqlStmt(p, SQL_CONTENT_INSERT, &pContentInsert, &apVal[1]); if( rc==SQLITE_OK ){ if( piRowid==0 ){ sqlite3_bind_int64(pContentInsert, 1, iRowid); } if( p->zLanguageid ){ rc = sqlite3_bind_int( pContentInsert, p->nColumn+2, sqlite3_value_int(apVal[p->nColumn+4]) ); } } if( rc!=SQLITE_OK ) return rc; /* There is a quirk here. The users INSERT statement may have specified ** a value for the "rowid" field, for the "docid" field, or for both. ** Which is a problem, since "rowid" and "docid" are aliases for the ** same value. For example: ** ** INSERT INTO fts3tbl(rowid, docid) VALUES(1, 2); ** ** In FTS3, this is an error. It is an error to specify non-NULL values ** for both docid and some other rowid alias. */ assert( p->nLanguageidBits==0 || piRowid==0 || sqlite3_value_type(apVal[1])!=SQLITE_NULL ); if( piRowid && p->nLanguageidBits==0 && SQLITE_NULL!=sqlite3_value_type(apVal[3+p->nColumn]) ){ if( SQLITE_NULL==sqlite3_value_type(apVal[0]) && SQLITE_NULL!=sqlite3_value_type(apVal[1]) ){ /* A rowid/docid conflict. */ return SQLITE_ERROR; } rc = sqlite3_bind_value(pContentInsert, 1, apVal[3+p->nColumn]); if( rc!=SQLITE_OK ) return rc; } /* Execute the statement to insert the record. Set *pRowid to the ** new docid value. */ sqlite3_step(pContentInsert); rc = sqlite3_reset(pContentInsert); if( piRowid ){ *piRowid = sqlite3_last_insert_rowid(p->db); } return rc; } /* ** Remove all data from the FTS3 table. Clear the hash table containing |
︙ | ︙ | |||
1032 1033 1034 1035 1036 1037 1038 | ** The first element in the apVal[] array is assumed to contain the docid ** (an integer) of a row about to be deleted. Remove all terms from the ** full-text index. */ static void fts3DeleteTerms( int *pRC, /* Result code */ Fts3Table *p, /* The FTS table to delete from */ | | | > | 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 | ** The first element in the apVal[] array is assumed to contain the docid ** (an integer) of a row about to be deleted. Remove all terms from the ** full-text index. */ static void fts3DeleteTerms( int *pRC, /* Result code */ Fts3Table *p, /* The FTS table to delete from */ i64 iRowid, /* The rowid to be deleted */ u32 *aSz, /* Sizes of deleted document written here */ int *pbFound /* OUT: Set to true if row really does exist */ ){ int rc; sqlite3_stmt *pSelect; assert( *pbFound==0 ); if( *pRC ) return; rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pSelect, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int64(pSelect, 1, iRowid); if( SQLITE_ROW==sqlite3_step(pSelect) ){ int i; int iLangid = langidFromSelect(p, pSelect); rc = fts3PendingTermsDocid(p, iLangid, sqlite3_column_int64(pSelect, 0)); for(i=1; rc==SQLITE_OK && i<=p->nColumn; i++){ const char *zText = (const char *)sqlite3_column_text(pSelect, i); rc = fts3PendingTermsAdd(p, iLangid, zText, -1, &aSz[i-1]); |
︙ | ︙ | |||
2342 2343 2344 2345 2346 2347 2348 | ** are different from that integer. i.e. if deleting the document with docid ** pRowid would mean the FTS3 table were empty. ** ** If successful, *pisEmpty is set to true if the table is empty except for ** document pRowid, or false otherwise, and SQLITE_OK is returned. If an ** error occurs, an SQLite error code is returned. */ | | | > | 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 | ** are different from that integer. i.e. if deleting the document with docid ** pRowid would mean the FTS3 table were empty. ** ** If successful, *pisEmpty is set to true if the table is empty except for ** document pRowid, or false otherwise, and SQLITE_OK is returned. If an ** error occurs, an SQLite error code is returned. */ static int fts3IsEmpty(Fts3Table *p, i64 iRowid, int *pisEmpty){ sqlite3_stmt *pStmt; int rc; if( p->zContentTbl ){ /* If using the content=xxx option, assume the table is never empty */ *pisEmpty = 0; rc = SQLITE_OK; }else{ rc = fts3SqlStmt(p, SQL_IS_EMPTY, &pStmt, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int64(pStmt, 1, iRowid); if( SQLITE_ROW==sqlite3_step(pStmt) ){ *pisEmpty = sqlite3_column_int(pStmt, 0); } rc = sqlite3_reset(pStmt); } } return rc; |
︙ | ︙ | |||
5193 5194 5195 5196 5197 5198 5199 | /* ** SQLite value pRowid contains the rowid of a row that may or may not be ** present in the FTS3 table. If it is, delete it and adjust the contents ** of subsiduary data structures accordingly. */ static int fts3DeleteByRowid( Fts3Table *p, | | | | | | > > > > > > > > > > > > > > > > > < > > > > > > > > > > | > | | > > > > > > > > > | 5228 5229 5230 5231 5232 5233 5234 5235 5236 5237 5238 5239 5240 5241 5242 5243 5244 5245 5246 5247 5248 5249 5250 5251 5252 5253 5254 5255 5256 5257 5258 5259 5260 5261 5262 5263 5264 5265 5266 5267 5268 5269 5270 5271 5272 5273 5274 5275 5276 5277 5278 5279 5280 5281 5282 5283 5284 5285 5286 5287 5288 5289 5290 5291 5292 5293 5294 5295 5296 5297 5298 5299 5300 5301 5302 5303 5304 5305 5306 5307 5308 5309 5310 5311 5312 5313 5314 5315 5316 5317 5318 5319 5320 5321 5322 5323 5324 5325 5326 5327 5328 5329 5330 5331 5332 5333 5334 5335 5336 5337 5338 5339 5340 5341 5342 5343 5344 5345 5346 5347 5348 5349 5350 5351 5352 5353 5354 5355 5356 5357 5358 5359 | /* ** SQLite value pRowid contains the rowid of a row that may or may not be ** present in the FTS3 table. If it is, delete it and adjust the contents ** of subsiduary data structures accordingly. */ static int fts3DeleteByRowid( Fts3Table *p, i64 iRowid, int *pnChng, /* IN/OUT: Decrement if row is deleted */ u32 *aSzDel ){ int rc = SQLITE_OK; /* Return code */ int bFound = 0; /* True if *pRowid really is in the table */ fts3DeleteTerms(&rc, p, iRowid, aSzDel, &bFound); if( bFound && rc==SQLITE_OK ){ int isEmpty = 0; /* Deleting *pRowid leaves the table empty */ rc = fts3IsEmpty(p, iRowid, &isEmpty); if( rc==SQLITE_OK ){ if( isEmpty ){ /* Deleting this row means the whole table is empty. In this case ** delete the contents of all three tables and throw away any ** data in the pendingTerms hash table. */ rc = fts3DeleteAll(p, 1); *pnChng = 0; memset(aSzDel, 0, sizeof(u32) * (p->nColumn+1) * 2); }else{ *pnChng = *pnChng - 1; if( p->zContentTbl==0 ){ fts3SqlExecI64(&rc, p, SQL_DELETE_CONTENT, iRowid); } if( p->bHasDocsize ){ fts3SqlExecI64(&rc, p, SQL_DELETE_DOCSIZE, iRowid); } } } } return rc; } /* ** Convert a docid (iDocid) and a language id (iLangid) to a rowid, ** according to the configured languageid_bits= value belonging to ** FTS table *p. */ i64 sqlite3Fts3DocidToRowid(Fts3Table *p, i64 iDocid, int iLangid){ i64 iRowid = iDocid; if( p->nLanguageidBits ){ iRowid = (iRowid << p->nLanguageidBits) + iLangid; } return iRowid; } i64 sqlite3Fts3RowidToDocid(Fts3Table *p, i64 iRowid){ return (iRowid >> p->nLanguageidBits); } /* ** This function does the work for the xUpdate method of FTS3 virtual ** tables. The schema of the virtual table being: ** ** CREATE TABLE <table name>( ** <user columns>, ** <table name> HIDDEN, ** docid HIDDEN, ** <langid> HIDDEN ** ); ** */ int sqlite3Fts3UpdateMethod( sqlite3_vtab *pVtab, /* FTS3 vtab object */ int nArg, /* Size of argument array */ sqlite3_value **apVal, /* Array of arguments */ sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */ ){ Fts3Table *p = (Fts3Table *)pVtab; int rc = SQLITE_OK; /* Return Code */ int isRemove = 0; /* True for an UPDATE or DELETE */ u32 *aSzIns = 0; /* Sizes of inserted documents */ u32 *aSzDel = 0; /* Sizes of deleted documents */ int nChng = 0; /* Net change in number of documents */ int bInsertDone = 0; int iLangid = 0; assert( p->pSegments==0 ); assert( nArg==1 /* DELETE operations */ || nArg==(2 + p->nColumn + 3) /* INSERT or UPDATE operations */ ); /* Check for a "special" INSERT operation. One of the form: ** ** INSERT INTO xyz(xyz) VALUES('command'); */ if( nArg>1 && sqlite3_value_type(apVal[0])==SQLITE_NULL && sqlite3_value_type(apVal[p->nColumn+2])!=SQLITE_NULL ){ rc = fts3SpecialInsert(p, apVal[p->nColumn+2]); goto update_out; } /* If this is an INSERT or UPDATE, check that the new value for the ** languageid is within range. A languageid can never be a negative ** value. If the languageid_bits option was specified when this table ** was created, it must also be less than (2 ^ nLanguageidBits). ** ** Also check that if a non-zero languageid_bits value was configured, ** the specified rowid value must be NULL. */ if( nArg>1 ){ iLangid = sqlite3_value_int(apVal[2 + p->nColumn + 2]); if( iLangid<0 || (p->nLanguageidBits && iLangid>=(1<<p->nLanguageidBits)) ){ rc = SQLITE_CONSTRAINT; goto update_out; } if( p->nLanguageidBits && sqlite3_value_type(apVal[0])==SQLITE_NULL && sqlite3_value_type(apVal[1])!=SQLITE_NULL ){ rc = SQLITE_CONSTRAINT; goto update_out; } } /* Allocate space to hold the change in document sizes */ aSzDel = sqlite3_malloc( sizeof(aSzDel[0])*(p->nColumn+1)*2 ); if( aSzDel==0 ){ rc = SQLITE_NOMEM; goto update_out; |
︙ | ︙ | |||
5300 5301 5302 5303 5304 5305 5306 | ** should be deleted from the database before inserting the new row. Or, ** if the on-conflict mode is other than REPLACE, then this method must ** detect the conflict and return SQLITE_CONSTRAINT before beginning to ** modify the database file. */ if( nArg>1 && p->zContentTbl==0 ){ /* Find the value object that holds the new rowid value. */ | | | > > > > | | > > > > > > > > > > > > > > > > > > > > > > > > | | | | | | | | | | | | | | | | | | | | | | | | | > > | < | | 5371 5372 5373 5374 5375 5376 5377 5378 5379 5380 5381 5382 5383 5384 5385 5386 5387 5388 5389 5390 5391 5392 5393 5394 5395 5396 5397 5398 5399 5400 5401 5402 5403 5404 5405 5406 5407 5408 5409 5410 5411 5412 5413 5414 5415 5416 5417 5418 5419 5420 5421 5422 5423 5424 5425 5426 5427 5428 5429 5430 5431 5432 5433 5434 5435 5436 5437 5438 5439 5440 5441 5442 5443 5444 5445 5446 5447 5448 5449 5450 5451 5452 5453 5454 5455 5456 5457 5458 5459 5460 5461 5462 5463 | ** should be deleted from the database before inserting the new row. Or, ** if the on-conflict mode is other than REPLACE, then this method must ** detect the conflict and return SQLITE_CONSTRAINT before beginning to ** modify the database file. */ if( nArg>1 && p->zContentTbl==0 ){ /* Find the value object that holds the new rowid value. */ sqlite3_value *pNewDocid = apVal[3+p->nColumn]; if( sqlite3_value_type(pNewDocid)==SQLITE_NULL ){ if( p->nLanguageidBits ){ rc = SQLITE_CONSTRAINT; goto update_out; } pNewDocid = apVal[1]; } if( sqlite3_value_type(pNewDocid)!=SQLITE_NULL ){ int e = sqlite3_value_numeric_type(pNewDocid); i64 iRowid = sqlite3_value_int64(pNewDocid); /* Check that the value specified by the user may be losslessly ** converted to an integer. If not, return a "data mismatch" error. */ if( (e!=SQLITE_INTEGER) && (e!=SQLITE_FLOAT || (double)iRowid!=sqlite3_value_double(pNewDocid)) ){ rc = SQLITE_MISMATCH; goto update_out; } if( p->nLanguageidBits ){ /* Check for an out-of-range docid value. */ if( iRowid>=((i64)1 << (63 - p->nLanguageidBits)) || iRowid<-1*((i64)1 << (63 - p->nLanguageidBits)) ){ rc = SQLITE_CONSTRAINT; goto update_out; } iRowid = sqlite3Fts3DocidToRowid(p, iRowid, iLangid); } if( sqlite3_value_type(apVal[0])==SQLITE_NULL || sqlite3_value_int64(apVal[0])!=iRowid ){ /* The new rowid is not NULL (in this case the rowid will be ** automatically assigned and there is no chance of a conflict), and ** the statement is either an INSERT or an UPDATE that modifies the ** rowid column. So if the conflict mode is REPLACE, then delete any ** existing row with rowid=pNewRowid. ** ** Or, if the conflict mode is not REPLACE, insert the new record into ** the %_content table. If we hit the duplicate rowid constraint (or ** any other error) while doing so, return immediately. ** ** This branch may also run if pNewRowid contains a value that cannot ** be losslessly converted to an integer. In this case, the eventual ** call to fts3InsertData() (either just below or further on in this ** function) will return SQLITE_MISMATCH. If fts3DeleteByRowid is ** invoked, it will delete zero rows (since no row will have ** docid=$pNewRowid if $pNewRowid is not an integer value). */ if( sqlite3_vtab_on_conflict(p->db)==SQLITE_REPLACE ){ rc = fts3DeleteByRowid(p, iRowid, &nChng, aSzDel); }else{ rc = fts3InsertData(p, apVal, 0, iRowid); bInsertDone = 1; *pRowid = iRowid; } } } } if( rc!=SQLITE_OK ){ goto update_out; } /* If this is a DELETE or UPDATE operation, remove the old record. */ if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ assert( sqlite3_value_type(apVal[0])==SQLITE_INTEGER ); rc = fts3DeleteByRowid(p, sqlite3_value_int64(apVal[0]), &nChng, aSzDel); isRemove = 1; } /* If this is an INSERT or UPDATE operation, insert the new record. */ if( nArg>1 && rc==SQLITE_OK ){ if( bInsertDone==0 ){ rc = fts3InsertData(p, apVal, pRowid, 0); if( rc==SQLITE_CONSTRAINT && p->zContentTbl==0 ){ rc = FTS_CORRUPT_VTAB; } } if( rc==SQLITE_OK && (!isRemove || *pRowid!=p->iPrevDocid ) ){ rc = fts3PendingTermsDocid(p, iLangid, *pRowid); } |
︙ | ︙ |
Added test/fts4langid2.test.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 | # 2012 March 01 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #************************************************************************* # This file implements regression tests for SQLite library. The # focus of this script is testing the languageid=xxx FTS4 option. # set testdir [file dirname $argv0] source $testdir/tester.tcl set ::testprefix fts4langid2 # If SQLITE_ENABLE_FTS3 is defined, omit this file. ifcapable !fts3 { finish_test return } #------------------------------------------------------------------------- # Test out-of-range values for the languageid_bits= parameter. # do_catchsql_test 1.1 { CREATE VIRTUAL TABLE t1 USING fts4(languageid=lid, languageid_bits=33); } {1 {languageid_bits parameter out of range}} do_catchsql_test 1.2 { CREATE VIRTUAL TABLE t1 USING fts4(languageid=lid, languageid_bits=-1); } {1 {languageid_bits parameter out of range}} do_catchsql_test 1.3 { CREATE VIRTUAL TABLE t1 USING fts4(languageid=lid, languageid_bits=0); CREATE VIRTUAL TABLE t2 USING fts4(languageid=lid, languageid_bits=32); } {0 {}} do_execsql_test 1.4 { DROP TABLE t1; DROP TABLE t2; } #------------------------------------------------------------------------- # Test out-of-range values in the languageid column. # do_execsql_test 2.1 { CREATE VIRTUAL TABLE t1 USING fts4(languageid=lid, languageid_bits=8); CREATE VIRTUAL TABLE t2 USING fts4(languageid=lid, languageid_bits=7); } do_catchsql_test 2.2 { INSERT INTO t1(docid, lid, content) VALUES(1, 256, 'abc def'); } {1 {constraint failed}} do_catchsql_test 2.3 { INSERT INTO t2(docid, lid, content) VALUES(1, 128, 'abc def'); } {1 {constraint failed}} do_catchsql_test 2.3 { INSERT INTO t1(docid, lid, content) VALUES(1, -1, 'abc def'); } {1 {constraint failed}} do_execsql_test 2.4 { DROP TABLE t1; DROP TABLE t2; } #------------------------------------------------------------------------- # Test that if languageid_bits is set to a non-zero value it is # not possible to specify a non-NULL rowid, even if it is the same # as the docid. # do_execsql_test 3.1 { CREATE VIRTUAL TABLE t1 USING fts4(languageid=lid, languageid_bits=4); CREATE VIRTUAL TABLE t2 USING fts4(languageid=lid, languageid_bits=0); } do_catchsql_test 3.2.1 { INSERT INTO t1(rowid, lid, content) VALUES(1, 0, 'abc def'); } {1 {constraint failed}} do_catchsql_test 3.2.2 { INSERT INTO t2(rowid, lid, content) VALUES(1, 0, 'abc def'); } {0 {}} do_catchsql_test 3.3 { INSERT INTO t1(rowid, docid, lid, content) VALUES(2, 2, 0, 'abc def'); } {1 {constraint failed}} do_catchsql_test 3.4 { INSERT INTO t1(lid, content) VALUES(0, 'one two def'); } {1 {constraint failed}} do_execsql_test 3.4 { DROP TABLE t1; DROP TABLE t2; } #------------------------------------------------------------------------- # do_execsql_test 4.1 { CREATE VIRTUAL TABLE t1 USING fts4(languageid=lid, languageid_bits=5); } do_execsql_test 4.2 { INSERT INTO t1 (docid, lid, content) VALUES(1, 0, '1 2 3'); INSERT INTO t1 (docid, lid, content) VALUES(1, 1, '1 2 3 4'); } do_execsql_test 4.3 { SELECT docid, lid FROM t1; } {1 0 1 1} do_execsql_test 4.4 { SELECT docid, lid, content FROM t1 WHERE t1 MATCH '2'; } {1 0 {1 2 3}} do_execsql_test 4.5 { SELECT docid, lid, content FROM t1 WHERE t1 MATCH '2' AND lid=1; } {1 1 {1 2 3 4}} breakpoint do_execsql_test 4.6 { UPDATE t1 SET content = 'x y z' || lid; SELECT docid, lid FROM t1; } {1 0 1 1} finish_test |