Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Allow RBU tables to be named "data[0-9]*_<target>" instead of strictly "data_<target>". Also update RBU so that it always processes data tables in order sorted by name. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | trunk |
Files: | files | file ages | folders |
SHA1: |
287aa30601506f168d355c35176a3383 |
User & Date: | dan 2015-07-30 11:38:19.207 |
Context
2015-07-30
| ||
20:26 | Add the --rbu switch to the "sqldiff" utility. (check-in: 098bea26da user: dan tags: trunk) | |
11:38 | Allow RBU tables to be named "data[0-9]*_<target>" instead of strictly "data_<target>". Also update RBU so that it always processes data tables in order sorted by name. (check-in: 287aa30601 user: dan tags: trunk) | |
11:26 | Return an error message (instead of segfaulting) if test function fts5_expr() is invoked with no arguments. (check-in: 8e818b8985 user: dan tags: trunk) | |
Changes
Added ext/rbu/rbufts.test.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 | # 2014 August 30 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #*********************************************************************** # # This file contains tests for the RBU module. More specifically, it # contains tests to ensure that RBU works with FTS tables. # if {![info exists testdir]} { set testdir [file join [file dirname [info script]] .. .. test] } source $testdir/tester.tcl set ::testprefix rbufts ifcapable !fts3 { finish_test return } proc step_rbu {target rbu} { while 1 { sqlite3rbu rbu $target $rbu set rc [rbu step] rbu close if {$rc != "SQLITE_OK"} break } set rc } do_execsql_test 1.0 { CREATE TABLE t1(i INTEGER PRIMARY KEY, a, b); CREATE VIRTUAL TABLE xx USING fts4(content=t1, a, b); INSERT INTO t1(rowid, a, b) VALUES(10, 'a b c', 'c b a'); INSERT INTO t1(rowid, a, b) VALUES(20, 'a b c', 'd e f'); INSERT INTO t1(rowid, a, b) VALUES(30, 'd e f', 'a b c'); INSERT INTO t1(rowid, a, b) VALUES(40, 'd e f', 'd e f'); } do_execsql_test 1.1 { INSERT INTO xx(xx) VALUES('rebuild'); INSERT INTO xx(xx) VALUES('integrity-check'); } forcedelete rbu.db do_test 2.0 { sqlite3 dbrbu rbu.db dbrbu eval { CREATE TABLE data_t1(i, a, b, rbu_control); INSERT INTO data_t1 VALUES(20, NULL, NULL, 1); -- delete INSERT INTO data_t1 VALUES(30, 'x y z', NULL, '.x.'); -- update INSERT INTO data_t1 VALUES(50, '1 2 3', 'x y z', 0); -- insert CREATE VIEW data0_xx AS SELECT i AS rbu_rowid, a, b, CASE WHEN rbu_control IN (0, 1) THEN rbu_control ELSE substr(rbu_control, 2) END AS rbu_control FROM data_t1; } dbrbu close step_rbu test.db rbu.db } {SQLITE_DONE} do_execsql_test 2.1 { INSERT INTO xx(xx) VALUES('integrity-check'); } finish_test |
Changes to ext/rbu/sqlite3rbu.c.
︙ | ︙ | |||
235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 | u8 *abNotNull; /* Array of flags, set on NOT NULL columns */ u8 *abIndexed; /* Array of flags, set on indexed & PK cols */ int eType; /* Table type - an RBU_PK_XXX value */ /* Output variables. zTbl==0 implies EOF. */ int bCleanup; /* True in "cleanup" state */ const char *zTbl; /* Name of target db table */ const char *zIdx; /* Name of target db index (or null) */ int iTnum; /* Root page of current object */ int iPkTnum; /* If eType==EXTERNAL, root of PK index */ int bUnique; /* Current index is unique */ /* Statements created by rbuObjIterPrepareAll() */ int nCol; /* Number of columns in current object */ sqlite3_stmt *pSelect; /* Source data */ sqlite3_stmt *pInsert; /* Statement for INSERT operations */ sqlite3_stmt *pDelete; /* Statement for DELETE ops */ | > | | 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 | u8 *abNotNull; /* Array of flags, set on NOT NULL columns */ u8 *abIndexed; /* Array of flags, set on indexed & PK cols */ int eType; /* Table type - an RBU_PK_XXX value */ /* Output variables. zTbl==0 implies EOF. */ int bCleanup; /* True in "cleanup" state */ const char *zTbl; /* Name of target db table */ const char *zDataTbl; /* Name of rbu db table (or null) */ const char *zIdx; /* Name of target db index (or null) */ int iTnum; /* Root page of current object */ int iPkTnum; /* If eType==EXTERNAL, root of PK index */ int bUnique; /* Current index is unique */ /* Statements created by rbuObjIterPrepareAll() */ int nCol; /* Number of columns in current object */ sqlite3_stmt *pSelect; /* Source data */ sqlite3_stmt *pInsert; /* Statement for INSERT operations */ sqlite3_stmt *pDelete; /* Statement for DELETE ops */ sqlite3_stmt *pTmpInsert; /* Insert into rbu_tmp_$zDataTbl */ /* Last UPDATE used (for PK b-tree updates only), or NULL. */ RbuUpdateStmt *pRbuUpdate; }; /* ** Values for RbuObjIter.eType |
︙ | ︙ | |||
522 523 524 525 526 527 528 | pIter->bCleanup = 0; rc = sqlite3_step(pIter->pTblIter); if( rc!=SQLITE_ROW ){ rc = resetAndCollectError(pIter->pTblIter, &p->zErrmsg); pIter->zTbl = 0; }else{ pIter->zTbl = (const char*)sqlite3_column_text(pIter->pTblIter, 0); | > | | 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 | pIter->bCleanup = 0; rc = sqlite3_step(pIter->pTblIter); if( rc!=SQLITE_ROW ){ rc = resetAndCollectError(pIter->pTblIter, &p->zErrmsg); pIter->zTbl = 0; }else{ pIter->zTbl = (const char*)sqlite3_column_text(pIter->pTblIter, 0); pIter->zDataTbl = (const char*)sqlite3_column_text(pIter->pTblIter,1); rc = (pIter->zDataTbl && pIter->zTbl) ? SQLITE_OK : SQLITE_NOMEM; } }else{ if( pIter->zIdx==0 ){ sqlite3_stmt *pIdx = pIter->pIdxIter; rc = sqlite3_bind_text(pIdx, 1, pIter->zTbl, -1, SQLITE_STATIC); } if( rc==SQLITE_OK ){ |
︙ | ︙ | |||
553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 | if( rc!=SQLITE_OK ){ rbuObjIterFinalize(pIter); p->rc = rc; } return rc; } /* ** Initialize the iterator structure passed as the second argument. ** ** If no error occurs, SQLITE_OK is returned and the iterator is left ** pointing to the first entry. Otherwise, an error code and message is ** left in the RBU handle passed as the first argument. A copy of the ** error code is returned. */ static int rbuObjIterFirst(sqlite3rbu *p, RbuObjIter *pIter){ int rc; memset(pIter, 0, sizeof(RbuObjIter)); rc = prepareAndCollectError(p->dbRbu, &pIter->pTblIter, &p->zErrmsg, | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | | > | 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 | if( rc!=SQLITE_OK ){ rbuObjIterFinalize(pIter); p->rc = rc; } return rc; } /* ** The implementation of the rbu_target_name() SQL function. This function ** accepts one argument - the name of a table in the RBU database. If the ** table name matches the pattern: ** ** data[0-9]_<name> ** ** where <name> is any sequence of 1 or more characters, <name> is returned. ** Otherwise, if the only argument does not match the above pattern, an SQL ** NULL is returned. ** ** "data_t1" -> "t1" ** "data0123_t2" -> "t2" ** "dataAB_t3" -> NULL */ static void rbuTargetNameFunc( sqlite3_context *context, int argc, sqlite3_value **argv ){ const char *zIn; assert( argc==1 ); zIn = (const char*)sqlite3_value_text(argv[0]); if( zIn && strlen(zIn)>4 && memcmp("data", zIn, 4)==0 ){ int i; for(i=4; zIn[i]>='0' && zIn[i]<='9'; i++); if( zIn[i]=='_' && zIn[i+1] ){ sqlite3_result_text(context, &zIn[i+1], -1, SQLITE_STATIC); } } } /* ** Initialize the iterator structure passed as the second argument. ** ** If no error occurs, SQLITE_OK is returned and the iterator is left ** pointing to the first entry. Otherwise, an error code and message is ** left in the RBU handle passed as the first argument. A copy of the ** error code is returned. */ static int rbuObjIterFirst(sqlite3rbu *p, RbuObjIter *pIter){ int rc; memset(pIter, 0, sizeof(RbuObjIter)); rc = prepareAndCollectError(p->dbRbu, &pIter->pTblIter, &p->zErrmsg, "SELECT rbu_target_name(name) AS target, name FROM sqlite_master " "WHERE type IN ('table', 'view') AND target IS NOT NULL " "ORDER BY name" ); if( rc==SQLITE_OK ){ rc = prepareAndCollectError(p->dbMain, &pIter->pIdxIter, &p->zErrmsg, "SELECT name, rootpage, sql IS NULL OR substr(8, 6)=='UNIQUE' " " FROM main.sqlite_master " " WHERE type='index' AND tbl_name = ?" |
︙ | ︙ | |||
913 914 915 916 917 918 919 | || pIter->eType==RBU_PK_VTAB ); /* Populate the azTblCol[] and nTblCol variables based on the columns ** of the input table. Ignore any input table columns that begin with ** "rbu_". */ p->rc = prepareFreeAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg, | | | 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 | || pIter->eType==RBU_PK_VTAB ); /* Populate the azTblCol[] and nTblCol variables based on the columns ** of the input table. Ignore any input table columns that begin with ** "rbu_". */ p->rc = prepareFreeAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg, sqlite3_mprintf("SELECT * FROM '%q'", pIter->zDataTbl) ); if( p->rc==SQLITE_OK ){ nCol = sqlite3_column_count(pStmt); rbuAllocateIterArrays(p, pIter, nCol); } for(i=0; p->rc==SQLITE_OK && i<nCol; i++){ const char *zName = (const char*)sqlite3_column_name(pStmt, i); |
︙ | ︙ | |||
938 939 940 941 942 943 944 | pStmt = 0; if( p->rc==SQLITE_OK && bRbuRowid!=(pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE) ){ p->rc = SQLITE_ERROR; p->zErrmsg = sqlite3_mprintf( | | | | | 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 | pStmt = 0; if( p->rc==SQLITE_OK && bRbuRowid!=(pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE) ){ p->rc = SQLITE_ERROR; p->zErrmsg = sqlite3_mprintf( "table %q %s rbu_rowid column", pIter->zDataTbl, (bRbuRowid ? "may not have" : "requires") ); } /* Check that all non-HIDDEN columns in the destination table are also ** present in the input table. Populate the abTblPk[], azTblType[] and ** aiTblOrder[] arrays at the same time. */ if( p->rc==SQLITE_OK ){ p->rc = prepareFreeAndCollectError(p->dbMain, &pStmt, &p->zErrmsg, sqlite3_mprintf("PRAGMA table_info(%Q)", pIter->zTbl) ); } while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ const char *zName = (const char*)sqlite3_column_text(pStmt, 1); if( zName==0 ) break; /* An OOM - finalize() below returns S_NOMEM */ for(i=iOrder; i<pIter->nTblCol; i++){ if( 0==strcmp(zName, pIter->azTblCol[i]) ) break; } if( i==pIter->nTblCol ){ p->rc = SQLITE_ERROR; p->zErrmsg = sqlite3_mprintf("column missing from %q: %s", pIter->zDataTbl, zName ); }else{ int iPk = sqlite3_column_int(pStmt, 5); int bNotNull = sqlite3_column_int(pStmt, 3); const char *zType = (const char*)sqlite3_column_text(pStmt, 2); if( i!=iOrder ){ |
︙ | ︙ | |||
1515 1516 1517 1518 1519 1520 1521 | int bRbuRowid = (pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE); char *zBind = rbuObjIterGetBindlist(p, pIter->nTblCol + 1 + bRbuRowid); if( zBind ){ assert( pIter->pTmpInsert==0 ); p->rc = prepareFreeAndCollectError( p->dbRbu, &pIter->pTmpInsert, &p->zErrmsg, sqlite3_mprintf( "INSERT INTO %s.'rbu_tmp_%q'(rbu_control,%s%s) VALUES(%z)", | | | 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 | int bRbuRowid = (pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE); char *zBind = rbuObjIterGetBindlist(p, pIter->nTblCol + 1 + bRbuRowid); if( zBind ){ assert( pIter->pTmpInsert==0 ); p->rc = prepareFreeAndCollectError( p->dbRbu, &pIter->pTmpInsert, &p->zErrmsg, sqlite3_mprintf( "INSERT INTO %s.'rbu_tmp_%q'(rbu_control,%s%s) VALUES(%z)", p->zStateDb, pIter->zDataTbl, zCollist, zRbuRowid, zBind )); } } static void rbuTmpInsertFunc( sqlite3_context *pCtx, int nVal, |
︙ | ︙ | |||
1611 1612 1613 1614 1615 1616 1617 | /* Create the SELECT statement to read keys in sorted order */ if( p->rc==SQLITE_OK ){ char *zSql; if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ zSql = sqlite3_mprintf( "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' ORDER BY %s%s", | | | | | | 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 | /* Create the SELECT statement to read keys in sorted order */ if( p->rc==SQLITE_OK ){ char *zSql; if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ zSql = sqlite3_mprintf( "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' ORDER BY %s%s", zCollist, p->zStateDb, pIter->zDataTbl, zCollist, zLimit ); }else{ zSql = sqlite3_mprintf( "SELECT %s, rbu_control FROM '%q' " "WHERE typeof(rbu_control)='integer' AND rbu_control!=1 " "UNION ALL " "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' " "ORDER BY %s%s", zCollist, pIter->zDataTbl, zCollist, p->zStateDb, pIter->zDataTbl, zCollist, zLimit ); } p->rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pSelect, pz, zSql); } sqlite3_free(zImposterCols); |
︙ | ︙ | |||
1650 1651 1652 1653 1654 1655 1656 | zCollist = rbuObjIterGetCollist(p, pIter); pIter->nCol = pIter->nTblCol; /* Create the SELECT statement to read keys from data_xxx */ if( p->rc==SQLITE_OK ){ p->rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pSelect, pz, sqlite3_mprintf( | | | > | 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 | zCollist = rbuObjIterGetCollist(p, pIter); pIter->nCol = pIter->nTblCol; /* Create the SELECT statement to read keys from data_xxx */ if( p->rc==SQLITE_OK ){ p->rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pSelect, pz, sqlite3_mprintf( "SELECT %s, rbu_control%s FROM '%q'%s", zCollist, (bRbuRowid ? ", rbu_rowid" : ""), pIter->zDataTbl, zLimit ) ); } /* Create the imposter table or tables (if required). */ rbuCreateImposterTable(p, pIter); rbuCreateImposterTable2(p, pIter); |
︙ | ︙ | |||
1689 1690 1691 1692 1693 1694 1695 | if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ zRbuRowid = ", rbu_rowid"; } /* Create the rbu_tmp_xxx table and the triggers to populate it. */ rbuMPrintfExec(p, p->dbRbu, "CREATE TABLE IF NOT EXISTS %s.'rbu_tmp_%q' AS " | | | | | | 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 | if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ zRbuRowid = ", rbu_rowid"; } /* Create the rbu_tmp_xxx table and the triggers to populate it. */ rbuMPrintfExec(p, p->dbRbu, "CREATE TABLE IF NOT EXISTS %s.'rbu_tmp_%q' AS " "SELECT *%s FROM '%q' WHERE 0;" , p->zStateDb, pIter->zDataTbl , (pIter->eType==RBU_PK_EXTERNAL ? ", 0 AS rbu_rowid" : "") , pIter->zDataTbl ); rbuMPrintfExec(p, p->dbMain, "CREATE TEMP TRIGGER rbu_delete_tr BEFORE DELETE ON \"%s%w\" " "BEGIN " " SELECT rbu_tmp_insert(2, %s);" "END;" |
︙ | ︙ | |||
1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 | } if( p->rc==SQLITE_OK ){ p->rc = sqlite3_create_function(p->dbMain, "rbu_tmp_insert", -1, SQLITE_UTF8, (void*)p, rbuTmpInsertFunc, 0, 0 ); } if( p->rc==SQLITE_OK ){ p->rc = sqlite3_file_control(p->dbMain, "main", SQLITE_FCNTL_RBU, (void*)p); } rbuMPrintfExec(p, p->dbMain, "SELECT * FROM sqlite_master"); /* Mark the database file just opened as an RBU target database. If | > > > > > > | 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 | } if( p->rc==SQLITE_OK ){ p->rc = sqlite3_create_function(p->dbMain, "rbu_tmp_insert", -1, SQLITE_UTF8, (void*)p, rbuTmpInsertFunc, 0, 0 ); } if( p->rc==SQLITE_OK ){ p->rc = sqlite3_create_function(p->dbRbu, "rbu_target_name", 1, SQLITE_UTF8, (void*)p, rbuTargetNameFunc, 0, 0 ); } if( p->rc==SQLITE_OK ){ p->rc = sqlite3_file_control(p->dbMain, "main", SQLITE_FCNTL_RBU, (void*)p); } rbuMPrintfExec(p, p->dbMain, "SELECT * FROM sqlite_master"); /* Mark the database file just opened as an RBU target database. If |
︙ | ︙ | |||
2399 2400 2401 2402 2403 2404 2405 | if( pIter->bCleanup ){ /* Clean up the rbu_tmp_xxx table for the previous table. It ** cannot be dropped as there are currently active SQL statements. ** But the contents can be deleted. */ if( pIter->abIndexed ){ rbuMPrintfExec(p, p->dbRbu, | | | 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 | if( pIter->bCleanup ){ /* Clean up the rbu_tmp_xxx table for the previous table. It ** cannot be dropped as there are currently active SQL statements. ** But the contents can be deleted. */ if( pIter->abIndexed ){ rbuMPrintfExec(p, p->dbRbu, "DELETE FROM %s.'rbu_tmp_%q'", p->zStateDb, pIter->zDataTbl ); } }else{ rbuObjIterPrepareAll(p, pIter, 0); /* Advance to the next row to process. */ if( p->rc==SQLITE_OK ){ |
︙ | ︙ |
Changes to ext/rbu/sqlite3rbu.h.
︙ | ︙ | |||
93 94 95 96 97 98 99 100 101 102 103 104 105 106 | ** ** Then the RBU database should contain: ** ** CREATE TABLE data_t1(a INTEGER, b TEXT, c, rbu_control); ** ** The order of the columns in the data_% table does not matter. ** ** If the target database table is a virtual table or a table that has no ** PRIMARY KEY declaration, the data_% table must also contain a column ** named "rbu_rowid". This column is mapped to the tables implicit primary ** key column - "rowid". Virtual tables for which the "rowid" column does ** not function like a primary key value cannot be updated using RBU. For ** example, if the target db contains either of the following: ** | > > > > > > > > > > > > | 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 | ** ** Then the RBU database should contain: ** ** CREATE TABLE data_t1(a INTEGER, b TEXT, c, rbu_control); ** ** The order of the columns in the data_% table does not matter. ** ** Instead of a regular table, the RBU database may also contain virtual ** tables or view named using the data_<target> naming scheme. ** ** Instead of the plain data_<target> naming scheme, RBU database tables ** may also be named data<integer>_<target>, where <integer> is any sequence ** of zero or more numeric characters (0-9). This can be significant because ** tables within the RBU database are always processed in order sorted by ** name. By judicious selection of the the <integer> portion of the names ** of the RBU tables the user can therefore control the order in which they ** are processed. This can be useful, for example, to ensure that "external ** content" FTS4 tables are updated before their underlying content tables. ** ** If the target database table is a virtual table or a table that has no ** PRIMARY KEY declaration, the data_% table must also contain a column ** named "rbu_rowid". This column is mapped to the tables implicit primary ** key column - "rowid". Virtual tables for which the "rowid" column does ** not function like a primary key value cannot be updated using RBU. For ** example, if the target db contains either of the following: ** |
︙ | ︙ |