/ Check-in [84da122d]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Merge the latest trunk changes.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | tempfiles-25
Files: files | file ages | folders
SHA1: 84da122dd6e70ed603fea781dca204ae2f668c53
User & Date: drh 2016-04-23 19:55:59
Context
2016-04-23
21:16
Merge the temporary directory search algorithm fix from trunk. check-in: 9b8fec60 user: drh tags: tempfiles-25
19:55
Merge the latest trunk changes. check-in: 84da122d user: drh tags: tempfiles-25
17:24
Fix a problem with mixing temp-files and mmap-mode. check-in: c80c5c62 user: dan tags: tempfiles-25
2016-04-21
23:37
Change the temporary directory search algorithm on unix so that directories with only -wx permission are allowed. And do not allow "." to be returned if it lacks -wx permission. check-in: 67985761 user: drh tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/rbu/rbu.c.

    20     20   #include <string.h>
    21     21   
    22     22   /*
    23     23   ** Print a usage message and exit.
    24     24   */
    25     25   void usage(const char *zArgv0){
    26     26     fprintf(stderr, 
    27         -"Usage: %s [-step NSTEP] TARGET-DB RBU-DB\n"
           27  +"Usage: %s ?OPTIONS? TARGET-DB RBU-DB\n"
           28  +"\n"
           29  +"Where options are:\n"
           30  +"\n"
           31  +"    -step NSTEP\n"
           32  +"    -vacuum\n"
           33  +"\n"
           34  +"  If the -vacuum switch is not present, argument RBU-DB must be an RBU\n"
           35  +"  database containing an update suitable for target database TARGET-DB.\n"
           36  +"  Or, if -vacuum is specified, then TARGET-DB is a database to vacuum using\n"
           37  +"  RBU, and RBU-DB is used as the state database for the vacuum (refer to\n"
           38  +"  API documentation for details).\n"
    28     39   "\n"
    29         -"  Argument RBU-DB must be an RBU database containing an update suitable for\n"
    30         -"  target database TARGET-DB. If NSTEP is set to less than or equal to zero\n"
    31         -"  (the default value), this program attempts to apply the entire update to\n"
    32         -"  the target database.\n"
           40  +"  If NSTEP is set to less than or equal to zero (the default value), this \n"
           41  +"  program attempts to perform the entire update or vacuum operation before\n"
           42  +"  exiting\n"
    33     43   "\n"
    34     44   "  If NSTEP is greater than zero, then a maximum of NSTEP calls are made\n"
    35     45   "  to sqlite3rbu_step(). If the RBU update has not been completely applied\n"
    36     46   "  after the NSTEP'th call is made, the state is saved in the database RBU-DB\n"
    37     47   "  and the program exits. Subsequent invocations of this (or any other RBU)\n"
    38     48   "  application will use this state to resume applying the RBU update to the\n"
    39     49   "  target db.\n"
................................................................................
    65     75     int i;
    66     76     const char *zTarget;            /* Target database to apply RBU to */
    67     77     const char *zRbu;               /* Database containing RBU */
    68     78     char zBuf[200];                 /* Buffer for printf() */
    69     79     char *zErrmsg;                  /* Error message, if any */
    70     80     sqlite3rbu *pRbu;               /* RBU handle */
    71     81     int nStep = 0;                  /* Maximum number of step() calls */
           82  +  int bVacuum = 0;
    72     83     int rc;
    73     84     sqlite3_int64 nProgress = 0;
           85  +  int nArg = argc-2;
    74     86   
    75         -  /* Process command line arguments. Following this block local variables 
    76         -  ** zTarget, zRbu and nStep are all set. */
    77         -  if( argc==5 ){
    78         -    size_t nArg1 = strlen(argv[1]);
    79         -    if( nArg1>5 || nArg1<2 || memcmp("-step", argv[1], nArg1) ) usage(argv[0]);
    80         -    nStep = atoi(argv[2]);
    81         -  }else if( argc!=3 ){
    82         -    usage(argv[0]);
           87  +  if( argc<3 ) usage(argv[0]);
           88  +  for(i=1; i<nArg; i++){
           89  +    const char *zArg = argv[i];
           90  +    int nArg = strlen(zArg);
           91  +    if( nArg>1 && nArg<=8 && 0==memcmp(zArg, "-vacuum", nArg) ){
           92  +      bVacuum = 1;
           93  +    }else if( nArg>1 && nArg<=5 && 0==memcmp(zArg, "-step", nArg) && i<nArg-1 ){
           94  +      i++;
           95  +      nStep = atoi(argv[i]);
           96  +    }else{
           97  +      usage(argv[0]);
           98  +    }
    83     99     }
          100  +
    84    101     zTarget = argv[argc-2];
    85    102     zRbu = argv[argc-1];
    86    103   
    87    104     report_default_vfs();
    88    105   
    89         -  /* Open an RBU handle. If nStep is less than or equal to zero, call
          106  +  /* Open an RBU handle. A vacuum handle if -vacuum was specified, or a
          107  +  ** regular RBU update handle otherwise.  */
          108  +  if( bVacuum ){
          109  +    pRbu = sqlite3rbu_vacuum(zTarget, zRbu);
          110  +  }else{
          111  +    pRbu = sqlite3rbu_open(zTarget, zRbu, 0);
          112  +  }
          113  +  report_rbu_vfs(pRbu);
          114  +
          115  +  /* If nStep is less than or equal to zero, call
    90    116     ** sqlite3rbu_step() until either the RBU has been completely applied
    91    117     ** or an error occurs. Or, if nStep is greater than zero, call
    92    118     ** sqlite3rbu_step() a maximum of nStep times.  */
    93         -  pRbu = sqlite3rbu_open(zTarget, zRbu, 0);
    94         -  report_rbu_vfs(pRbu);
    95    119     for(i=0; (nStep<=0 || i<nStep) && sqlite3rbu_step(pRbu)==SQLITE_OK; i++);
    96    120     nProgress = sqlite3rbu_progress(pRbu);
    97    121     rc = sqlite3rbu_close(pRbu, &zErrmsg);
    98    122   
    99    123     /* Let the user know what happened. */
   100    124     switch( rc ){
   101    125       case SQLITE_OK:

Added ext/rbu/rbufault3.test.

            1  +# 2016 April 20
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +#
           12  +# This file contains fault injection tests for RBU vacuum operations.
           13  +#
           14  +
           15  +source [file join [file dirname [info script]] rbu_common.tcl]
           16  +source $testdir/malloc_common.tcl
           17  +set ::testprefix rbufault3
           18  +
           19  +foreach {fault errlist} {
           20  +  oom-* {
           21  +    {1 SQLITE_NOMEM}
           22  +    {1 SQLITE_IOERR_NOMEM}
           23  +    {1 {SQLITE_NOMEM - out of memory}}
           24  +  }
           25  +
           26  +  ioerr-* {
           27  +    {1 {SQLITE_IOERR - disk I/O error}}
           28  +    {1 SQLITE_IOERR} 
           29  +    {1 SQLITE_IOERR_WRITE} 
           30  +    {1 SQLITE_IOERR_FSYNC} 
           31  +    {1 SQLITE_IOERR_READ} 
           32  +    {1 {SQLITE_IOERR - unable to open database: test.db2}} 
           33  +    {1 {SQLITE_ERROR - unable to open database: test.db2}} 
           34  +    {1 {SQLITE_ERROR - SQL logic error or missing database}}
           35  +  }
           36  +
           37  +  cantopen* {
           38  +    {1 {SQLITE_CANTOPEN - unable to open database: test.db2}}  
           39  +    {1 {SQLITE_CANTOPEN - unable to open database: test.db2}}  
           40  +    {1 {SQLITE_CANTOPEN - unable to open database file}}  
           41  +    {1 SQLITE_CANTOPEN} 
           42  +  }
           43  +
           44  +} {
           45  +
           46  +  reset_db
           47  +  do_execsql_test 0 {
           48  +    CREATE TABLE target(x UNIQUE, y, z, PRIMARY KEY(y));
           49  +    INSERT INTO target VALUES(1, 2, 3);
           50  +    INSERT INTO target VALUES(4, 5, 6);
           51  +    INSERT INTO target VALUES(7, 8, 9);
           52  +    CREATE INDEX i1 ON target(z);
           53  +  }
           54  +  faultsim_save_and_close
           55  +
           56  +  do_faultsim_test 1 -faults $fault -prep {
           57  +    faultsim_restore_and_reopen
           58  +    forcedelete test.db2
           59  +  } -body {
           60  +    sqlite3rbu_vacuum rbu test.db test.db2
           61  +    while {[rbu step]=="SQLITE_OK"} {}
           62  +    rbu close
           63  +  } -test {
           64  +    eval [list faultsim_test_result {0 SQLITE_DONE} {*}$::errlist]
           65  +  }
           66  +
           67  +  do_faultsim_test 2 -faults $fault -prep {
           68  +    faultsim_restore_and_reopen
           69  +    forcedelete test.db2
           70  +  } -body {
           71  +    sqlite3rbu_vacuum rbu test.db test.db2
           72  +    rbu step
           73  +    rbu close
           74  +  } -test {
           75  +    eval [list faultsim_test_result {0 SQLITE_OK} {*}$::errlist]
           76  +  }
           77  +
           78  +  forcedelete test.db2
           79  +  sqlite3rbu_vacuum rbu test.db test.db2
           80  +  rbu step
           81  +  rbu close
           82  +  faultsim_save_and_close
           83  +
           84  +  do_faultsim_test 3 -faults $fault -prep {
           85  +    faultsim_restore_and_reopen
           86  +    forcedelete test.db2
           87  +  } -body {
           88  +    sqlite3rbu_vacuum rbu test.db test.db2
           89  +    rbu step
           90  +    rbu close
           91  +  } -test {
           92  +    eval [list faultsim_test_result {0 SQLITE_OK} {*}$::errlist]
           93  +  }
           94  +
           95  +}
           96  +
           97  +finish_test
           98  +

Added ext/rbu/rbuvacuum.test.

            1  +# 2016 April 15
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +#
           12  +# This file contains tests for the RBU module. More specifically, it
           13  +# contains tests to ensure that the sqlite3rbu_vacuum() API works as
           14  +# expected.
           15  +#
           16  +
           17  +source [file join [file dirname [info script]] rbu_common.tcl]
           18  +set ::testprefix rbuvacuum
           19  +
           20  +proc do_rbu_vacuum_test {tn step} {
           21  +  uplevel [list do_test $tn.1 {
           22  +    if {$step==0} { sqlite3rbu_vacuum rbu test.db state.db }
           23  +    while 1 {
           24  +      if {$step==1} { sqlite3rbu_vacuum rbu test.db state.db }
           25  +      set rc [rbu step]
           26  +      if {$rc!="SQLITE_OK"} break
           27  +      if {$step==1} { rbu close }
           28  +    }
           29  +    rbu close
           30  +  } {SQLITE_DONE}]
           31  +
           32  +  uplevel [list do_execsql_test $tn.2 {
           33  +    PRAGMA integrity_check
           34  +  } ok]
           35  +}
           36  +
           37  +foreach step {0 1} {
           38  +
           39  +  set ::testprefix rbuvacuum-step=$step
           40  +  reset_db
           41  +
           42  +  # Simplest possible vacuum.
           43  +  do_execsql_test 1.0 {
           44  +    PRAGMA page_size = 1024;
           45  +    CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
           46  +    INSERT INTO t1 VALUES(1, 2, 3);
           47  +    INSERT INTO t1 VALUES(4, 5, 6);
           48  +    INSERT INTO t1 VALUES(7, 8, 9);
           49  +    PRAGMA integrity_check;
           50  +  } {ok}
           51  +  do_rbu_vacuum_test 1.1 $step
           52  +
           53  +  # A vacuum that actually reclaims space.
           54  +  do_execsql_test 1.2.1 {
           55  +    INSERT INTO t1 VALUES(8, randomblob(900), randomblob(900));
           56  +    INSERT INTO t1 VALUES(9, randomblob(900), randomblob(900));
           57  +    INSERT INTO t1 VALUES(10, randomblob(900), randomblob(900));
           58  +    INSERT INTO t1 VALUES(11, randomblob(900), randomblob(900));
           59  +    INSERT INTO t1 VALUES(12, randomblob(900), randomblob(900));
           60  +    PRAGMA page_count;
           61  +  } {12}
           62  +  do_execsql_test 1.2.2 {
           63  +    DELETE FROM t1 WHERE rowid BETWEEN 8 AND 11;
           64  +    PRAGMA page_count;
           65  +  } {12}
           66  +  do_rbu_vacuum_test 1.2.3 $step
           67  +  do_execsql_test 1.2.4 {
           68  +    PRAGMA page_count;
           69  +  } {3}
           70  +  
           71  +  # Add an index to the table.
           72  +  do_execsql_test 1.3.1 {
           73  +    CREATE INDEX t1b ON t1(b);
           74  +    INSERT INTO t1 VALUES(13, randomblob(900), randomblob(900));
           75  +    INSERT INTO t1 VALUES(14, randomblob(900), randomblob(900));
           76  +    INSERT INTO t1 VALUES(15, randomblob(900), randomblob(900));
           77  +    INSERT INTO t1 VALUES(16, randomblob(900), randomblob(900));
           78  +    PRAGMA page_count;
           79  +  } {18}
           80  +  do_execsql_test 1.3.2 {
           81  +    DELETE FROM t1 WHERE rowid BETWEEN 12 AND 15;
           82  +    PRAGMA page_count;
           83  +  } {18}
           84  +  do_rbu_vacuum_test 1.3.3 $step
           85  +  do_execsql_test 1.3.4 {
           86  +    PRAGMA page_count;
           87  +  } {5}
           88  +
           89  +  # WITHOUT ROWID table.
           90  +  do_execsql_test 1.4.1 {
           91  +    CREATE TABLE t2(a, b, c, PRIMARY KEY(a, b)) WITHOUT ROWID;
           92  +
           93  +    INSERT INTO t2 VALUES(randomblob(900), 1, randomblob(900));
           94  +    INSERT INTO t2 VALUES(randomblob(900), 2, randomblob(900));
           95  +    INSERT INTO t2 VALUES(randomblob(900), 3, randomblob(900));
           96  +    INSERT INTO t2 VALUES(randomblob(900), 4, randomblob(900));
           97  +    INSERT INTO t2 VALUES(randomblob(900), 6, randomblob(900));
           98  +    INSERT INTO t2 VALUES(randomblob(900), 7, randomblob(900));
           99  +    INSERT INTO t2 VALUES(randomblob(900), 8, randomblob(900));
          100  +
          101  +    DELETE FROM t2 WHERE b BETWEEN 2 AND 7;
          102  +    PRAGMA page_count;
          103  +  } {20}
          104  +  do_rbu_vacuum_test 1.4.2 $step
          105  +  do_execsql_test 1.4.3 {
          106  +    PRAGMA page_count;
          107  +  } {10}
          108  +  
          109  +  # WITHOUT ROWID table with an index.
          110  +  do_execsql_test 1.4.1 {
          111  +    CREATE INDEX t2c ON t2(c);
          112  +
          113  +    INSERT INTO t2 VALUES(randomblob(900), 9, randomblob(900));
          114  +    INSERT INTO t2 VALUES(randomblob(900), 10, randomblob(900));
          115  +    INSERT INTO t2 VALUES(randomblob(900), 11, randomblob(900));
          116  +    INSERT INTO t2 VALUES(randomblob(900), 12, randomblob(900));
          117  +    INSERT INTO t2 VALUES(randomblob(900), 13, randomblob(900));
          118  +
          119  +    DELETE FROM t2 WHERE b BETWEEN 8 AND 12;
          120  +    PRAGMA page_count;
          121  +  } {35}
          122  +  do_rbu_vacuum_test 1.4.2 $step
          123  +  do_execsql_test 1.4.3 {
          124  +    PRAGMA page_count;
          125  +  } {15}
          126  +  do_execsql_test 1.4.4 {
          127  +    VACUUM;
          128  +    PRAGMA page_count;
          129  +  } {15}
          130  +
          131  +  do_execsql_test 1.5.1 {
          132  +    CREATE TABLE t3(a, b, c);
          133  +    INSERT INTO t3 VALUES('a', 'b', 'c');
          134  +    INSERT INTO t3 VALUES('d', 'e', 'f');
          135  +    INSERT INTO t3 VALUES('g', 'h', 'i');
          136  +  }
          137  +  do_rbu_vacuum_test 1.5.2 $step
          138  +  do_execsql_test 1.5.3 {
          139  +    SELECT * FROM t3
          140  +  } {a b c d e f g h i}
          141  +  do_execsql_test 1.5.4 {
          142  +    CREATE INDEX t3a ON t3(a);
          143  +    CREATE INDEX t3b ON t3(b);
          144  +    CREATE INDEX t3c ON t3(c);
          145  +    INSERT INTO t3 VALUES('j', 'k', 'l');
          146  +    DELETE FROM t3 WHERE a = 'g';
          147  +  }
          148  +  do_rbu_vacuum_test 1.5.5 $step
          149  +  do_execsql_test 1.5.6 {
          150  +    SELECT rowid, * FROM t3 ORDER BY b
          151  +  } {1 a b c 2 d e f 4 j k l}
          152  +
          153  +  do_execsql_test 1.6.1 {
          154  +    CREATE TABLE t4(a PRIMARY KEY, b, c);
          155  +    INSERT INTO t4 VALUES('a', 'b', 'c');
          156  +    INSERT INTO t4 VALUES('d', 'e', 'f');
          157  +    INSERT INTO t4 VALUES('g', 'h', 'i');
          158  +  }
          159  +  do_rbu_vacuum_test 1.6.2 $step
          160  +  do_execsql_test 1.6.3 {
          161  +    SELECT * FROM t4
          162  +  } {a b c d e f g h i}
          163  +  do_execsql_test 1.6.4 {
          164  +    CREATE INDEX t4a ON t4(a);
          165  +    CREATE INDEX t4b ON t4(b);
          166  +    CREATE INDEX t4c ON t4(c);
          167  +    
          168  +    INSERT INTO t4 VALUES('j', 'k', 'l');
          169  +    DELETE FROM t4 WHERE a='g';
          170  +  }
          171  +  do_rbu_vacuum_test 1.6.5 $step
          172  +  do_execsql_test 1.6.6 {
          173  +    SELECT * FROM t4 ORDER BY b
          174  +  } {a b c d e f j k l}
          175  +
          176  +  reset_db
          177  +  do_execsql_test 1.7.0 {
          178  +    CREATE TABLE t1(a INTEGER PRIMARY KEY AUTOINCREMENT, b);
          179  +    INSERT INTO t1 VALUES(NULL, 'one');
          180  +    INSERT INTO t1 VALUES(NULL, 'two');
          181  +    DELETE FROM t1 WHERE a=2;
          182  +    INSERT INTO t1 VALUES(NULL, 'three');
          183  +    INSERT INTO t1 VALUES(NULL, 'four');
          184  +    DELETE FROM t1 WHERE a=4;
          185  +    INSERT INTO t1 VALUES(NULL, 'five');
          186  +    INSERT INTO t1 VALUES(NULL, 'six');
          187  +    DELETE FROM t1 WHERE a=6;
          188  +    SELECT * FROM t1;
          189  +  } {1 one 3 three 5 five}
          190  +  do_rbu_vacuum_test 1.7.1 $step
          191  +  do_execsql_test 1.7.2 {
          192  +    INSERT INTO t1 VALUES(NULL, 'seven');
          193  +    SELECT * FROM t1;
          194  +  } {1 one 3 three 5 five 7 seven}
          195  +
          196  +  reset_db
          197  +  do_execsql_test 1.8.0 {
          198  +    CREATE TABLE t1(a INTEGER PRIMARY KEY AUTOINCREMENT, b);
          199  +    CREATE INDEX i1 ON t1(b);
          200  +    INSERT INTO t1 VALUES(NULL, 'one');
          201  +    INSERT INTO t1 VALUES(NULL, 'two');
          202  +    INSERT INTO t1 VALUES(NULL, 'three');
          203  +    INSERT INTO t1 VALUES(NULL, 'four');
          204  +    INSERT INTO t1 VALUES(NULL, 'five');
          205  +    INSERT INTO t1 VALUES(NULL, 'six');
          206  +    ANALYZE;
          207  +    SELECT * FROM sqlite_stat1;
          208  +  } {t1 i1 {6 1}}
          209  +  do_rbu_vacuum_test 1.8.1 $step
          210  +  do_execsql_test 1.7.2 {
          211  +    SELECT * FROM sqlite_stat1;
          212  +  } {t1 i1 {6 1}}
          213  +
          214  +  reset_db
          215  +  do_execsql_test 1.9.0 {
          216  +    PRAGMA page_size = 8192;
          217  +    PRAGMA auto_vacuum = 2;
          218  +    PRAGMA user_version = 412;
          219  +    PRAGMA application_id = 413;
          220  +
          221  +    CREATE TABLE t1(a INTEGER PRIMARY KEY AUTOINCREMENT, b);
          222  +    CREATE INDEX i1 ON t1(b);
          223  +    INSERT INTO t1 VALUES(NULL, 'one');
          224  +    INSERT INTO t1 VALUES(NULL, 'two');
          225  +    INSERT INTO t1 VALUES(NULL, 'three');
          226  +    INSERT INTO t1 VALUES(NULL, 'four');
          227  +    INSERT INTO t1 VALUES(NULL, 'five');
          228  +    INSERT INTO t1 VALUES(NULL, 'six');
          229  +
          230  +    PRAGMA main.page_size;
          231  +    PRAGMA main.auto_vacuum;
          232  +    PRAGMA main.user_version;
          233  +    PRAGMA main.application_id;
          234  +  } {8192 2 412 413}
          235  +
          236  +  do_rbu_vacuum_test 1.9.1 $step
          237  +  do_execsql_test 1.9.2 {
          238  +    PRAGMA main.page_size;
          239  +    PRAGMA main.auto_vacuum;
          240  +    PRAGMA main.user_version;
          241  +    PRAGMA main.application_id;
          242  +  } {8192 2 412 413}
          243  +
          244  +  # Vacuum a database with a large sqlite_master table.
          245  +  #
          246  +  reset_db
          247  +  do_test 1.10.1 {
          248  +    for {set i 1} {$i < 50} {incr i} {
          249  +      execsql "PRAGMA page_size = 1024"
          250  +      execsql "CREATE TABLE t$i (a, b, c, PRIMARY KEY(a, b));"
          251  +      execsql "
          252  +        INSERT INTO t$i VALUES(1, 2, 3);
          253  +        INSERT INTO t$i VALUES(4, 5, 6);
          254  +      "
          255  +    }
          256  +  } {}
          257  +  do_rbu_vacuum_test 1.10.2 $step
          258  +}
          259  +
          260  +#-------------------------------------------------------------------------
          261  +# Test some error cases:
          262  +#
          263  +#   2.1.* the db being vacuumed being in wal mode already.
          264  +#   2.2.* database modified mid vacuum.
          265  +#
          266  +reset_db
          267  +do_execsql_test 2.1.0 {
          268  +  CREATE TABLE t1(a, b);
          269  +  INSERT INTO t1 VALUES(1, 2);
          270  +  INSERT INTO t1 VALUES(3, 4);
          271  +  INSERT INTO t1 VALUES(5, 6);
          272  +  INSERT INTO t1 VALUES(7, 8);
          273  +  PRAGMA journal_mode = wal;
          274  +  INSERT INTO t1 VALUES(9, 10);
          275  +} wal
          276  +do_test 2.1.1 {
          277  +  sqlite3rbu_vacuum rbu test.db state.db
          278  +  rbu step
          279  +} {SQLITE_ERROR}
          280  +do_test 2.1.2 {
          281  +  list [catch { rbu close } msg] $msg
          282  +} {1 {SQLITE_ERROR - cannot vacuum wal mode database}}
          283  +
          284  +reset_db
          285  +do_execsql_test 2.2.0 {
          286  +  CREATE TABLE tx(a PRIMARY KEY, b BLOB);
          287  +  INSERT INTO tx VALUES(1, randomblob(900));
          288  +  INSERT INTO tx SELECT a+1, randomblob(900) FROM tx;
          289  +  INSERT INTO tx SELECT a+2, randomblob(900) FROM tx;
          290  +  INSERT INTO tx SELECT a+4, randomblob(900) FROM tx;
          291  +  INSERT INTO tx SELECT a+8, randomblob(900) FROM tx;
          292  +}
          293  +db_save_and_close
          294  +for {set i 1} 1 {incr i} {
          295  +  db_restore_and_reopen
          296  +
          297  +  sqlite3rbu_vacuum rbu test.db state.db
          298  +  for {set step 0} {$step<$i} {incr step} { rbu step }
          299  +  rbu close
          300  +  if {[file exists test.db-wal]} break
          301  +
          302  +  execsql { INSERT INTO tx VALUES(20, 20) }
          303  +
          304  +  do_test 2.2.$i.1 {
          305  +    sqlite3rbu_vacuum rbu test.db state.db 
          306  +    rbu step
          307  +  } {SQLITE_BUSY}
          308  +  do_test 2.2.$i.2 {
          309  +    list [catch { rbu close } msg] $msg
          310  +  } {1 {SQLITE_BUSY - database modified during rbu vacuum}}
          311  +}
          312  +
          313  +catch { db close }
          314  +finish_test
          315  +

Changes to ext/rbu/sqlite3rbu.c.

   172    172   typedef struct RbuState RbuState;
   173    173   typedef struct rbu_vfs rbu_vfs;
   174    174   typedef struct rbu_file rbu_file;
   175    175   typedef struct RbuUpdateStmt RbuUpdateStmt;
   176    176   
   177    177   #if !defined(SQLITE_AMALGAMATION)
   178    178   typedef unsigned int u32;
          179  +typedef unsigned short u16;
   179    180   typedef unsigned char u8;
   180    181   typedef sqlite3_int64 i64;
   181    182   #endif
   182    183   
   183    184   /*
   184    185   ** These values must match the values defined in wal.c for the equivalent
   185    186   ** locks. These are not magic numbers as they are part of the SQLite file
   186    187   ** format.
   187    188   */
   188    189   #define WAL_LOCK_WRITE  0
   189    190   #define WAL_LOCK_CKPT   1
   190    191   #define WAL_LOCK_READ0  3
          192  +
          193  +#define SQLITE_FCNTL_RBUCNT    5149216
   191    194   
   192    195   /*
   193    196   ** A structure to store values read from the rbu_state table in memory.
   194    197   */
   195    198   struct RbuState {
   196    199     int eStage;
   197    200     char *zTbl;
................................................................................
   363    366     u32 mLock;
   364    367     int nFrame;                     /* Entries in aFrame[] array */
   365    368     int nFrameAlloc;                /* Allocated size of aFrame[] array */
   366    369     RbuFrame *aFrame;
   367    370     int pgsz;
   368    371     u8 *aBuf;
   369    372     i64 iWalCksum;
          373  +
          374  +  /* Used in RBU vacuum mode only */
          375  +  int nRbu;                       /* Number of RBU VFS in the stack */
          376  +  rbu_file *pRbuFd;               /* Fd for main db of dbRbu */
   370    377   };
   371    378   
   372    379   /*
   373    380   ** An rbu VFS is implemented using an instance of this structure.
   374    381   */
   375    382   struct rbu_vfs {
   376    383     sqlite3_vfs base;               /* rbu VFS shim methods */
................................................................................
   388    395     sqlite3_file *pReal;            /* Underlying file handle */
   389    396     rbu_vfs *pRbuVfs;               /* Pointer to the rbu_vfs object */
   390    397     sqlite3rbu *pRbu;               /* Pointer to rbu object (rbu target only) */
   391    398   
   392    399     int openFlags;                  /* Flags this file was opened with */
   393    400     u32 iCookie;                    /* Cookie value for main db files */
   394    401     u8 iWriteVer;                   /* "write-version" value for main db files */
          402  +  u8 bNolock;                     /* True to fail EXCLUSIVE locks */
   395    403   
   396    404     int nShm;                       /* Number of entries in apShm[] array */
   397    405     char **apShm;                   /* Array of mmap'd *-shm regions */
   398    406     char *zDel;                     /* Delete this when closing file */
   399    407   
   400    408     const char *zWal;               /* Wal filename for this main db file */
   401    409     rbu_file *pWalFd;               /* Wal file descriptor for this main db */
   402    410     rbu_file *pMainNext;            /* Next MAIN_DB file */
   403    411   };
   404    412   
          413  +/*
          414  +** True for an RBU vacuum handle, or false otherwise.
          415  +*/
          416  +#define rbuIsVacuum(p) ((p)->zTarget==0)
          417  +
   405    418   
   406    419   /*************************************************************************
   407    420   ** The following three functions, found below:
   408    421   **
   409    422   **   rbuDeltaGetInt()
   410    423   **   rbuDeltaChecksum()
   411    424   **   rbuDeltaApply()
................................................................................
   846    859     }
   847    860     return rc;
   848    861   }
   849    862   
   850    863   
   851    864   /*
   852    865   ** The implementation of the rbu_target_name() SQL function. This function
   853         -** accepts one argument - the name of a table in the RBU database. If the
   854         -** table name matches the pattern:
          866  +** accepts one or two arguments. The first argument is the name of a table -
          867  +** the name of a table in the RBU database.  The second, if it is present, is 1
          868  +** for a view or 0 for a table. 
          869  +**
          870  +** For a non-vacuum RBU handle, if the table name matches the pattern:
   855    871   **
   856    872   **     data[0-9]_<name>
   857    873   **
   858    874   ** where <name> is any sequence of 1 or more characters, <name> is returned.
   859    875   ** Otherwise, if the only argument does not match the above pattern, an SQL
   860    876   ** NULL is returned.
   861    877   **
   862    878   **     "data_t1"     -> "t1"
   863    879   **     "data0123_t2" -> "t2"
   864    880   **     "dataAB_t3"   -> NULL
          881  +**
          882  +** For an rbu vacuum handle, a copy of the first argument is returned if
          883  +** the second argument is either missing or 0 (not a view).
   865    884   */
   866    885   static void rbuTargetNameFunc(
   867         -  sqlite3_context *context,
          886  +  sqlite3_context *pCtx,
   868    887     int argc,
   869    888     sqlite3_value **argv
   870    889   ){
          890  +  sqlite3rbu *p = sqlite3_user_data(pCtx);
   871    891     const char *zIn;
   872         -  assert( argc==1 );
          892  +  assert( argc==1 || argc==2 );
   873    893   
   874    894     zIn = (const char*)sqlite3_value_text(argv[0]);
   875         -  if( zIn && strlen(zIn)>4 && memcmp("data", zIn, 4)==0 ){
   876         -    int i;
   877         -    for(i=4; zIn[i]>='0' && zIn[i]<='9'; i++);
   878         -    if( zIn[i]=='_' && zIn[i+1] ){
   879         -      sqlite3_result_text(context, &zIn[i+1], -1, SQLITE_STATIC);
          895  +  if( zIn ){
          896  +    if( rbuIsVacuum(p) ){
          897  +      if( argc==1 || 0==sqlite3_value_int(argv[1]) ){
          898  +        sqlite3_result_text(pCtx, zIn, -1, SQLITE_STATIC);
          899  +      }
          900  +    }else{
          901  +      if( strlen(zIn)>4 && memcmp("data", zIn, 4)==0 ){
          902  +        int i;
          903  +        for(i=4; zIn[i]>='0' && zIn[i]<='9'; i++);
          904  +        if( zIn[i]=='_' && zIn[i+1] ){
          905  +          sqlite3_result_text(pCtx, &zIn[i+1], -1, SQLITE_STATIC);
          906  +        }
          907  +      }
   880    908       }
   881    909     }
   882    910   }
   883    911   
   884    912   /*
   885    913   ** Initialize the iterator structure passed as the second argument.
   886    914   **
................................................................................
   890    918   ** error code is returned.
   891    919   */
   892    920   static int rbuObjIterFirst(sqlite3rbu *p, RbuObjIter *pIter){
   893    921     int rc;
   894    922     memset(pIter, 0, sizeof(RbuObjIter));
   895    923   
   896    924     rc = prepareAndCollectError(p->dbRbu, &pIter->pTblIter, &p->zErrmsg, 
   897         -      "SELECT rbu_target_name(name) AS target, name FROM sqlite_master "
          925  +      "SELECT rbu_target_name(name, type='view') AS target, name "
          926  +      "FROM sqlite_master "
   898    927         "WHERE type IN ('table', 'view') AND target IS NOT NULL "
   899    928         "ORDER BY name"
   900    929     );
   901    930   
   902    931     if( rc==SQLITE_OK ){
   903    932       rc = prepareAndCollectError(p->dbMain, &pIter->pIdxIter, &p->zErrmsg,
   904    933           "SELECT name, rootpage, sql IS NULL OR substr(8, 6)=='UNIQUE' "
................................................................................
  1266   1295           bRbuRowid = 1;
  1267   1296         }
  1268   1297       }
  1269   1298       sqlite3_finalize(pStmt);
  1270   1299       pStmt = 0;
  1271   1300   
  1272   1301       if( p->rc==SQLITE_OK
         1302  +     && rbuIsVacuum(p)==0
  1273   1303        && bRbuRowid!=(pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE)
  1274   1304       ){
  1275   1305         p->rc = SQLITE_ERROR;
  1276   1306         p->zErrmsg = sqlite3_mprintf(
  1277   1307             "table %q %s rbu_rowid column", pIter->zDataTbl,
  1278   1308             (bRbuRowid ? "may not have" : "requires")
  1279   1309         );
................................................................................
  1405   1435         /* An integer primary key. If the table has an explicit IPK, use
  1406   1436         ** its name. Otherwise, use "rbu_rowid".  */
  1407   1437         if( pIter->eType==RBU_PK_IPK ){
  1408   1438           int i;
  1409   1439           for(i=0; pIter->abTblPk[i]==0; i++);
  1410   1440           assert( i<pIter->nTblCol );
  1411   1441           zCol = pIter->azTblCol[i];
         1442  +      }else if( rbuIsVacuum(p) ){
         1443  +        zCol = "_rowid_";
  1412   1444         }else{
  1413   1445           zCol = "rbu_rowid";
  1414   1446         }
  1415   1447         zType = "INTEGER";
  1416   1448       }else{
  1417   1449         zCol = pIter->azTblCol[iCid];
  1418   1450         zType = pIter->azTblType[iCid];
................................................................................
  1945   1977           p->rc = prepareFreeAndCollectError(
  1946   1978               p->dbMain, &pIter->pInsert, &p->zErrmsg,
  1947   1979             sqlite3_mprintf("INSERT INTO \"rbu_imp_%w\" VALUES(%s)", zTbl, zBind)
  1948   1980           );
  1949   1981         }
  1950   1982   
  1951   1983         /* And to delete index entries */
  1952         -      if( p->rc==SQLITE_OK ){
         1984  +      if( rbuIsVacuum(p)==0 && p->rc==SQLITE_OK ){
  1953   1985           p->rc = prepareFreeAndCollectError(
  1954   1986               p->dbMain, &pIter->pDelete, &p->zErrmsg,
  1955   1987             sqlite3_mprintf("DELETE FROM \"rbu_imp_%w\" WHERE %s", zTbl, zWhere)
  1956   1988           );
  1957   1989         }
  1958   1990   
  1959   1991         /* Create the SELECT statement to read keys in sorted order */
  1960   1992         if( p->rc==SQLITE_OK ){
  1961   1993           char *zSql;
         1994  +        if( rbuIsVacuum(p) ){
         1995  +          zSql = sqlite3_mprintf(
         1996  +              "SELECT %s, 0 AS rbu_control FROM '%q' ORDER BY %s%s",
         1997  +              zCollist, 
         1998  +              pIter->zDataTbl,
         1999  +              zCollist, zLimit
         2000  +          );
         2001  +        }else
         2002  +
  1962   2003           if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){
  1963   2004             zSql = sqlite3_mprintf(
  1964   2005                 "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' ORDER BY %s%s",
  1965   2006                 zCollist, p->zStateDb, pIter->zDataTbl,
  1966   2007                 zCollist, zLimit
  1967   2008             );
  1968   2009           }else{
................................................................................
  1981   2022         }
  1982   2023   
  1983   2024         sqlite3_free(zImposterCols);
  1984   2025         sqlite3_free(zImposterPK);
  1985   2026         sqlite3_free(zWhere);
  1986   2027         sqlite3_free(zBind);
  1987   2028       }else{
  1988         -      int bRbuRowid = (pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE);
         2029  +      int bRbuRowid = (pIter->eType==RBU_PK_VTAB)
         2030  +                    ||(pIter->eType==RBU_PK_NONE)
         2031  +                    ||(pIter->eType==RBU_PK_EXTERNAL && rbuIsVacuum(p));
  1989   2032         const char *zTbl = pIter->zTbl;       /* Table this step applies to */
  1990   2033         const char *zWrite;                   /* Imposter table name */
  1991   2034   
  1992   2035         char *zBindings = rbuObjIterGetBindlist(p, pIter->nTblCol + bRbuRowid);
  1993   2036         char *zWhere = rbuObjIterGetWhere(p, pIter);
  1994   2037         char *zOldlist = rbuObjIterGetOldlist(p, pIter, "old");
  1995   2038         char *zNewlist = rbuObjIterGetOldlist(p, pIter, "new");
................................................................................
  2008   2051               sqlite3_mprintf(
  2009   2052                 "INSERT INTO \"%s%w\"(%s%s) VALUES(%s)", 
  2010   2053                 zWrite, zTbl, zCollist, (bRbuRowid ? ", _rowid_" : ""), zBindings
  2011   2054               )
  2012   2055           );
  2013   2056         }
  2014   2057   
  2015         -      /* Create the DELETE statement to write to the target PK b-tree */
  2016         -      if( p->rc==SQLITE_OK ){
         2058  +      /* Create the DELETE statement to write to the target PK b-tree.
         2059  +      ** Because it only performs INSERT operations, this is not required for
         2060  +      ** an rbu vacuum handle.  */
         2061  +      if( rbuIsVacuum(p)==0 && p->rc==SQLITE_OK ){
  2017   2062           p->rc = prepareFreeAndCollectError(p->dbMain, &pIter->pDelete, pz,
  2018   2063               sqlite3_mprintf(
  2019   2064                 "DELETE FROM \"%s%w\" WHERE %s", zWrite, zTbl, zWhere
  2020   2065               )
  2021   2066           );
  2022   2067         }
  2023   2068   
  2024         -      if( pIter->abIndexed ){
         2069  +      if( rbuIsVacuum(p)==0 && pIter->abIndexed ){
  2025   2070           const char *zRbuRowid = "";
  2026   2071           if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){
  2027   2072             zRbuRowid = ", rbu_rowid";
  2028   2073           }
  2029   2074   
  2030   2075           /* Create the rbu_tmp_xxx table and the triggers to populate it. */
  2031   2076           rbuMPrintfExec(p, p->dbRbu,
................................................................................
  2067   2112           }
  2068   2113   
  2069   2114           rbuObjIterPrepareTmpInsert(p, pIter, zCollist, zRbuRowid);
  2070   2115         }
  2071   2116   
  2072   2117         /* Create the SELECT statement to read keys from data_xxx */
  2073   2118         if( p->rc==SQLITE_OK ){
         2119  +        const char *zRbuRowid = "";
         2120  +        if( bRbuRowid ){
         2121  +          zRbuRowid = rbuIsVacuum(p) ? ",_rowid_ " : ",rbu_rowid";
         2122  +        }
  2074   2123           p->rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pSelect, pz,
  2075   2124               sqlite3_mprintf(
  2076         -              "SELECT %s, rbu_control%s FROM '%q'%s", 
  2077         -              zCollist, (bRbuRowid ? ", rbu_rowid" : ""), 
         2125  +              "SELECT %s,%s rbu_control%s FROM '%q'%s", 
         2126  +              zCollist, 
         2127  +              (rbuIsVacuum(p) ? "0 AS " : ""),
         2128  +              zRbuRowid,
  2078   2129                 pIter->zDataTbl, zLimit
  2079   2130               )
  2080   2131           );
  2081   2132         }
  2082   2133   
  2083   2134         sqlite3_free(zWhere);
  2084   2135         sqlite3_free(zOldlist);
................................................................................
  2165   2216       sqlite3_free(zWhere);
  2166   2217       sqlite3_free(zSet);
  2167   2218     }
  2168   2219   
  2169   2220     return p->rc;
  2170   2221   }
  2171   2222   
  2172         -static sqlite3 *rbuOpenDbhandle(sqlite3rbu *p, const char *zName){
         2223  +static sqlite3 *rbuOpenDbhandle(
         2224  +  sqlite3rbu *p, 
         2225  +  const char *zName, 
         2226  +  int bUseVfs
         2227  +){
  2173   2228     sqlite3 *db = 0;
  2174   2229     if( p->rc==SQLITE_OK ){
  2175   2230       const int flags = SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_URI;
  2176         -    p->rc = sqlite3_open_v2(zName, &db, flags, p->zVfsName);
         2231  +    p->rc = sqlite3_open_v2(zName, &db, flags, bUseVfs ? p->zVfsName : 0);
  2177   2232       if( p->rc ){
  2178   2233         p->zErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db));
  2179   2234         sqlite3_close(db);
  2180   2235         db = 0;
  2181   2236       }
  2182   2237     }
  2183   2238     return db;
  2184   2239   }
         2240  +
         2241  +/*
         2242  +** Free an RbuState object allocated by rbuLoadState().
         2243  +*/
         2244  +static void rbuFreeState(RbuState *p){
         2245  +  if( p ){
         2246  +    sqlite3_free(p->zTbl);
         2247  +    sqlite3_free(p->zIdx);
         2248  +    sqlite3_free(p);
         2249  +  }
         2250  +}
         2251  +
         2252  +/*
         2253  +** Allocate an RbuState object and load the contents of the rbu_state 
         2254  +** table into it. Return a pointer to the new object. It is the 
         2255  +** responsibility of the caller to eventually free the object using
         2256  +** sqlite3_free().
         2257  +**
         2258  +** If an error occurs, leave an error code and message in the rbu handle
         2259  +** and return NULL.
         2260  +*/
         2261  +static RbuState *rbuLoadState(sqlite3rbu *p){
         2262  +  RbuState *pRet = 0;
         2263  +  sqlite3_stmt *pStmt = 0;
         2264  +  int rc;
         2265  +  int rc2;
         2266  +
         2267  +  pRet = (RbuState*)rbuMalloc(p, sizeof(RbuState));
         2268  +  if( pRet==0 ) return 0;
         2269  +
         2270  +  rc = prepareFreeAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg, 
         2271  +      sqlite3_mprintf("SELECT k, v FROM %s.rbu_state", p->zStateDb)
         2272  +  );
         2273  +  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
         2274  +    switch( sqlite3_column_int(pStmt, 0) ){
         2275  +      case RBU_STATE_STAGE:
         2276  +        pRet->eStage = sqlite3_column_int(pStmt, 1);
         2277  +        if( pRet->eStage!=RBU_STAGE_OAL
         2278  +         && pRet->eStage!=RBU_STAGE_MOVE
         2279  +         && pRet->eStage!=RBU_STAGE_CKPT
         2280  +        ){
         2281  +          p->rc = SQLITE_CORRUPT;
         2282  +        }
         2283  +        break;
         2284  +
         2285  +      case RBU_STATE_TBL:
         2286  +        pRet->zTbl = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc);
         2287  +        break;
         2288  +
         2289  +      case RBU_STATE_IDX:
         2290  +        pRet->zIdx = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc);
         2291  +        break;
         2292  +
         2293  +      case RBU_STATE_ROW:
         2294  +        pRet->nRow = sqlite3_column_int(pStmt, 1);
         2295  +        break;
         2296  +
         2297  +      case RBU_STATE_PROGRESS:
         2298  +        pRet->nProgress = sqlite3_column_int64(pStmt, 1);
         2299  +        break;
         2300  +
         2301  +      case RBU_STATE_CKPT:
         2302  +        pRet->iWalCksum = sqlite3_column_int64(pStmt, 1);
         2303  +        break;
         2304  +
         2305  +      case RBU_STATE_COOKIE:
         2306  +        pRet->iCookie = (u32)sqlite3_column_int64(pStmt, 1);
         2307  +        break;
         2308  +
         2309  +      case RBU_STATE_OALSZ:
         2310  +        pRet->iOalSz = (u32)sqlite3_column_int64(pStmt, 1);
         2311  +        break;
         2312  +
         2313  +      case RBU_STATE_PHASEONESTEP:
         2314  +        pRet->nPhaseOneStep = sqlite3_column_int64(pStmt, 1);
         2315  +        break;
         2316  +
         2317  +      default:
         2318  +        rc = SQLITE_CORRUPT;
         2319  +        break;
         2320  +    }
         2321  +  }
         2322  +  rc2 = sqlite3_finalize(pStmt);
         2323  +  if( rc==SQLITE_OK ) rc = rc2;
         2324  +
         2325  +  p->rc = rc;
         2326  +  return pRet;
         2327  +}
         2328  +
  2185   2329   
  2186   2330   /*
  2187   2331   ** Open the database handle and attach the RBU database as "rbu". If an
  2188   2332   ** error occurs, leave an error code and message in the RBU handle.
  2189   2333   */
  2190   2334   static void rbuOpenDatabase(sqlite3rbu *p){
  2191   2335     assert( p->rc==SQLITE_OK );
  2192   2336     assert( p->dbMain==0 && p->dbRbu==0 );
         2337  +  assert( rbuIsVacuum(p) || p->zTarget!=0 );
  2193   2338   
  2194         -  p->eStage = 0;
  2195         -  p->dbMain = rbuOpenDbhandle(p, p->zTarget);
  2196         -  p->dbRbu = rbuOpenDbhandle(p, p->zRbu);
         2339  +  /* Open the RBU database */
         2340  +  p->dbRbu = rbuOpenDbhandle(p, p->zRbu, 1);
         2341  +
         2342  +  if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){
         2343  +    sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_RBUCNT, (void*)p);
         2344  +  }
  2197   2345   
  2198   2346     /* If using separate RBU and state databases, attach the state database to
  2199   2347     ** the RBU db handle now.  */
  2200   2348     if( p->zState ){
  2201   2349       rbuMPrintfExec(p, p->dbRbu, "ATTACH %Q AS stat", p->zState);
  2202   2350       memcpy(p->zStateDb, "stat", 4);
  2203   2351     }else{
  2204   2352       memcpy(p->zStateDb, "main", 4);
  2205   2353     }
         2354  +
         2355  +#if 0
         2356  +  if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){
         2357  +    p->rc = sqlite3_exec(p->dbRbu, "BEGIN", 0, 0, 0);
         2358  +  }
         2359  +#endif
         2360  +
         2361  +  /* If it has not already been created, create the rbu_state table */
         2362  +  rbuMPrintfExec(p, p->dbRbu, RBU_CREATE_STATE, p->zStateDb);
         2363  +
         2364  +#if 0
         2365  +  if( rbuIsVacuum(p) ){
         2366  +    if( p->rc==SQLITE_OK ){
         2367  +      int rc2;
         2368  +      int bOk = 0;
         2369  +      sqlite3_stmt *pCnt = 0;
         2370  +      p->rc = prepareAndCollectError(p->dbRbu, &pCnt, &p->zErrmsg,
         2371  +          "SELECT count(*) FROM stat.sqlite_master"
         2372  +      );
         2373  +      if( p->rc==SQLITE_OK 
         2374  +       && sqlite3_step(pCnt)==SQLITE_ROW
         2375  +       && 1==sqlite3_column_int(pCnt, 0)
         2376  +      ){
         2377  +        bOk = 1;
         2378  +      }
         2379  +      rc2 = sqlite3_finalize(pCnt);
         2380  +      if( p->rc==SQLITE_OK ) p->rc = rc2;
         2381  +
         2382  +      if( p->rc==SQLITE_OK && bOk==0 ){
         2383  +        p->rc = SQLITE_ERROR;
         2384  +        p->zErrmsg = sqlite3_mprintf("invalid state database");
         2385  +      }
         2386  +    
         2387  +      if( p->rc==SQLITE_OK ){
         2388  +        p->rc = sqlite3_exec(p->dbRbu, "COMMIT", 0, 0, 0);
         2389  +      }
         2390  +    }
         2391  +  }
         2392  +#endif
         2393  +
         2394  +  if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){
         2395  +    int bOpen = 0;
         2396  +    int rc;
         2397  +    p->nRbu = 0;
         2398  +    p->pRbuFd = 0;
         2399  +    rc = sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_RBUCNT, (void*)p);
         2400  +    if( rc!=SQLITE_NOTFOUND ) p->rc = rc;
         2401  +    if( p->eStage>=RBU_STAGE_MOVE ){
         2402  +      bOpen = 1;
         2403  +    }else{
         2404  +      RbuState *pState = rbuLoadState(p);
         2405  +      if( pState ){
         2406  +        bOpen = (pState->eStage>RBU_STAGE_MOVE);
         2407  +        rbuFreeState(pState);
         2408  +      }
         2409  +    }
         2410  +    if( bOpen ) p->dbMain = rbuOpenDbhandle(p, p->zRbu, p->nRbu<=1);
         2411  +  }
         2412  +
         2413  +  p->eStage = 0;
         2414  +  if( p->rc==SQLITE_OK && p->dbMain==0 ){
         2415  +    if( !rbuIsVacuum(p) ){
         2416  +      p->dbMain = rbuOpenDbhandle(p, p->zTarget, 1);
         2417  +    }else if( p->pRbuFd->pWalFd ){
         2418  +      p->rc = SQLITE_ERROR;
         2419  +      p->zErrmsg = sqlite3_mprintf("cannot vacuum wal mode database");
         2420  +    }else{
         2421  +      char *zTarget;
         2422  +      char *zExtra = 0;
         2423  +      if( strlen(p->zRbu)>=5 && 0==memcmp("file:", p->zRbu, 5) ){
         2424  +        zExtra = &p->zRbu[5];
         2425  +        while( *zExtra ){
         2426  +          if( *zExtra++=='?' ) break;
         2427  +        }
         2428  +        if( *zExtra=='\0' ) zExtra = 0;
         2429  +      }
         2430  +
         2431  +      zTarget = sqlite3_mprintf("file:%s-vacuum?rbu_memory=1%s%s", 
         2432  +          sqlite3_db_filename(p->dbRbu, "main"),
         2433  +          (zExtra==0 ? "" : "&"), (zExtra==0 ? "" : zExtra)
         2434  +      );
         2435  +
         2436  +      if( zTarget==0 ){
         2437  +        p->rc = SQLITE_NOMEM;
         2438  +        return;
         2439  +      }
         2440  +      p->dbMain = rbuOpenDbhandle(p, zTarget, p->nRbu<=1);
         2441  +      sqlite3_free(zTarget);
         2442  +    }
         2443  +  }
  2206   2444   
  2207   2445     if( p->rc==SQLITE_OK ){
  2208   2446       p->rc = sqlite3_create_function(p->dbMain, 
  2209   2447           "rbu_tmp_insert", -1, SQLITE_UTF8, (void*)p, rbuTmpInsertFunc, 0, 0
  2210   2448       );
  2211   2449     }
  2212   2450   
................................................................................
  2214   2452       p->rc = sqlite3_create_function(p->dbMain, 
  2215   2453           "rbu_fossil_delta", 2, SQLITE_UTF8, 0, rbuFossilDeltaFunc, 0, 0
  2216   2454       );
  2217   2455     }
  2218   2456   
  2219   2457     if( p->rc==SQLITE_OK ){
  2220   2458       p->rc = sqlite3_create_function(p->dbRbu, 
  2221         -        "rbu_target_name", 1, SQLITE_UTF8, (void*)p, rbuTargetNameFunc, 0, 0
         2459  +        "rbu_target_name", -1, SQLITE_UTF8, (void*)p, rbuTargetNameFunc, 0, 0
  2222   2460       );
  2223   2461     }
  2224   2462   
  2225   2463     if( p->rc==SQLITE_OK ){
  2226   2464       p->rc = sqlite3_file_control(p->dbMain, "main", SQLITE_FCNTL_RBU, (void*)p);
  2227   2465     }
  2228   2466     rbuMPrintfExec(p, p->dbMain, "SELECT * FROM sqlite_master");
................................................................................
  2473   2711   ** on the database file. This proc moves the *-oal file to the *-wal path,
  2474   2712   ** then reopens the database file (this time in vanilla, non-oal, WAL mode).
  2475   2713   ** If an error occurs, leave an error code and error message in the rbu 
  2476   2714   ** handle.
  2477   2715   */
  2478   2716   static void rbuMoveOalFile(sqlite3rbu *p){
  2479   2717     const char *zBase = sqlite3_db_filename(p->dbMain, "main");
         2718  +  const char *zMove = zBase;
         2719  +  char *zOal;
         2720  +  char *zWal;
  2480   2721   
  2481         -  char *zWal = sqlite3_mprintf("%s-wal", zBase);
  2482         -  char *zOal = sqlite3_mprintf("%s-oal", zBase);
         2722  +  if( rbuIsVacuum(p) ){
         2723  +    zMove = sqlite3_db_filename(p->dbRbu, "main");
         2724  +  }
         2725  +  zOal = sqlite3_mprintf("%s-oal", zMove);
         2726  +  zWal = sqlite3_mprintf("%s-wal", zMove);
  2483   2727   
  2484   2728     assert( p->eStage==RBU_STAGE_MOVE );
  2485   2729     assert( p->rc==SQLITE_OK && p->zErrmsg==0 );
  2486   2730     if( zWal==0 || zOal==0 ){
  2487   2731       p->rc = SQLITE_NOMEM;
  2488   2732     }else{
  2489   2733       /* Move the *-oal file to *-wal. At this point connection p->db is
................................................................................
  2496   2740       rbuLockDatabase(p);
  2497   2741       if( p->rc==SQLITE_OK ){
  2498   2742         rbuFileSuffix3(zBase, zWal);
  2499   2743         rbuFileSuffix3(zBase, zOal);
  2500   2744   
  2501   2745         /* Re-open the databases. */
  2502   2746         rbuObjIterFinalize(&p->objiter);
  2503         -      sqlite3_close(p->dbMain);
  2504   2747         sqlite3_close(p->dbRbu);
         2748  +      sqlite3_close(p->dbMain);
  2505   2749         p->dbMain = 0;
  2506   2750         p->dbRbu = 0;
  2507   2751   
  2508   2752   #if defined(_WIN32_WCE)
  2509   2753         {
  2510   2754           LPWSTR zWideOal;
  2511   2755           LPWSTR zWideWal;
................................................................................
  2659   2903         continue;
  2660   2904       }
  2661   2905   
  2662   2906       pVal = sqlite3_column_value(pIter->pSelect, i);
  2663   2907       p->rc = sqlite3_bind_value(pWriter, i+1, pVal);
  2664   2908       if( p->rc ) return;
  2665   2909     }
  2666         -  if( pIter->zIdx==0
  2667         -   && (pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE) 
  2668         -  ){
  2669         -    /* For a virtual table, or a table with no primary key, the 
  2670         -    ** SELECT statement is:
  2671         -    **
  2672         -    **   SELECT <cols>, rbu_control, rbu_rowid FROM ....
  2673         -    **
  2674         -    ** Hence column_value(pIter->nCol+1).
  2675         -    */
  2676         -    assertColumnName(pIter->pSelect, pIter->nCol+1, "rbu_rowid");
  2677         -    pVal = sqlite3_column_value(pIter->pSelect, pIter->nCol+1);
  2678         -    p->rc = sqlite3_bind_value(pWriter, pIter->nCol+1, pVal);
         2910  +  if( pIter->zIdx==0 ){
         2911  +    if( pIter->eType==RBU_PK_VTAB 
         2912  +     || pIter->eType==RBU_PK_NONE 
         2913  +     || (pIter->eType==RBU_PK_EXTERNAL && rbuIsVacuum(p)) 
         2914  +    ){
         2915  +      /* For a virtual table, or a table with no primary key, the 
         2916  +      ** SELECT statement is:
         2917  +      **
         2918  +      **   SELECT <cols>, rbu_control, rbu_rowid FROM ....
         2919  +      **
         2920  +      ** Hence column_value(pIter->nCol+1).
         2921  +      */
         2922  +      assertColumnName(pIter->pSelect, pIter->nCol+1, 
         2923  +          rbuIsVacuum(p) ? "rowid" : "rbu_rowid"
         2924  +      );
         2925  +      pVal = sqlite3_column_value(pIter->pSelect, pIter->nCol+1);
         2926  +      p->rc = sqlite3_bind_value(pWriter, pIter->nCol+1, pVal);
         2927  +    }
  2679   2928     }
  2680   2929     if( p->rc==SQLITE_OK ){
  2681   2930       sqlite3_step(pWriter);
  2682   2931       p->rc = resetAndCollectError(pWriter, &p->zErrmsg);
  2683   2932     }
  2684   2933   }
  2685   2934   
................................................................................
  2750   2999       }
  2751   3000     }
  2752   3001     return p->rc;
  2753   3002   }
  2754   3003   
  2755   3004   /*
  2756   3005   ** Increment the schema cookie of the main database opened by p->dbMain.
         3006  +**
         3007  +** Or, if this is an RBU vacuum, set the schema cookie of the main db
         3008  +** opened by p->dbMain to one more than the schema cookie of the main
         3009  +** db opened by p->dbRbu.
  2757   3010   */
  2758   3011   static void rbuIncrSchemaCookie(sqlite3rbu *p){
  2759   3012     if( p->rc==SQLITE_OK ){
         3013  +    sqlite3 *dbread = (rbuIsVacuum(p) ? p->dbRbu : p->dbMain);
  2760   3014       int iCookie = 1000000;
  2761   3015       sqlite3_stmt *pStmt;
  2762   3016   
  2763         -    p->rc = prepareAndCollectError(p->dbMain, &pStmt, &p->zErrmsg, 
         3017  +    p->rc = prepareAndCollectError(dbread, &pStmt, &p->zErrmsg, 
  2764   3018           "PRAGMA schema_version"
  2765   3019       );
  2766   3020       if( p->rc==SQLITE_OK ){
  2767   3021         /* Coverage: it may be that this sqlite3_step() cannot fail. There
  2768   3022         ** is already a transaction open, so the prepared statement cannot
  2769   3023         ** throw an SQLITE_SCHEMA exception. The only database page the
  2770   3024         ** statement reads is page 1, which is guaranteed to be in the cache.
................................................................................
  2784   3038   ** Update the contents of the rbu_state table within the rbu database. The
  2785   3039   ** value stored in the RBU_STATE_STAGE column is eStage. All other values
  2786   3040   ** are determined by inspecting the rbu handle passed as the first argument.
  2787   3041   */
  2788   3042   static void rbuSaveState(sqlite3rbu *p, int eStage){
  2789   3043     if( p->rc==SQLITE_OK || p->rc==SQLITE_DONE ){
  2790   3044       sqlite3_stmt *pInsert = 0;
         3045  +    rbu_file *pFd = (rbuIsVacuum(p) ? p->pRbuFd : p->pTargetFd);
  2791   3046       int rc;
  2792   3047   
  2793   3048       assert( p->zErrmsg==0 );
  2794   3049       rc = prepareFreeAndCollectError(p->dbRbu, &pInsert, &p->zErrmsg, 
  2795   3050           sqlite3_mprintf(
  2796   3051             "INSERT OR REPLACE INTO %s.rbu_state(k, v) VALUES "
  2797   3052             "(%d, %d), "
................................................................................
  2806   3061             p->zStateDb,
  2807   3062             RBU_STATE_STAGE, eStage,
  2808   3063             RBU_STATE_TBL, p->objiter.zTbl, 
  2809   3064             RBU_STATE_IDX, p->objiter.zIdx, 
  2810   3065             RBU_STATE_ROW, p->nStep, 
  2811   3066             RBU_STATE_PROGRESS, p->nProgress,
  2812   3067             RBU_STATE_CKPT, p->iWalCksum,
  2813         -          RBU_STATE_COOKIE, (i64)p->pTargetFd->iCookie,
         3068  +          RBU_STATE_COOKIE, (i64)pFd->iCookie,
  2814   3069             RBU_STATE_OALSZ, p->iOalSz,
  2815   3070             RBU_STATE_PHASEONESTEP, p->nPhaseOneStep
  2816   3071         )
  2817   3072       );
  2818   3073       assert( pInsert==0 || rc==SQLITE_OK );
  2819   3074   
  2820   3075       if( rc==SQLITE_OK ){
................................................................................
  2836   3091           RbuObjIter *pIter = &p->objiter;
  2837   3092           while( p->rc==SQLITE_OK && pIter->zTbl ){
  2838   3093   
  2839   3094             if( pIter->bCleanup ){
  2840   3095               /* Clean up the rbu_tmp_xxx table for the previous table. It 
  2841   3096               ** cannot be dropped as there are currently active SQL statements.
  2842   3097               ** But the contents can be deleted.  */
  2843         -            if( pIter->abIndexed ){
         3098  +            if( rbuIsVacuum(p)==0 && pIter->abIndexed ){
  2844   3099                 rbuMPrintfExec(p, p->dbRbu, 
  2845   3100                     "DELETE FROM %s.'rbu_tmp_%q'", p->zStateDb, pIter->zDataTbl
  2846   3101                 );
  2847   3102               }
  2848   3103             }else{
  2849   3104               rbuObjIterPrepareAll(p, pIter, 0);
  2850   3105   
................................................................................
  2923   3178       }
  2924   3179       return p->rc;
  2925   3180     }else{
  2926   3181       return SQLITE_NOMEM;
  2927   3182     }
  2928   3183   }
  2929   3184   
  2930         -/*
  2931         -** Free an RbuState object allocated by rbuLoadState().
  2932         -*/
  2933         -static void rbuFreeState(RbuState *p){
  2934         -  if( p ){
  2935         -    sqlite3_free(p->zTbl);
  2936         -    sqlite3_free(p->zIdx);
  2937         -    sqlite3_free(p);
  2938         -  }
  2939         -}
  2940         -
  2941         -/*
  2942         -** Allocate an RbuState object and load the contents of the rbu_state 
  2943         -** table into it. Return a pointer to the new object. It is the 
  2944         -** responsibility of the caller to eventually free the object using
  2945         -** sqlite3_free().
  2946         -**
  2947         -** If an error occurs, leave an error code and message in the rbu handle
  2948         -** and return NULL.
  2949         -*/
  2950         -static RbuState *rbuLoadState(sqlite3rbu *p){
  2951         -  RbuState *pRet = 0;
  2952         -  sqlite3_stmt *pStmt = 0;
  2953         -  int rc;
  2954         -  int rc2;
  2955         -
  2956         -  pRet = (RbuState*)rbuMalloc(p, sizeof(RbuState));
  2957         -  if( pRet==0 ) return 0;
  2958         -
  2959         -  rc = prepareFreeAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg, 
  2960         -      sqlite3_mprintf("SELECT k, v FROM %s.rbu_state", p->zStateDb)
  2961         -  );
  2962         -  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
  2963         -    switch( sqlite3_column_int(pStmt, 0) ){
  2964         -      case RBU_STATE_STAGE:
  2965         -        pRet->eStage = sqlite3_column_int(pStmt, 1);
  2966         -        if( pRet->eStage!=RBU_STAGE_OAL
  2967         -         && pRet->eStage!=RBU_STAGE_MOVE
  2968         -         && pRet->eStage!=RBU_STAGE_CKPT
  2969         -        ){
  2970         -          p->rc = SQLITE_CORRUPT;
  2971         -        }
  2972         -        break;
  2973         -
  2974         -      case RBU_STATE_TBL:
  2975         -        pRet->zTbl = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc);
  2976         -        break;
  2977         -
  2978         -      case RBU_STATE_IDX:
  2979         -        pRet->zIdx = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc);
  2980         -        break;
  2981         -
  2982         -      case RBU_STATE_ROW:
  2983         -        pRet->nRow = sqlite3_column_int(pStmt, 1);
  2984         -        break;
  2985         -
  2986         -      case RBU_STATE_PROGRESS:
  2987         -        pRet->nProgress = sqlite3_column_int64(pStmt, 1);
  2988         -        break;
  2989         -
  2990         -      case RBU_STATE_CKPT:
  2991         -        pRet->iWalCksum = sqlite3_column_int64(pStmt, 1);
  2992         -        break;
  2993         -
  2994         -      case RBU_STATE_COOKIE:
  2995         -        pRet->iCookie = (u32)sqlite3_column_int64(pStmt, 1);
  2996         -        break;
  2997         -
  2998         -      case RBU_STATE_OALSZ:
  2999         -        pRet->iOalSz = (u32)sqlite3_column_int64(pStmt, 1);
  3000         -        break;
  3001         -
  3002         -      case RBU_STATE_PHASEONESTEP:
  3003         -        pRet->nPhaseOneStep = sqlite3_column_int64(pStmt, 1);
  3004         -        break;
  3005         -
  3006         -      default:
  3007         -        rc = SQLITE_CORRUPT;
  3008         -        break;
  3009         -    }
  3010         -  }
  3011         -  rc2 = sqlite3_finalize(pStmt);
  3012         -  if( rc==SQLITE_OK ) rc = rc2;
  3013         -
  3014         -  p->rc = rc;
  3015         -  return pRet;
  3016         -}
  3017         -
  3018   3185   /*
  3019   3186   ** Compare strings z1 and z2, returning 0 if they are identical, or non-zero
  3020   3187   ** otherwise. Either or both argument may be NULL. Two NULL values are
  3021   3188   ** considered equal, and NULL is considered distinct from all other values.
  3022   3189   */
  3023   3190   static int rbuStrCompare(const char *z1, const char *z2){
  3024   3191     if( z1==0 && z2==0 ) return 0;
................................................................................
  3201   3368           p->rc = sqlite3_finalize(pStmt);
  3202   3369         }
  3203   3370       }
  3204   3371     }
  3205   3372   }
  3206   3373   
  3207   3374   /*
  3208         -** Open and return a new RBU handle. 
         3375  +** The second argument passed to this function is the name of a PRAGMA 
         3376  +** setting - "page_size", "auto_vacuum", "user_version" or "application_id".
         3377  +** This function executes the following on sqlite3rbu.dbRbu:
         3378  +**
         3379  +**   "PRAGMA main.$zPragma"
         3380  +**
         3381  +** where $zPragma is the string passed as the second argument, then
         3382  +** on sqlite3rbu.dbMain:
         3383  +**
         3384  +**   "PRAGMA main.$zPragma = $val"
         3385  +**
         3386  +** where $val is the value returned by the first PRAGMA invocation.
         3387  +**
         3388  +** In short, it copies the value  of the specified PRAGMA setting from
         3389  +** dbRbu to dbMain.
  3209   3390   */
  3210         -sqlite3rbu *sqlite3rbu_open(
         3391  +static void rbuCopyPragma(sqlite3rbu *p, const char *zPragma){
         3392  +  if( p->rc==SQLITE_OK ){
         3393  +    sqlite3_stmt *pPragma = 0;
         3394  +    p->rc = prepareFreeAndCollectError(p->dbRbu, &pPragma, &p->zErrmsg, 
         3395  +        sqlite3_mprintf("PRAGMA main.%s", zPragma)
         3396  +    );
         3397  +    if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPragma) ){
         3398  +      p->rc = rbuMPrintfExec(p, p->dbMain, "PRAGMA main.%s = %d",
         3399  +          zPragma, sqlite3_column_int(pPragma, 0)
         3400  +      );
         3401  +    }
         3402  +    rbuFinalize(p, pPragma);
         3403  +  }
         3404  +}
         3405  +
         3406  +/*
         3407  +** The RBU handle passed as the only argument has just been opened and 
         3408  +** the state database is empty. If this RBU handle was opened for an
         3409  +** RBU vacuum operation, create the schema in the target db.
         3410  +*/
         3411  +static void rbuCreateTargetSchema(sqlite3rbu *p){
         3412  +  sqlite3_stmt *pSql = 0;
         3413  +  sqlite3_stmt *pInsert = 0;
         3414  +
         3415  +  assert( rbuIsVacuum(p) );
         3416  +  p->rc = sqlite3_exec(p->dbMain, "PRAGMA writable_schema=1", 0,0, &p->zErrmsg);
         3417  +  if( p->rc==SQLITE_OK ){
         3418  +    p->rc = prepareAndCollectError(p->dbRbu, &pSql, &p->zErrmsg, 
         3419  +      "SELECT sql FROM sqlite_master WHERE sql!='' AND rootpage!=0"
         3420  +      " AND name!='sqlite_sequence' "
         3421  +      " ORDER BY type DESC"
         3422  +    );
         3423  +  }
         3424  +
         3425  +  while( p->rc==SQLITE_OK && sqlite3_step(pSql)==SQLITE_ROW ){
         3426  +    const char *zSql = (const char*)sqlite3_column_text(pSql, 0);
         3427  +    p->rc = sqlite3_exec(p->dbMain, zSql, 0, 0, &p->zErrmsg);
         3428  +  }
         3429  +  rbuFinalize(p, pSql);
         3430  +  if( p->rc!=SQLITE_OK ) return;
         3431  +
         3432  +  if( p->rc==SQLITE_OK ){
         3433  +    p->rc = prepareAndCollectError(p->dbRbu, &pSql, &p->zErrmsg, 
         3434  +        "SELECT * FROM sqlite_master WHERE rootpage=0 OR rootpage IS NULL" 
         3435  +    );
         3436  +  }
         3437  +
         3438  +  if( p->rc==SQLITE_OK ){
         3439  +    p->rc = prepareAndCollectError(p->dbMain, &pInsert, &p->zErrmsg, 
         3440  +        "INSERT INTO sqlite_master VALUES(?,?,?,?,?)"
         3441  +    );
         3442  +  }
         3443  +
         3444  +  while( p->rc==SQLITE_OK && sqlite3_step(pSql)==SQLITE_ROW ){
         3445  +    int i;
         3446  +    for(i=0; i<5; i++){
         3447  +      sqlite3_bind_value(pInsert, i+1, sqlite3_column_value(pSql, i));
         3448  +    }
         3449  +    sqlite3_step(pInsert);
         3450  +    p->rc = sqlite3_reset(pInsert);
         3451  +  }
         3452  +  if( p->rc==SQLITE_OK ){
         3453  +    p->rc = sqlite3_exec(p->dbMain, "PRAGMA writable_schema=0",0,0,&p->zErrmsg);
         3454  +  }
         3455  +
         3456  +  rbuFinalize(p, pSql);
         3457  +  rbuFinalize(p, pInsert);
         3458  +}
         3459  +
         3460  +
         3461  +static sqlite3rbu *openRbuHandle(
  3211   3462     const char *zTarget, 
  3212   3463     const char *zRbu,
  3213   3464     const char *zState
  3214   3465   ){
  3215   3466     sqlite3rbu *p;
  3216         -  size_t nTarget = strlen(zTarget);
         3467  +  size_t nTarget = zTarget ? strlen(zTarget) : 0;
  3217   3468     size_t nRbu = strlen(zRbu);
  3218   3469     size_t nState = zState ? strlen(zState) : 0;
  3219   3470     size_t nByte = sizeof(sqlite3rbu) + nTarget+1 + nRbu+1+ nState+1;
  3220   3471   
  3221   3472     p = (sqlite3rbu*)sqlite3_malloc64(nByte);
  3222   3473     if( p ){
  3223   3474       RbuState *pState = 0;
  3224   3475   
  3225   3476       /* Create the custom VFS. */
  3226   3477       memset(p, 0, sizeof(sqlite3rbu));
  3227   3478       rbuCreateVfs(p);
  3228   3479   
  3229         -    /* Open the target database */
         3480  +    /* Open the target, RBU and state databases */
  3230   3481       if( p->rc==SQLITE_OK ){
  3231         -      p->zTarget = (char*)&p[1];
  3232         -      memcpy(p->zTarget, zTarget, nTarget+1);
  3233         -      p->zRbu = &p->zTarget[nTarget+1];
         3482  +      char *pCsr = (char*)&p[1];
         3483  +      if( zTarget ){
         3484  +        p->zTarget = pCsr;
         3485  +        memcpy(p->zTarget, zTarget, nTarget+1);
         3486  +        pCsr += nTarget+1;
         3487  +      }
         3488  +      p->zRbu = pCsr;
  3234   3489         memcpy(p->zRbu, zRbu, nRbu+1);
         3490  +      pCsr += nRbu+1;
  3235   3491         if( zState ){
  3236         -        p->zState = &p->zRbu[nRbu+1];
         3492  +        p->zState = pCsr;
  3237   3493           memcpy(p->zState, zState, nState+1);
  3238   3494         }
  3239   3495         rbuOpenDatabase(p);
  3240   3496       }
  3241   3497   
  3242         -    /* If it has not already been created, create the rbu_state table */
  3243         -    rbuMPrintfExec(p, p->dbRbu, RBU_CREATE_STATE, p->zStateDb);
  3244         -
  3245   3498       if( p->rc==SQLITE_OK ){
  3246   3499         pState = rbuLoadState(p);
  3247   3500         assert( pState || p->rc!=SQLITE_OK );
  3248   3501         if( p->rc==SQLITE_OK ){
  3249   3502   
  3250   3503           if( pState->eStage==0 ){ 
  3251   3504             rbuDeleteOalFile(p);
................................................................................
  3267   3520           p->zErrmsg = sqlite3_mprintf("cannot update wal mode database");
  3268   3521         }else if( p->eStage==RBU_STAGE_MOVE ){
  3269   3522           p->eStage = RBU_STAGE_CKPT;
  3270   3523           p->nStep = 0;
  3271   3524         }
  3272   3525       }
  3273   3526   
  3274         -    if( p->rc==SQLITE_OK
         3527  +    if( p->rc==SQLITE_OK 
  3275   3528        && (p->eStage==RBU_STAGE_OAL || p->eStage==RBU_STAGE_MOVE)
  3276         -     && pState->eStage!=0 && p->pTargetFd->iCookie!=pState->iCookie
  3277         -    ){   
  3278         -      /* At this point (pTargetFd->iCookie) contains the value of the
  3279         -      ** change-counter cookie (the thing that gets incremented when a 
  3280         -      ** transaction is committed in rollback mode) currently stored on 
  3281         -      ** page 1 of the database file. */
  3282         -      p->rc = SQLITE_BUSY;
  3283         -      p->zErrmsg = sqlite3_mprintf("database modified during rbu update");
         3529  +     && pState->eStage!=0
         3530  +    ){
         3531  +      rbu_file *pFd = (rbuIsVacuum(p) ? p->pRbuFd : p->pTargetFd);
         3532  +      if( pFd->iCookie!=pState->iCookie ){   
         3533  +        /* At this point (pTargetFd->iCookie) contains the value of the
         3534  +        ** change-counter cookie (the thing that gets incremented when a 
         3535  +        ** transaction is committed in rollback mode) currently stored on 
         3536  +        ** page 1 of the database file. */
         3537  +        p->rc = SQLITE_BUSY;
         3538  +        p->zErrmsg = sqlite3_mprintf("database modified during rbu %s",
         3539  +            (rbuIsVacuum(p) ? "vacuum" : "update")
         3540  +        );
         3541  +      }
  3284   3542       }
  3285   3543   
  3286   3544       if( p->rc==SQLITE_OK ){
  3287   3545         if( p->eStage==RBU_STAGE_OAL ){
  3288   3546           sqlite3 *db = p->dbMain;
         3547  +
         3548  +        if( pState->eStage==0 && rbuIsVacuum(p) ){
         3549  +          rbuCopyPragma(p, "page_size");
         3550  +          rbuCopyPragma(p, "auto_vacuum");
         3551  +        }
  3289   3552   
  3290   3553           /* Open transactions both databases. The *-oal file is opened or
  3291   3554           ** created at this point. */
  3292         -        p->rc = sqlite3_exec(db, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg);
         3555  +        if( p->rc==SQLITE_OK ){
         3556  +          p->rc = sqlite3_exec(db, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg);
         3557  +        }
  3293   3558           if( p->rc==SQLITE_OK ){
  3294         -          p->rc = sqlite3_exec(p->dbRbu, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg);
         3559  +          p->rc = sqlite3_exec(p->dbRbu, "BEGIN", 0, 0, &p->zErrmsg);
  3295   3560           }
  3296   3561   
  3297   3562           /* Check if the main database is a zipvfs db. If it is, set the upper
  3298   3563           ** level pager to use "journal_mode=off". This prevents it from 
  3299   3564           ** generating a large journal using a temp file.  */
  3300   3565           if( p->rc==SQLITE_OK ){
  3301   3566             int frc = sqlite3_file_control(db, "main", SQLITE_FCNTL_ZIPVFS, 0);
  3302   3567             if( frc==SQLITE_OK ){
  3303   3568               p->rc = sqlite3_exec(db, "PRAGMA journal_mode=off",0,0,&p->zErrmsg);
  3304   3569             }
  3305   3570           }
         3571  +
         3572  +        /* If this is an RBU vacuum operation and the state table was empty
         3573  +        ** when this handle was opened, create the target database schema. */
         3574  +        if( p->rc==SQLITE_OK && pState->eStage==0 && rbuIsVacuum(p) ){
         3575  +          rbuCreateTargetSchema(p);
         3576  +          rbuCopyPragma(p, "user_version");
         3577  +          rbuCopyPragma(p, "application_id");
         3578  +        }
  3306   3579   
  3307   3580           /* Point the object iterator at the first object */
  3308   3581           if( p->rc==SQLITE_OK ){
  3309   3582             p->rc = rbuObjIterFirst(p, &p->objiter);
  3310   3583           }
  3311   3584   
  3312   3585           /* If the RBU database contains no data_xxx tables, declare the RBU
................................................................................
  3332   3605   
  3333   3606       rbuFreeState(pState);
  3334   3607     }
  3335   3608   
  3336   3609     return p;
  3337   3610   }
  3338   3611   
         3612  +/*
         3613  +** Open and return a new RBU handle. 
         3614  +*/
         3615  +sqlite3rbu *sqlite3rbu_open(
         3616  +  const char *zTarget, 
         3617  +  const char *zRbu,
         3618  +  const char *zState
         3619  +){
         3620  +  /* TODO: Check that zTarget and zRbu are non-NULL */
         3621  +  return openRbuHandle(zTarget, zRbu, zState);
         3622  +}
         3623  +
         3624  +/*
         3625  +** Open a handle to begin or resume an RBU VACUUM operation.
         3626  +*/
         3627  +sqlite3rbu *sqlite3rbu_vacuum(
         3628  +  const char *zTarget, 
         3629  +  const char *zState
         3630  +){
         3631  +  /* TODO: Check that both arguments are non-NULL */
         3632  +  return openRbuHandle(0, zTarget, zState);
         3633  +}
  3339   3634   
  3340   3635   /*
  3341   3636   ** Return the database handle used by pRbu.
  3342   3637   */
  3343   3638   sqlite3 *sqlite3rbu_db(sqlite3rbu *pRbu, int bRbu){
  3344   3639     sqlite3 *db = 0;
  3345   3640     if( pRbu ){
................................................................................
  3385   3680   
  3386   3681       if( p->rc==SQLITE_OK && p->eStage==RBU_STAGE_OAL ){
  3387   3682         p->rc = sqlite3_exec(p->dbRbu, "COMMIT", 0, 0, &p->zErrmsg);
  3388   3683       }
  3389   3684   
  3390   3685       /* Close any open statement handles. */
  3391   3686       rbuObjIterFinalize(&p->objiter);
         3687  +
         3688  +    /* If this is an RBU vacuum handle and the vacuum has either finished
         3689  +    ** successfully or encountered an error, delete the contents of the 
         3690  +    ** state table. This causes the next call to sqlite3rbu_vacuum() 
         3691  +    ** specifying the current target and state databases to start a new
         3692  +    ** vacuum from scratch.  */
         3693  +    if( rbuIsVacuum(p) && p->rc!=SQLITE_OK && p->dbRbu ){
         3694  +      int rc2 = sqlite3_exec(p->dbRbu, "DELETE FROM stat.rbu_state", 0, 0, 0);
         3695  +      if( p->rc==SQLITE_DONE && rc2!=SQLITE_OK ) p->rc = rc2;
         3696  +    }
  3392   3697   
  3393   3698       /* Close the open database handle and VFS object. */
  3394         -    sqlite3_close(p->dbMain);
  3395   3699       sqlite3_close(p->dbRbu);
         3700  +    sqlite3_close(p->dbMain);
  3396   3701       rbuDeleteVfs(p);
  3397   3702       sqlite3_free(p->aBuf);
  3398   3703       sqlite3_free(p->aFrame);
  3399   3704   
  3400   3705       rbuEditErrmsg(p);
  3401   3706       rc = p->rc;
  3402   3707       *pzErrmsg = p->zErrmsg;
................................................................................
  3589   3894   */
  3590   3895   static u32 rbuGetU32(u8 *aBuf){
  3591   3896     return ((u32)aBuf[0] << 24)
  3592   3897          + ((u32)aBuf[1] << 16)
  3593   3898          + ((u32)aBuf[2] <<  8)
  3594   3899          + ((u32)aBuf[3]);
  3595   3900   }
         3901  +
         3902  +/*
         3903  +** Write an unsigned 32-bit value in big-endian format to the supplied
         3904  +** buffer.
         3905  +*/
         3906  +static void rbuPutU32(u8 *aBuf, u32 iVal){
         3907  +  aBuf[0] = (iVal >> 24) & 0xFF;
         3908  +  aBuf[1] = (iVal >> 16) & 0xFF;
         3909  +  aBuf[2] = (iVal >>  8) & 0xFF;
         3910  +  aBuf[3] = (iVal >>  0) & 0xFF;
         3911  +}
         3912  +
         3913  +static void rbuPutU16(u8 *aBuf, u16 iVal){
         3914  +  aBuf[0] = (iVal >>  8) & 0xFF;
         3915  +  aBuf[1] = (iVal >>  0) & 0xFF;
         3916  +}
  3596   3917   
  3597   3918   /*
  3598   3919   ** Read data from an rbuVfs-file.
  3599   3920   */
  3600   3921   static int rbuVfsRead(
  3601   3922     sqlite3_file *pFile, 
  3602   3923     void *zBuf, 
................................................................................
  3615   3936        && (p->openFlags & SQLITE_OPEN_WAL) 
  3616   3937        && iOfst>=pRbu->iOalSz 
  3617   3938       ){
  3618   3939         rc = SQLITE_OK;
  3619   3940         memset(zBuf, 0, iAmt);
  3620   3941       }else{
  3621   3942         rc = p->pReal->pMethods->xRead(p->pReal, zBuf, iAmt, iOfst);
         3943  +#if 1
         3944  +      /* If this is being called to read the first page of the target 
         3945  +      ** database as part of an rbu vacuum operation, synthesize the 
         3946  +      ** contents of the first page if it does not yet exist. Otherwise,
         3947  +      ** SQLite will not check for a *-wal file.  */
         3948  +      if( pRbu && rbuIsVacuum(pRbu) 
         3949  +          && rc==SQLITE_IOERR_SHORT_READ && iOfst==0
         3950  +          && (p->openFlags & SQLITE_OPEN_MAIN_DB)
         3951  +          && pRbu->rc==SQLITE_OK
         3952  +      ){
         3953  +        sqlite3_file *pFd = (sqlite3_file*)pRbu->pRbuFd;
         3954  +        rc = pFd->pMethods->xRead(pFd, zBuf, iAmt, iOfst);
         3955  +        if( rc==SQLITE_OK ){
         3956  +          u8 *aBuf = (u8*)zBuf;
         3957  +          u32 iRoot = rbuGetU32(&aBuf[52]) ? 1 : 0;
         3958  +          rbuPutU32(&aBuf[52], iRoot);      /* largest root page number */
         3959  +          rbuPutU32(&aBuf[36], 0);          /* number of free pages */
         3960  +          rbuPutU32(&aBuf[32], 0);          /* first page on free list trunk */
         3961  +          rbuPutU32(&aBuf[28], 1);          /* size of db file in pages */
         3962  +          rbuPutU32(&aBuf[24], pRbu->pRbuFd->iCookie+1);  /* Change counter */
         3963  +
         3964  +          if( iAmt>100 ){
         3965  +            memset(&aBuf[100], 0, iAmt-100);
         3966  +            rbuPutU16(&aBuf[105], iAmt & 0xFFFF);
         3967  +            aBuf[100] = 0x0D;
         3968  +          }
         3969  +        }
         3970  +      }
         3971  +#endif
  3622   3972       }
  3623   3973       if( rc==SQLITE_OK && iOfst==0 && (p->openFlags & SQLITE_OPEN_MAIN_DB) ){
  3624   3974         /* These look like magic numbers. But they are stable, as they are part
  3625   3975          ** of the definition of the SQLite file format, which may not change. */
  3626   3976         u8 *pBuf = (u8*)zBuf;
  3627   3977         p->iCookie = rbuGetU32(&pBuf[24]);
  3628   3978         p->iWriteVer = pBuf[19];
................................................................................
  3689   4039   }
  3690   4040   
  3691   4041   /*
  3692   4042   ** Return the current file-size of an rbuVfs-file.
  3693   4043   */
  3694   4044   static int rbuVfsFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){
  3695   4045     rbu_file *p = (rbu_file *)pFile;
  3696         -  return p->pReal->pMethods->xFileSize(p->pReal, pSize);
         4046  +  int rc;
         4047  +  rc = p->pReal->pMethods->xFileSize(p->pReal, pSize);
         4048  +
         4049  +  /* If this is an RBU vacuum operation and this is the target database,
         4050  +  ** pretend that it has at least one page. Otherwise, SQLite will not
         4051  +  ** check for the existance of a *-wal file. rbuVfsRead() contains 
         4052  +  ** similar logic.  */
         4053  +  if( rc==SQLITE_OK && *pSize==0 
         4054  +   && p->pRbu && rbuIsVacuum(p->pRbu) 
         4055  +   && (p->openFlags & SQLITE_OPEN_MAIN_DB)
         4056  +  ){
         4057  +    *pSize = 1024;
         4058  +  }
         4059  +  return rc;
  3697   4060   }
  3698   4061   
  3699   4062   /*
  3700   4063   ** Lock an rbuVfs-file.
  3701   4064   */
  3702   4065   static int rbuVfsLock(sqlite3_file *pFile, int eLock){
  3703   4066     rbu_file *p = (rbu_file*)pFile;
  3704   4067     sqlite3rbu *pRbu = p->pRbu;
  3705   4068     int rc = SQLITE_OK;
  3706   4069   
  3707   4070     assert( p->openFlags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_TEMP_DB) );
  3708         -  if( pRbu && eLock==SQLITE_LOCK_EXCLUSIVE && pRbu->eStage!=RBU_STAGE_DONE ){
         4071  +  if( eLock==SQLITE_LOCK_EXCLUSIVE 
         4072  +   && (p->bNolock || (pRbu && pRbu->eStage!=RBU_STAGE_DONE))
         4073  +  ){
  3709   4074       /* Do not allow EXCLUSIVE locks. Preventing SQLite from taking this 
  3710   4075       ** prevents it from checkpointing the database from sqlite3_close(). */
  3711   4076       rc = SQLITE_BUSY;
  3712   4077     }else{
  3713   4078       rc = p->pReal->pMethods->xLock(p->pReal, eLock);
  3714   4079     }
  3715   4080   
................................................................................
  3763   4128           pRbu->pTargetFd = p;
  3764   4129           p->pRbu = pRbu;
  3765   4130           if( p->pWalFd ) p->pWalFd->pRbu = pRbu;
  3766   4131           rc = SQLITE_OK;
  3767   4132         }
  3768   4133       }
  3769   4134       return rc;
         4135  +  }
         4136  +  else if( op==SQLITE_FCNTL_RBUCNT ){
         4137  +    sqlite3rbu *pRbu = (sqlite3rbu*)pArg;
         4138  +    pRbu->nRbu++;
         4139  +    pRbu->pRbuFd = p;
         4140  +    p->bNolock = 1;
  3770   4141     }
  3771   4142   
  3772   4143     rc = xControl(p->pReal, op, pArg);
  3773   4144     if( rc==SQLITE_OK && op==SQLITE_FCNTL_VFSNAME ){
  3774   4145       rbu_vfs *pRbuVfs = p->pRbuVfs;
  3775   4146       char *zIn = *(char**)pArg;
  3776   4147       char *zOut = sqlite3_mprintf("rbu(%s)/%z", pRbuVfs->base.zName, zIn);
................................................................................
  3926   4297   static rbu_file *rbuFindMaindb(rbu_vfs *pRbuVfs, const char *zWal){
  3927   4298     rbu_file *pDb;
  3928   4299     sqlite3_mutex_enter(pRbuVfs->mutex);
  3929   4300     for(pDb=pRbuVfs->pMain; pDb && pDb->zWal!=zWal; pDb=pDb->pMainNext){}
  3930   4301     sqlite3_mutex_leave(pRbuVfs->mutex);
  3931   4302     return pDb;
  3932   4303   }
         4304  +
         4305  +/* 
         4306  +** A main database named zName has just been opened. The following 
         4307  +** function returns a pointer to a buffer owned by SQLite that contains
         4308  +** the name of the *-wal file this db connection will use. SQLite
         4309  +** happens to pass a pointer to this buffer when using xAccess()
         4310  +** or xOpen() to operate on the *-wal file.  
         4311  +*/
         4312  +static const char *rbuMainToWal(const char *zName, int flags){
         4313  +  int n = (int)strlen(zName);
         4314  +  const char *z = &zName[n];
         4315  +  if( flags & SQLITE_OPEN_URI ){
         4316  +    int odd = 0;
         4317  +    while( 1 ){
         4318  +      if( z[0]==0 ){
         4319  +        odd = 1 - odd;
         4320  +        if( odd && z[1]==0 ) break;
         4321  +      }
         4322  +      z++;
         4323  +    }
         4324  +    z += 2;
         4325  +  }else{
         4326  +    while( *z==0 ) z++;
         4327  +  }
         4328  +  z += (n + 8 + 1);
         4329  +  return z;
         4330  +}
  3933   4331   
  3934   4332   /*
  3935   4333   ** Open an rbu file handle.
  3936   4334   */
  3937   4335   static int rbuVfsOpen(
  3938   4336     sqlite3_vfs *pVfs,
  3939   4337     const char *zName,
................................................................................
  3962   4360       0, 0                          /* xFetch, xUnfetch */
  3963   4361     };
  3964   4362     rbu_vfs *pRbuVfs = (rbu_vfs*)pVfs;
  3965   4363     sqlite3_vfs *pRealVfs = pRbuVfs->pRealVfs;
  3966   4364     rbu_file *pFd = (rbu_file *)pFile;
  3967   4365     int rc = SQLITE_OK;
  3968   4366     const char *zOpen = zName;
         4367  +  int oflags = flags;
  3969   4368   
  3970   4369     memset(pFd, 0, sizeof(rbu_file));
  3971   4370     pFd->pReal = (sqlite3_file*)&pFd[1];
  3972   4371     pFd->pRbuVfs = pRbuVfs;
  3973   4372     pFd->openFlags = flags;
  3974   4373     if( zName ){
  3975   4374       if( flags & SQLITE_OPEN_MAIN_DB ){
  3976   4375         /* A main database has just been opened. The following block sets
  3977   4376         ** (pFd->zWal) to point to a buffer owned by SQLite that contains
  3978   4377         ** the name of the *-wal file this db connection will use. SQLite
  3979   4378         ** happens to pass a pointer to this buffer when using xAccess()
  3980   4379         ** or xOpen() to operate on the *-wal file.  */
  3981         -      int n = (int)strlen(zName);
  3982         -      const char *z = &zName[n];
  3983         -      if( flags & SQLITE_OPEN_URI ){
  3984         -        int odd = 0;
  3985         -        while( 1 ){
  3986         -          if( z[0]==0 ){
  3987         -            odd = 1 - odd;
  3988         -            if( odd && z[1]==0 ) break;
  3989         -          }
  3990         -          z++;
  3991         -        }
  3992         -        z += 2;
  3993         -      }else{
  3994         -        while( *z==0 ) z++;
  3995         -      }
  3996         -      z += (n + 8 + 1);
  3997         -      pFd->zWal = z;
         4380  +      pFd->zWal = rbuMainToWal(zName, flags);
  3998   4381       }
  3999   4382       else if( flags & SQLITE_OPEN_WAL ){
  4000   4383         rbu_file *pDb = rbuFindMaindb(pRbuVfs, zName);
  4001   4384         if( pDb ){
  4002   4385           if( pDb->pRbu && pDb->pRbu->eStage==RBU_STAGE_OAL ){
  4003   4386             /* This call is to open a *-wal file. Intead, open the *-oal. This
  4004   4387             ** code ensures that the string passed to xOpen() is terminated by a
  4005   4388             ** pair of '\0' bytes in case the VFS attempts to extract a URI 
  4006   4389             ** parameter from it.  */
  4007         -          size_t nCopy = strlen(zName);
  4008         -          char *zCopy = sqlite3_malloc64(nCopy+2);
         4390  +          const char *zBase = zName;
         4391  +          size_t nCopy;
         4392  +          char *zCopy;
         4393  +          if( rbuIsVacuum(pDb->pRbu) ){
         4394  +            zBase = sqlite3_db_filename(pDb->pRbu->dbRbu, "main");
         4395  +            zBase = rbuMainToWal(zBase, SQLITE_OPEN_URI);
         4396  +          }
         4397  +          nCopy = strlen(zBase);
         4398  +          zCopy = sqlite3_malloc64(nCopy+2);
  4009   4399             if( zCopy ){
  4010         -            memcpy(zCopy, zName, nCopy);
         4400  +            memcpy(zCopy, zBase, nCopy);
  4011   4401               zCopy[nCopy-3] = 'o';
  4012   4402               zCopy[nCopy] = '\0';
  4013   4403               zCopy[nCopy+1] = '\0';
  4014   4404               zOpen = (const char*)(pFd->zDel = zCopy);
  4015   4405             }else{
  4016   4406               rc = SQLITE_NOMEM;
  4017   4407             }
  4018   4408             pFd->pRbu = pDb->pRbu;
  4019   4409           }
  4020   4410           pDb->pWalFd = pFd;
  4021   4411         }
  4022   4412       }
  4023   4413     }
         4414  +
         4415  +  if( oflags & SQLITE_OPEN_MAIN_DB 
         4416  +   && sqlite3_uri_boolean(zName, "rbu_memory", 0) 
         4417  +  ){
         4418  +    assert( oflags & SQLITE_OPEN_MAIN_DB );
         4419  +    oflags =  SQLITE_OPEN_TEMP_DB | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE |
         4420  +              SQLITE_OPEN_EXCLUSIVE | SQLITE_OPEN_DELETEONCLOSE;
         4421  +    zOpen = 0;
         4422  +  }
  4024   4423   
  4025   4424     if( rc==SQLITE_OK ){
  4026         -    rc = pRealVfs->xOpen(pRealVfs, zOpen, pFd->pReal, flags, pOutFlags);
         4425  +    rc = pRealVfs->xOpen(pRealVfs, zOpen, pFd->pReal, oflags, pOutFlags);
  4027   4426     }
  4028   4427     if( pFd->pReal->pMethods ){
  4029   4428       /* The xOpen() operation has succeeded. Set the sqlite3_file.pMethods
  4030   4429       ** pointer and, if the file is a main database file, link it into the
  4031   4430       ** mutex protected linked list of all such files.  */
  4032   4431       pFile->pMethods = &rbuvfs_io_methods;
  4033   4432       if( flags & SQLITE_OPEN_MAIN_DB ){

Changes to ext/rbu/sqlite3rbu.h.

   309    309   ** the zipvfs_create_vfs() API below for details on using RBU with zipvfs.
   310    310   */
   311    311   sqlite3rbu *sqlite3rbu_open(
   312    312     const char *zTarget, 
   313    313     const char *zRbu,
   314    314     const char *zState
   315    315   );
          316  +
          317  +/*
          318  +** Open an RBU handle to perform an RBU vacuum on database file zTarget.
          319  +** An RBU vacuum is similar to SQLite's built-in VACUUM command, except
          320  +** that it can be suspended and resumed like an RBU update.
          321  +**
          322  +** The second argument to this function, which may not be NULL, identifies 
          323  +** a database in which to store the state of the RBU vacuum operation if
          324  +** it is suspended. The first time sqlite3rbu_vacuum() is called, to start
          325  +** an RBU vacuum operation, the state database should either not exist or
          326  +** be empty (contain no tables). If an RBU vacuum is suspended by calling
          327  +** sqlite3rbu_close() on the RBU handle before sqlite3rbu_step() has
          328  +** returned SQLITE_DONE, the vacuum state is stored in the state database. 
          329  +** The vacuum can be resumed by calling this function to open a new RBU
          330  +** handle specifying the same target and state databases.
          331  +**
          332  +** This function does not delete the state database after an RBU vacuum
          333  +** is completed, even if it created it. However, if the call to
          334  +** sqlite3rbu_close() returns any value other than SQLITE_OK, the contents
          335  +** of the state tables within the state database are zeroed. This way,
          336  +** the next call to sqlite3rbu_vacuum() opens a handle that starts a 
          337  +** new RBU vacuum operation.
          338  +**
          339  +** As with sqlite3rbu_open(), Zipvfs users should rever to the comment
          340  +** describing the sqlite3rbu_create_vfs() API function below for 
          341  +** a description of the complications associated with using RBU with 
          342  +** zipvfs databases.
          343  +*/
          344  +sqlite3rbu *sqlite3rbu_vacuum(
          345  +  const char *zTarget, 
          346  +  const char *zState
          347  +);
   316    348   
   317    349   /*
   318    350   ** Internally, each RBU connection uses a separate SQLite database 
   319    351   ** connection to access the target and rbu update databases. This
   320    352   ** API allows the application direct access to these database handles.
   321    353   **
   322    354   ** The first argument passed to this function must be a valid, open, RBU

Changes to ext/rbu/test_rbu.c.

   182    182     if( objc==5 ) zStateDb = Tcl_GetString(objv[4]);
   183    183   
   184    184     pRbu = sqlite3rbu_open(zTarget, zRbu, zStateDb);
   185    185     Tcl_CreateObjCommand(interp, zCmd, test_sqlite3rbu_cmd, (ClientData)pRbu, 0);
   186    186     Tcl_SetObjResult(interp, objv[1]);
   187    187     return TCL_OK;
   188    188   }
          189  +
          190  +/*
          191  +** Tclcmd: sqlite3rbu_vacuum CMD <target-db> <state-db>
          192  +*/
          193  +static int test_sqlite3rbu_vacuum(
          194  +  ClientData clientData,
          195  +  Tcl_Interp *interp,
          196  +  int objc,
          197  +  Tcl_Obj *CONST objv[]
          198  +){
          199  +  sqlite3rbu *pRbu = 0;
          200  +  const char *zCmd;
          201  +  const char *zTarget;
          202  +  const char *zStateDb = 0;
          203  +
          204  +  if( objc!=4 ){
          205  +    Tcl_WrongNumArgs(interp, 1, objv, "NAME TARGET-DB STATE-DB");
          206  +    return TCL_ERROR;
          207  +  }
          208  +  zCmd = Tcl_GetString(objv[1]);
          209  +  zTarget = Tcl_GetString(objv[2]);
          210  +  zStateDb = Tcl_GetString(objv[3]);
          211  +
          212  +  pRbu = sqlite3rbu_vacuum(zTarget, zStateDb);
          213  +  Tcl_CreateObjCommand(interp, zCmd, test_sqlite3rbu_cmd, (ClientData)pRbu, 0);
          214  +  Tcl_SetObjResult(interp, objv[1]);
          215  +  return TCL_OK;
          216  +}
   189    217   
   190    218   /*
   191    219   ** Tclcmd: sqlite3rbu_create_vfs ?-default? NAME PARENT
   192    220   */
   193    221   static int test_sqlite3rbu_create_vfs(
   194    222     ClientData clientData,
   195    223     Tcl_Interp *interp,
................................................................................
   270    298   
   271    299   int SqliteRbu_Init(Tcl_Interp *interp){ 
   272    300     static struct {
   273    301        char *zName;
   274    302        Tcl_ObjCmdProc *xProc;
   275    303     } aObjCmd[] = {
   276    304       { "sqlite3rbu", test_sqlite3rbu },
          305  +    { "sqlite3rbu_vacuum", test_sqlite3rbu_vacuum },
   277    306       { "sqlite3rbu_create_vfs", test_sqlite3rbu_create_vfs },
   278    307       { "sqlite3rbu_destroy_vfs", test_sqlite3rbu_destroy_vfs },
   279    308       { "sqlite3rbu_internal_test", test_sqlite3rbu_internal_test },
   280    309     };
   281    310     int i;
   282    311     for(i=0; i<sizeof(aObjCmd)/sizeof(aObjCmd[0]); i++){
   283    312       Tcl_CreateObjCommand(interp, aObjCmd[i].zName, aObjCmd[i].xProc, 0, 0);

Changes to src/expr.c.

  2356   2356       if( pParse->nTempReg<ArraySize(pParse->aTempReg) ){
  2357   2357         pParse->aTempReg[pParse->nTempReg++] = p->iReg;
  2358   2358       }
  2359   2359       p->tempReg = 0;
  2360   2360     }
  2361   2361     p->iReg = 0;
  2362   2362     pParse->nColCache--;
  2363         -  assert( cacheIsValid(pParse) );
         2363  +  assert( pParse->db->mallocFailed || cacheIsValid(pParse) );
  2364   2364   }
  2365   2365   
  2366   2366   
  2367   2367   /*
  2368   2368   ** Record in the column cache that a particular column from a
  2369   2369   ** particular table is stored in a particular register.
  2370   2370   */
................................................................................
  2401   2401         p->iLevel = pParse->iCacheLevel;
  2402   2402         p->iTable = iTab;
  2403   2403         p->iColumn = iCol;
  2404   2404         p->iReg = iReg;
  2405   2405         p->tempReg = 0;
  2406   2406         p->lru = pParse->iCacheCnt++;
  2407   2407         pParse->nColCache++;
  2408         -      assert( cacheIsValid(pParse) );
         2408  +      assert( pParse->db->mallocFailed || cacheIsValid(pParse) );
  2409   2409         return;
  2410   2410       }
  2411   2411     }
  2412   2412   
  2413   2413     /* Replace the last recently used */
  2414   2414     minLru = 0x7fffffff;
  2415   2415     idxLru = -1;

Changes to src/os_unix.c.

  5416   5416   
  5417   5417     if( !azDirs[0] ) azDirs[0] = getenv("SQLITE_TMPDIR");
  5418   5418     if( !azDirs[1] ) azDirs[1] = getenv("TMPDIR");
  5419   5419     for(i=0; i<sizeof(azDirs)/sizeof(azDirs[0]); zDir=azDirs[i++]){
  5420   5420       if( zDir==0 ) continue;
  5421   5421       if( osStat(zDir, &buf) ) continue;
  5422   5422       if( !S_ISDIR(buf.st_mode) ) continue;
  5423         -    if( osAccess(zDir, 07) ) continue;
  5424         -    break;
         5423  +    if( osAccess(zDir, 03) ) continue;
         5424  +    return zDir;
  5425   5425     }
  5426         -  return zDir;
         5426  +  return 0;
  5427   5427   }
  5428   5428   
  5429   5429   /*
  5430   5430   ** Create a temporary file name in zBuf.  zBuf must be allocated
  5431   5431   ** by the calling process and must be big enough to hold at least
  5432   5432   ** pVfs->mxPathname bytes.
  5433   5433   */
................................................................................
  5435   5435     const char *zDir;
  5436   5436     int iLimit = 0;
  5437   5437   
  5438   5438     /* It's odd to simulate an io-error here, but really this is just
  5439   5439     ** using the io-error infrastructure to test that SQLite handles this
  5440   5440     ** function failing. 
  5441   5441     */
         5442  +  zBuf[0] = 0;
  5442   5443     SimulateIOError( return SQLITE_IOERR );
  5443   5444   
  5444   5445     zDir = unixTempFileDir();
         5446  +  if( zDir==0 ) return SQLITE_IOERR_GETTEMPPATH;
  5445   5447     do{
  5446   5448       u64 r;
  5447   5449       sqlite3_randomness(sizeof(r), &r);
  5448   5450       assert( nBuf>2 );
  5449   5451       zBuf[nBuf-2] = 0;
  5450   5452       sqlite3_snprintf(nBuf, zBuf, "%s/"SQLITE_TEMP_FILE_PREFIX"%llx%c",
  5451   5453                        zDir, r, 0);