/ Check-in [564ae829]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix an RBU problem causing spurious SQLITE_CONSTRAINT errors when restarting an RBU update in which more than one source table writes to a single target database table.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256:564ae8297d417ba4b7978e430d41f125007177673163f6ed9adc3a3974f73d24
User & Date: dan 2018-04-28 18:20:01
Context
2018-04-28
19:08
Test cases added for SQLITE_DBCONFIG_RESET_DATABASE. check-in: 08665a9e user: drh tags: trunk
18:20
Fix an RBU problem causing spurious SQLITE_CONSTRAINT errors when restarting an RBU update in which more than one source table writes to a single target database table. check-in: 564ae829 user: dan tags: trunk
13:21
Add the SQLITE_DBCONFIG_RESET_DATABASE control for resetting a corrupt database file without closing any database connections. Added the ".dbconfig" command to the CLI. check-in: a200a49e user: drh tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/rbu/rbu1.test.

135
136
137
138
139
140
141

142
143
144
145
146
147
148
} {

  eval $create_vfs

  foreach {tn2 cmd} {
      1 run_rbu 
      2 step_rbu 3 step_rbu_uri 4 step_rbu_state

  } {
    foreach {tn schema} {
      1 {
        CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
      }
      2 { 
        CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);







>







135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
} {

  eval $create_vfs

  foreach {tn2 cmd} {
      1 run_rbu 
      2 step_rbu 3 step_rbu_uri 4 step_rbu_state
      5 step_rbu_legacy
  } {
    foreach {tn schema} {
      1 {
        CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
      }
      2 { 
        CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);

Changes to ext/rbu/rbu_common.tcl.

65
66
67
68
69
70
71
















72
73
74
75
76
77
78
    set rc [rbu step]
    check_poststep_state $rc $target $state
    rbu close
    if {$rc != "SQLITE_OK"} break
  }
  set rc
}

















proc do_rbu_vacuum_test {tn step} {
  forcedelete state.db
  uplevel [list do_test $tn.1 {
    if {$step==0} { sqlite3rbu_vacuum rbu test.db state.db }
    while 1 {
      if {$step==1} { sqlite3rbu_vacuum rbu test.db state.db }







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
    set rc [rbu step]
    check_poststep_state $rc $target $state
    rbu close
    if {$rc != "SQLITE_OK"} break
  }
  set rc
}

proc step_rbu_legacy {target rbu} {
  while 1 {
    sqlite3rbu rbu $target $rbu
    set state [rbu state]
    check_prestep_state $target $state
    set rc [rbu step]
    check_poststep_state $rc $target $state
    rbu close
    if {$rc != "SQLITE_OK"} break
    sqlite3 tmpdb $rbu
    tmpdb eval { DELETE FROM rbu_state WHERE k==10 }
    tmpdb close
  }
  set rc
}

proc do_rbu_vacuum_test {tn step} {
  forcedelete state.db
  uplevel [list do_test $tn.1 {
    if {$step==0} { sqlite3rbu_vacuum rbu test.db state.db }
    while 1 {
      if {$step==1} { sqlite3rbu_vacuum rbu test.db state.db }

Added ext/rbu/rbusplit.test.































































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
# 2018 April 28
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#
#

source [file join [file dirname [info script]] rbu_common.tcl]
set ::testprefix rbusplit

db close
sqlite3_shutdown
sqlite3_config_uri 1

autoinstall_test_functions

proc build_db {db} {
  $db eval {
    CREATE TABLE t1(a PRIMARY KEY, b, c);
    CREATE TABLE t2(a PRIMARY KEY, b, c);

    CREATE INDEX t1c ON t1(c);
  }
}

proc build_rbu {filename} {
  forcedelete $filename
  sqlite3 dbRbu $filename
  dbRbu eval {
    CREATE TABLE data0_t1(a, b, c, rbu_control);
    CREATE TABLE data1_t1(a, b, c, rbu_control);
    CREATE TABLE data2_t1(a, b, c, rbu_control);
    CREATE TABLE data3_t1(a, b, c, rbu_control);

    CREATE TABLE data_t2(a, b, c, rbu_control);

    INSERT INTO data0_t1 VALUES(1, 1, 1, 0);
    INSERT INTO data0_t1 VALUES(2, 2, 2, 0);
    INSERT INTO data0_t1 VALUES(3, 3, 3, 0);
    INSERT INTO data0_t1 VALUES(4, 4, 4, 0);
    INSERT INTO data1_t1 VALUES(5, 5, 5, 0);
    INSERT INTO data1_t1 VALUES(6, 6, 6, 0);
    INSERT INTO data1_t1 VALUES(7, 7, 7, 0);
    INSERT INTO data1_t1 VALUES(8, 8, 8, 0);
    INSERT INTO data3_t1 VALUES(9, 9, 9, 0);

    INSERT INTO data_t2 VALUES(1, 1, 1, 0);
    INSERT INTO data_t2 VALUES(2, 2, 2, 0);
    INSERT INTO data_t2 VALUES(3, 3, 3, 0);
    INSERT INTO data_t2 VALUES(4, 4, 4, 0);
    INSERT INTO data_t2 VALUES(5, 5, 5, 0);
    INSERT INTO data_t2 VALUES(6, 6, 6, 0);
    INSERT INTO data_t2 VALUES(7, 7, 7, 0);
    INSERT INTO data_t2 VALUES(8, 8, 8, 0);
    INSERT INTO data_t2 VALUES(9, 9, 9, 0);
  }
  
  dbRbu close
}

foreach {tn cmd} {
  1 run_rbu
  2 step_rbu
} {
  reset_db
  build_db db
  build_rbu testrbu.db

  do_test 1.$tn.1 {
    $cmd test.db testrbu.db
  } {SQLITE_DONE}
  do_execsql_test 1.$tn.1 {
    SELECT * FROM t1;
  } {
    1 1 1 2 2 2 3 3 3 4 4 4
    5 5 5 6 6 6 7 7 7 8 8 8
    9 9 9
  }
  do_execsql_test 1.$tn.2 {
    SELECT * FROM t2;
  } {
    1 1 1 2 2 2 3 3 3 4 4 4
    5 5 5 6 6 6 7 7 7 8 8 8
    9 9 9
  }
}

finish_test

Changes to ext/rbu/sqlite3rbu.c.

149
150
151
152
153
154
155




156
157
158
159
160
161
162
163
164
165

166
167
168
169
170
171
172
...
201
202
203
204
205
206
207

208
209
210
211
212
213
214
....
2264
2265
2266
2267
2268
2269
2270

2271
2272
2273
2274
2275
2276
2277
....
2333
2334
2335
2336
2337
2338
2339




2340
2341
2342
2343
2344
2345
2346
....
3108
3109
3110
3111
3112
3113
3114
3115

3116
3117
3118
3119
3120
3121
3122
3123
3124
3125

3126
3127
3128
3129
3130
3131
3132
....
3374
3375
3376
3377
3378
3379
3380
3381

3382
3383
3384
3385
3386
3387
3388
**
** RBU_STATE_COOKIE:
**   Valid if STAGE==1. The current change-counter cookie value in the 
**   target db file.
**
** RBU_STATE_OALSZ:
**   Valid if STAGE==1. The size in bytes of the *-oal file.




*/
#define RBU_STATE_STAGE        1
#define RBU_STATE_TBL          2
#define RBU_STATE_IDX          3
#define RBU_STATE_ROW          4
#define RBU_STATE_PROGRESS     5
#define RBU_STATE_CKPT         6
#define RBU_STATE_COOKIE       7
#define RBU_STATE_OALSZ        8
#define RBU_STATE_PHASEONESTEP 9


#define RBU_STAGE_OAL         1
#define RBU_STAGE_MOVE        2
#define RBU_STAGE_CAPTURE     3
#define RBU_STAGE_CKPT        4
#define RBU_STAGE_DONE        5

................................................................................

/*
** A structure to store values read from the rbu_state table in memory.
*/
struct RbuState {
  int eStage;
  char *zTbl;

  char *zIdx;
  i64 iWalCksum;
  int nRow;
  i64 nProgress;
  u32 iCookie;
  i64 iOalSz;
  i64 nPhaseOneStep;
................................................................................

/*
** Free an RbuState object allocated by rbuLoadState().
*/
static void rbuFreeState(RbuState *p){
  if( p ){
    sqlite3_free(p->zTbl);

    sqlite3_free(p->zIdx);
    sqlite3_free(p);
  }
}

/*
** Allocate an RbuState object and load the contents of the rbu_state 
................................................................................
      case RBU_STATE_OALSZ:
        pRet->iOalSz = (u32)sqlite3_column_int64(pStmt, 1);
        break;

      case RBU_STATE_PHASEONESTEP:
        pRet->nPhaseOneStep = sqlite3_column_int64(pStmt, 1);
        break;





      default:
        rc = SQLITE_CORRUPT;
        break;
    }
  }
  rc2 = sqlite3_finalize(pStmt);
................................................................................
          "(%d, %Q), "
          "(%d, %Q), "
          "(%d, %d), "
          "(%d, %d), "
          "(%d, %lld), "
          "(%d, %lld), "
          "(%d, %lld), "
          "(%d, %lld) ",

          p->zStateDb,
          RBU_STATE_STAGE, eStage,
          RBU_STATE_TBL, p->objiter.zTbl, 
          RBU_STATE_IDX, p->objiter.zIdx, 
          RBU_STATE_ROW, p->nStep, 
          RBU_STATE_PROGRESS, p->nProgress,
          RBU_STATE_CKPT, p->iWalCksum,
          RBU_STATE_COOKIE, (i64)pFd->iCookie,
          RBU_STATE_OALSZ, p->iOalSz,
          RBU_STATE_PHASEONESTEP, p->nPhaseOneStep

      )
    );
    assert( pInsert==0 || rc==SQLITE_OK );

    if( rc==SQLITE_OK ){
      sqlite3_step(pInsert);
      rc = sqlite3_finalize(pInsert);
................................................................................
  assert( p->rc==SQLITE_OK );
  if( pState->zTbl ){
    RbuObjIter *pIter = &p->objiter;
    int rc = SQLITE_OK;

    while( rc==SQLITE_OK && pIter->zTbl && (pIter->bCleanup 
       || rbuStrCompare(pIter->zIdx, pState->zIdx)
       || rbuStrCompare(pIter->zTbl, pState->zTbl) 

    )){
      rc = rbuObjIterNext(p, pIter);
    }

    if( rc==SQLITE_OK && !pIter->zTbl ){
      rc = SQLITE_ERROR;
      p->zErrmsg = sqlite3_mprintf("rbu_state mismatch error");







>
>
>
>










>







 







>







 







>







 







>
>
>
>







 







|
>









|
>







 







|
>







149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
...
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
....
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
....
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
....
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
....
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
**
** RBU_STATE_COOKIE:
**   Valid if STAGE==1. The current change-counter cookie value in the 
**   target db file.
**
** RBU_STATE_OALSZ:
**   Valid if STAGE==1. The size in bytes of the *-oal file.
**
** RBU_STATE_DATATBL:
**   Only valid if STAGE==1. The RBU database name of the table 
**   currently being read.
*/
#define RBU_STATE_STAGE        1
#define RBU_STATE_TBL          2
#define RBU_STATE_IDX          3
#define RBU_STATE_ROW          4
#define RBU_STATE_PROGRESS     5
#define RBU_STATE_CKPT         6
#define RBU_STATE_COOKIE       7
#define RBU_STATE_OALSZ        8
#define RBU_STATE_PHASEONESTEP 9
#define RBU_STATE_DATATBL     10

#define RBU_STAGE_OAL         1
#define RBU_STAGE_MOVE        2
#define RBU_STAGE_CAPTURE     3
#define RBU_STAGE_CKPT        4
#define RBU_STAGE_DONE        5

................................................................................

/*
** A structure to store values read from the rbu_state table in memory.
*/
struct RbuState {
  int eStage;
  char *zTbl;
  char *zDataTbl;
  char *zIdx;
  i64 iWalCksum;
  int nRow;
  i64 nProgress;
  u32 iCookie;
  i64 iOalSz;
  i64 nPhaseOneStep;
................................................................................

/*
** Free an RbuState object allocated by rbuLoadState().
*/
static void rbuFreeState(RbuState *p){
  if( p ){
    sqlite3_free(p->zTbl);
    sqlite3_free(p->zDataTbl);
    sqlite3_free(p->zIdx);
    sqlite3_free(p);
  }
}

/*
** Allocate an RbuState object and load the contents of the rbu_state 
................................................................................
      case RBU_STATE_OALSZ:
        pRet->iOalSz = (u32)sqlite3_column_int64(pStmt, 1);
        break;

      case RBU_STATE_PHASEONESTEP:
        pRet->nPhaseOneStep = sqlite3_column_int64(pStmt, 1);
        break;

      case RBU_STATE_DATATBL:
        pRet->zDataTbl = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc);
        break;

      default:
        rc = SQLITE_CORRUPT;
        break;
    }
  }
  rc2 = sqlite3_finalize(pStmt);
................................................................................
          "(%d, %Q), "
          "(%d, %Q), "
          "(%d, %d), "
          "(%d, %d), "
          "(%d, %lld), "
          "(%d, %lld), "
          "(%d, %lld), "
          "(%d, %lld), "
          "(%d, %Q)  ",
          p->zStateDb,
          RBU_STATE_STAGE, eStage,
          RBU_STATE_TBL, p->objiter.zTbl, 
          RBU_STATE_IDX, p->objiter.zIdx, 
          RBU_STATE_ROW, p->nStep, 
          RBU_STATE_PROGRESS, p->nProgress,
          RBU_STATE_CKPT, p->iWalCksum,
          RBU_STATE_COOKIE, (i64)pFd->iCookie,
          RBU_STATE_OALSZ, p->iOalSz,
          RBU_STATE_PHASEONESTEP, p->nPhaseOneStep,
          RBU_STATE_DATATBL, p->objiter.zDataTbl
      )
    );
    assert( pInsert==0 || rc==SQLITE_OK );

    if( rc==SQLITE_OK ){
      sqlite3_step(pInsert);
      rc = sqlite3_finalize(pInsert);
................................................................................
  assert( p->rc==SQLITE_OK );
  if( pState->zTbl ){
    RbuObjIter *pIter = &p->objiter;
    int rc = SQLITE_OK;

    while( rc==SQLITE_OK && pIter->zTbl && (pIter->bCleanup 
       || rbuStrCompare(pIter->zIdx, pState->zIdx)
       || (pState->zDataTbl==0 && rbuStrCompare(pIter->zTbl, pState->zTbl))
       || (pState->zDataTbl && rbuStrCompare(pIter->zDataTbl, pState->zDataTbl))
    )){
      rc = rbuObjIterNext(p, pIter);
    }

    if( rc==SQLITE_OK && !pIter->zTbl ){
      rc = SQLITE_ERROR;
      p->zErrmsg = sqlite3_mprintf("rbu_state mismatch error");