SQLite

Check-in [cdc09867ed]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Optimize some cases of restarting an RBU vacuum.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | rbu-opt
Files: files | file ages | folders
SHA3-256: cdc09867ed6522026ae7bfac1f59cd79b60fba6d07d49b99b030a501a7059ee4
User & Date: dan 2019-05-04 20:04:42.728
Context
2019-05-06
20:40
Optimize further cases of restarting an RBU vacuum. (check-in: 6b3261bfa1 user: dan tags: rbu-opt)
2019-05-04
20:04
Optimize some cases of restarting an RBU vacuum. (check-in: cdc09867ed user: dan tags: rbu-opt)
17:32
Fix the NOT NULL logic in the theorem prover that determines when a partial index can be used. Ticket [5c6955204c392ae763a95]. (check-in: c2e439bccc user: drh tags: trunk)
Changes
Unified Diff Ignore Whitespace Patch
Changes to ext/rbu/rbu_common.tcl.
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
  }
  set rc
}

proc do_rbu_vacuum_test {tn step {statedb state.db}} {
  forcedelete $statedb
  if {$statedb=="" && $step==1} breakpoint
  uplevel [list do_test $tn.1 [string map [list %state% $statedb] {
    if {$step==0} { sqlite3rbu_vacuum rbu test.db {%state%}}
    while 1 {
      if {$step==1} { sqlite3rbu_vacuum rbu test.db {%state%}}
      set state [rbu state]
      check_prestep_state test.db $state
      set rc [rbu step]
      check_poststep_state $rc test.db $state
      if {$rc!="SQLITE_OK"} break
      if {$step==1} { rbu close }
    }
    rbu close
  }] {SQLITE_DONE}]

  uplevel [list do_execsql_test $tn.2 {
    PRAGMA integrity_check
  } ok]







|
|

|





|







85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
  }
  set rc
}

proc do_rbu_vacuum_test {tn step {statedb state.db}} {
  forcedelete $statedb
  if {$statedb=="" && $step==1} breakpoint
  uplevel [list do_test $tn.1 [string map [list %state% $statedb %step% $step] {
    if {%step%==0} { sqlite3rbu_vacuum rbu test.db {%state%}}
    while 1 {
      if {%step%==1} { sqlite3rbu_vacuum rbu test.db {%state%}}
      set state [rbu state]
      check_prestep_state test.db $state
      set rc [rbu step]
      check_poststep_state $rc test.db $state
      if {$rc!="SQLITE_OK"} break
      if {%step%==1} { rbu close }
    }
    rbu close
  }] {SQLITE_DONE}]

  uplevel [list do_execsql_test $tn.2 {
    PRAGMA integrity_check
  } ok]
Changes to ext/rbu/rbupartial.test.
76
77
78
79
80
81
82




83
84
85
86
    SELECT * FROM t1 ORDER BY %A%;
  } {
    1 10 {} b   7 8 4 d   10 11 12 e   13 14 {} f
  }

  set step 0
  do_rbu_vacuum_test $tn.1.5 0




  }]
}

finish_test







>
>
>
>




76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
    SELECT * FROM t1 ORDER BY %A%;
  } {
    1 10 {} b   7 8 4 d   10 11 12 e   13 14 {} f
  }

  set step 0
  do_rbu_vacuum_test $tn.1.5 0

  do_test $tn.1.6 {
    execsql { PRAGMA integrity_check }
  } {ok}
  }]
}

finish_test
Changes to ext/rbu/sqlite3rbu.c.
1377
1378
1379
1380
1381
1382
1383

1384
1385
1386
1387
1388
1389
1390
1391

        if( i!=iOrder ){
          SWAP(int, pIter->aiSrcOrder[i], pIter->aiSrcOrder[iOrder]);
          SWAP(char*, pIter->azTblCol[i], pIter->azTblCol[iOrder]);
        }

        pIter->azTblType[iOrder] = rbuStrndup(zType, &p->rc);

        pIter->abTblPk[iOrder] = (iPk!=0);
        pIter->abNotNull[iOrder] = (u8)bNotNull || (iPk!=0);
        iOrder++;
      }
    }

    rbuFinalize(p, pStmt);
    rbuObjIterCacheIndexedCols(p, pIter);







>
|







1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392

        if( i!=iOrder ){
          SWAP(int, pIter->aiSrcOrder[i], pIter->aiSrcOrder[iOrder]);
          SWAP(char*, pIter->azTblCol[i], pIter->azTblCol[iOrder]);
        }

        pIter->azTblType[iOrder] = rbuStrndup(zType, &p->rc);
        assert( iPk>=0 );
        pIter->abTblPk[iOrder] = (u8)iPk;
        pIter->abNotNull[iOrder] = (u8)bNotNull || (iPk!=0);
        iOrder++;
      }
    }

    rbuFinalize(p, pStmt);
    rbuObjIterCacheIndexedCols(p, pIter);
1411
1412
1413
1414
1415
1416
1417






















































































1418
1419
1420
1421
1422
1423
1424
  for(i=0; i<pIter->nTblCol; i++){
    const char *z = pIter->azTblCol[i];
    zList = rbuMPrintf(p, "%z%s\"%w\"", zList, zSep, z);
    zSep = ", ";
  }
  return zList;
}























































































/*
** This function is used to create a SELECT list (the list of SQL 
** expressions that follows a SELECT keyword) for a SELECT statement 
** used to read from an data_xxx or rbu_tmp_xxx table while updating the 
** index object currently indicated by the iterator object passed as the 
** second argument. A "PRAGMA index_xinfo = <idxname>" statement is used 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
  for(i=0; i<pIter->nTblCol; i++){
    const char *z = pIter->azTblCol[i];
    zList = rbuMPrintf(p, "%z%s\"%w\"", zList, zSep, z);
    zSep = ", ";
  }
  return zList;
}

static char *rbuObjIterGetPkList(
  sqlite3rbu *p,                  /* RBU object */
  RbuObjIter *pIter,              /* Object iterator for column names */
  const char *zExtra
){
  int iPk = 1;
  char *zRet = 0;
  const char *zSep = "";
  while( 1 ){
    int i;
    for(i=0; i<pIter->nTblCol; i++){
      if( (int)pIter->abTblPk[i]==iPk ){
        const char *zCol = pIter->azTblCol[i];
        zRet = rbuMPrintf(p, "%z%s\"%w\"%s", zRet, zSep, zCol, zExtra);
        zSep = ", ";
      }
    }
    if( i==pIter->nTblCol ) break;
    iPk++;
  }
  return zRet;
}

static char *rbuVacuumTableStart(
  sqlite3rbu *p, 
  RbuObjIter *pIter,
  int bRowid,
  const char *zWrite
){
  sqlite3_stmt *pMax = 0;
  char *zRet = 0;
  if( bRowid ){
    p->rc = prepareFreeAndCollectError(p->dbMain, &pMax, &p->zErrmsg, 
        sqlite3_mprintf(
          "SELECT max(_rowid_) FROM \"%s%w\"", zWrite, pIter->zTbl
        )
    );
    if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pMax) ){
      sqlite3_int64 iMax = sqlite3_column_int64(pMax, 0);
      zRet = rbuMPrintf(p, " WHERE _rowid_ > %lld ", iMax);
    }
    rbuFinalize(p, pMax);
  }else{
    char *zOrder = 0;
    char *zSelect = 0;
    char *zList = 0;
    int iPk = 1;
    const char *zSep = "";
    const char *zSep2 = "";
    while( 1 ){
      int i;
      for(i=0; i<pIter->nTblCol; i++){
        if( (int)pIter->abTblPk[i]==iPk ){
          const char *zCol = pIter->azTblCol[i];
          zOrder = rbuMPrintf(p, "%z%s\"%w\" DESC", zOrder, zSep, zCol);
          zList = rbuMPrintf(p, "%z%s\"%w\"", zList, zSep, zCol);
          zSelect = rbuMPrintf(p, "%z%squote(\"%w\")", zSelect, zSep2, zCol);
          zSep = ", ";
          zSep2 = "||','||";
        }
      }
      if( i==pIter->nTblCol ) break;
      iPk++;
    }

    if( p->rc==SQLITE_OK ){
      p->rc = prepareFreeAndCollectError(p->dbMain, &pMax, &p->zErrmsg, 
          sqlite3_mprintf(
            "SELECT %s FROM \"%s%w\" ORDER BY %s LIMIT 1", 
                zSelect, zWrite, pIter->zTbl, zOrder
          )
      );
      if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pMax) ){
        const char *zVal = (const char*)sqlite3_column_text(pMax, 0);
        zRet = rbuMPrintf(p, " WHERE (%s) > (%s) ", zList, zVal);
      }
      rbuFinalize(p, pMax);
    }

    sqlite3_free(zOrder);
    sqlite3_free(zSelect);
    sqlite3_free(zList);
  }
  return zRet;
}

/*
** This function is used to create a SELECT list (the list of SQL 
** expressions that follows a SELECT keyword) for a SELECT statement 
** used to read from an data_xxx or rbu_tmp_xxx table while updating the 
** index object currently indicated by the iterator object passed as the 
** second argument. A "PRAGMA index_xinfo = <idxname>" statement is used 
2216
2217
2218
2219
2220
2221
2222


2223
2224
2225

















2226
2227
2228
2229
2230
2231
2232


2233
2234



2235
2236
2237
2238
2239
2240
2241

        rbuObjIterPrepareTmpInsert(p, pIter, zCollist, zRbuRowid);
      }

      /* Create the SELECT statement to read keys from data_xxx */
      if( p->rc==SQLITE_OK ){
        const char *zRbuRowid = "";


        if( bRbuRowid ){
          zRbuRowid = rbuIsVacuum(p) ? ",_rowid_ " : ",rbu_rowid";
        }

















        p->rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pSelect, pz,
            sqlite3_mprintf(
              "SELECT %s,%s rbu_control%s FROM '%q'%s", 
              zCollist, 
              (rbuIsVacuum(p) ? "0 AS " : ""),
              zRbuRowid,
              pIter->zDataTbl, zLimit


            )
        );



      }

      sqlite3_free(zWhere);
      sqlite3_free(zOldlist);
      sqlite3_free(zNewlist);
      sqlite3_free(zBindings);
    }







>
>



>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|
|
|
|
|
|
|
>
>
|
|
>
>
>







2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352

        rbuObjIterPrepareTmpInsert(p, pIter, zCollist, zRbuRowid);
      }

      /* Create the SELECT statement to read keys from data_xxx */
      if( p->rc==SQLITE_OK ){
        const char *zRbuRowid = "";
        char *zStart = 0;
        char *zOrder = 0;
        if( bRbuRowid ){
          zRbuRowid = rbuIsVacuum(p) ? ",_rowid_ " : ",rbu_rowid";
        }

        if( rbuIsVacuum(p) ){
          if( nOffset ){
            zStart = rbuVacuumTableStart(p, pIter, bRbuRowid, zWrite);
            if( zStart ){
              sqlite3_free(zLimit);
              zLimit = 0;
            }
          }
          if( bRbuRowid ){
            zOrder = rbuMPrintf(p, "_rowid_");
          }else{
            zOrder = rbuObjIterGetPkList(p, pIter, "");
          }
        }

        if( p->rc==SQLITE_OK ){
          p->rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pSelect, pz,
              sqlite3_mprintf(
                "SELECT %s,%s rbu_control%s FROM '%q'%s %s %s %s",
                zCollist, 
                (rbuIsVacuum(p) ? "0 AS " : ""),
                zRbuRowid,
                pIter->zDataTbl, (zStart ? zStart : ""), 
                (zOrder ? "ORDER BY" : ""), zOrder,
                zLimit
              )
          );
        }
        sqlite3_free(zStart);
        sqlite3_free(zOrder);
      }

      sqlite3_free(zWhere);
      sqlite3_free(zOldlist);
      sqlite3_free(zNewlist);
      sqlite3_free(zBindings);
    }