SQLite

# 2015-07-31
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#
# Tests for the [sqldiff --rbu] command.
#
#
if {![info exists testdir]} {
  set testdir [file join [file dirname [info script]] .. .. test]
}
source $testdir/tester.tcl
set testprefix rbudiff

if {$tcl_platform(platform)=="windows"} {
  set PROG "sqldiff.exe"
} else {
  set PROG "./sqldiff"
}
if {![file exe $PROG]} {
  puts "rbudiff.test cannot run because $PROG is not available"
  finish_test
  return
}
db close

proc get_rbudiff_sql {db1 db2} {
  exec $::PROG --rbu $db1 $db2
}

proc step_rbu {target rbu} {
  while 1 {
    sqlite3rbu rbu $target $rbu
    set rc [rbu step]
    rbu close
    if {$rc != "SQLITE_OK"} break
  }
  set rc
}

proc apply_rbudiff {sql target} {
  forcedelete rbu.db
  sqlite3 rbudb rbu.db
  rbudb eval $sql
  rbudb close
  step_rbu $target rbu.db
}

proc rbudiff_cksum {db1} {
  sqlite3 dbtmp $db1
  set txt [dbtmp eval {
    SELECT a || '.' || b || '.' || c FROM t1 ORDER BY 1;
    SELECT a || '.' || b || '.' || c FROM t2 ORDER BY 1;
  }]
  dbtmp close
  md5 $txt
}

sqlite3 db test.db
do_execsql_test 1.0 {
  CREATE TABLE t1(a PRIMARY KEY, b, c);
  INSERT INTO t1 VALUES(1, 2, 3);
  INSERT INTO t1 VALUES(4, 5, 6);

  CREATE TABLE t2(a, b, c, PRIMARY KEY(b, c));
  INSERT INTO t2 VALUES(1, 2, 3);
  INSERT INTO t2 VALUES(4, 5, 6);
}

db close
forcedelete test.db2
forcecopy test.db test.db2

sqlite3 db test.db
do_execsql_test 1.1 {
  INSERT INTO t1 VALUES(7, 8, 9);
  DELETE FROM t1 WHERE a=4;
  UPDATE t1 SET c = 11 WHERE a = 1;

  INSERT INTO t2 VALUES(7, 8, 9);
  DELETE FROM t2 WHERE a=4;
  UPDATE t2 SET c = 11 WHERE a = 1;
}
db close

do_test 1.2 {
  set sql [get_rbudiff_sql test.db test.db2]
  apply_rbudiff $sql test.db
} {SQLITE_DONE}

do_test 1.3 { rbudiff_cksum test.db } [rbudiff_cksum test.db2]

finish_test

**    CREATE TABLE t4(x,y,z,PRIMARY KEY(y,z)) WITHOUT ROWID;
**    *pnPKey = 2
**    az = { "y", "z", "x", 0 }
**
**    CREATE TABLE t5(rowid,_rowid_,oid);
**    az = 0     // The rowid is not accessible
*/
static char **columnNames(
  const char *zDb,                /* Database ("main" or "aux") to query */
  const char *zTab,               /* Name of table to return details of */
  int *pnPKey,                    /* OUT: Number of PK columns */
  int *pbRowid                    /* OUT: True if PK is an implicit rowid */
){
  char **az = 0;           /* List of column names to be returned */
  int naz = 0;             /* Number of entries in az[] */
  sqlite3_stmt *pStmt;     /* SQL statement being run */
  char *zPkIdxName = 0;    /* Name of the PRIMARY KEY index */
  int truePk = 0;          /* PRAGMA table_info indentifies the PK to use */
  int nPK = 0;             /* Number of PRIMARY KEY columns */
  int i, j;                /* Loop counters */

      az = sqlite3_realloc(az, sizeof(char*)*(naz+2) );
      if( az==0 ) runtimeError("out of memory");
      az[naz++] = safeId((char*)sqlite3_column_text(pStmt,1));
    }
  }
  sqlite3_finalize(pStmt);
  if( az ) az[naz] = 0;

  /* If it is non-NULL, set *pbRowid to indicate whether or not the PK of 
  ** this table is an implicit rowid (*pbRowid==1) or not (*pbRowid==0).  */
  if( pbRowid ) *pbRowid = (az[0]==0);

  /* If this table has an implicit rowid for a PK, figure out how to refer
  ** to it. There are three options - "rowid", "_rowid_" and "oid". Any
  ** of these will work, unless the table has an explicit column of the
  ** same name.  */
  if( az[0]==0 ){
    const char *azRowid[] = { "rowid", "_rowid_", "oid" };
    for(i=0; i<sizeof(azRowid)/sizeof(azRowid[0]); i++){
      for(j=1; j<naz; j++){
        if( sqlite3_stricmp(az[j], azRowid[i])==0 ) break;
      }
      if( j>=naz ){

  pStmt = db_prepare("SELECT sql FROM aux.sqlite_master WHERE name=%Q", zTab);
  if( SQLITE_ROW==sqlite3_step(pStmt) ){
    fprintf(out, "%s;\n", sqlite3_column_text(pStmt,0));
  }
  sqlite3_finalize(pStmt);
  if( !g.bSchemaOnly ){
    az = columnNames("aux", zTab, &nPk, 0);
    strInit(&ins);
    if( az==0 ){
      pStmt = db_prepare("SELECT * FROM aux.%s", zId);
      strPrintf(&ins,"INSERT INTO %s VALUES", zId);
    }else{
      Str sql;
      strInit(&sql);

  strInit(&sql);
  if( g.fDebug==DEBUG_COLUMN_NAMES ){
    /* Simply run columnNames() on all tables of the origin
    ** database and show the results.  This is used for testing
    ** and debugging of the columnNames() function.
    */
    az = columnNames("aux",zTab, &nPk, 0);
    if( az==0 ){
      printf("Rowid not accessible for %s\n", zId);
    }else{
      printf("%s:", zId);
      for(i=0; az[i]; i++){
        printf(" %s", az[i]);
        if( i+1==nPk ) printf(" *");

  if( sqlite3_table_column_metadata(g.db,"main",zTab,0,0,0,0,0,0) ){
    /* Table missing from source */
    dump_table(zTab, out);
    goto end_diff_one_table;
  }

  az = columnNames("main", zTab, &nPk, 0);
  az2 = columnNames("aux", zTab, &nPk2, 0);
  if( az && az2 ){
    for(n=0; az[n]; n++){
      if( sqlite3_stricmp(az[n],az2[n])!=0 ) break;
    }
  }
  if( az==0
   || az2==0

end_diff_one_table:
  strFree(&sql);
  sqlite3_free(zId);
  namelistFree(az);
  namelistFree(az2);
  return;
}

/*
** Check that table zTab exists and has the same schema in both the "main"
** and "aux" databases currently opened by the global db handle. If they
** do not, output an error message on stderr and exit(1). Otherwise, if
** the schemas do match, return control to the caller.
*/
static void checkSchemasMatch(const char *zTab){
  sqlite3_stmt *pStmt = db_prepare(
      "SELECT A.sql=B.sql FROM main.sqlite_master A, aux.sqlite_master B"
      " WHERE A.name=%Q AND B.name=%Q", zTab, zTab
  );
  if( SQLITE_ROW==sqlite3_step(pStmt) ){
    if( sqlite3_column_int(pStmt,0)==0 ){
      runtimeError("schema changes for table %s", safeId(zTab));
    }
  }else{
    runtimeError("table %s missing from one or both databases", safeId(zTab));
  }
  sqlite3_finalize(pStmt);
}

static void strPrintfArray(
  Str *pStr,                      /* String object to append to */
  const char *zSep,               /* Separator string */
  const char *zFmt,               /* Format for each entry */
  char **az, int n                /* Array of strings & its size (or -1) */
){
  int i;
  for(i=0; az[i] && (i<n || n<0); i++){
    if( i!=0 ) strPrintf(pStr, "%s", zSep);
    strPrintf(pStr, zFmt, az[i], az[i], az[i]);
  }
}

static void getRbudiffQuery(
  const char *zTab,
  char **azCol,
  int nPK,
  int bOtaRowid,
  Str *pSql
){
  int i;

  /* First the newly inserted rows: **/ 
  strPrintf(pSql, "SELECT ");
  strPrintfArray(pSql, ", ", "%s", azCol, -1);
  strPrintf(pSql, ", 0");         /* Set ota_control to 0 for an insert */
  strPrintf(pSql, " FROM aux.%Q AS n WHERE NOT EXISTS (\n", zTab);
  strPrintf(pSql, "    SELECT 1 FROM ", zTab);
  strPrintf(pSql, " main.%Q AS o WHERE ", zTab);
  strPrintfArray(pSql, " AND ", "(n.%Q IS o.%Q)", azCol, nPK);
  strPrintf(pSql, "\n)");

  /* Deleted rows: */
  strPrintf(pSql, "\nUNION ALL\nSELECT ");
  strPrintfArray(pSql, ", ", "%s", azCol, nPK);
  strPrintf(pSql, ", ");
  strPrintfArray(pSql, ", ", "NULL", &azCol[nPK], -1);
  strPrintf(pSql, ", 1");         /* Set ota_control to 1 for a delete */
  strPrintf(pSql, " FROM main.%Q AS n WHERE NOT EXISTS (\n", zTab);
  strPrintf(pSql, "    SELECT 1 FROM ", zTab);
  strPrintf(pSql, " aux.%Q AS o WHERE ", zTab);
  strPrintfArray(pSql, " AND ", "(n.%Q IS o.%Q)", azCol, nPK);
  strPrintf(pSql, "\n) ");

  /* Updated rows: */
  strPrintf(pSql, "\nUNION ALL\nSELECT ");
  strPrintfArray(pSql, ", ", "n.%s", azCol, nPK);
  strPrintf(pSql, ",\n");
  strPrintfArray(pSql, " ,\n", 
      "    CASE WHEN n.%s IS o.%s THEN NULL ELSE n.%s END", &azCol[nPK], -1
  );

  if( bOtaRowid==0 ){
    strPrintf(pSql, ", '");
    strPrintfArray(pSql, "", ".", azCol, nPK);
    strPrintf(pSql, "' ||\n");
  }else{
    strPrintf(pSql, ",\n");
  }
  strPrintfArray(pSql, " ||\n", 
      "    CASE WHEN n.%s IS o.%s THEN '.' ELSE 'x' END", &azCol[nPK], -1
  );
  strPrintf(pSql, "\nAS ota_control");

  strPrintf(pSql, "\nFROM main.%Q AS o, aux.%Q AS n\nWHERE ", zTab, zTab);
  strPrintfArray(pSql, " AND ", "(n.%Q IS o.%Q)", azCol, nPK);
  strPrintf(pSql, " AND ota_control LIKE '%%x%%'");

  /* Now add an ORDER BY clause to sort everything by PK. */
  strPrintf(pSql, "\nORDER BY ");
  for(i=1; i<=nPK; i++) strPrintf(pSql, "%s%d", ((i>1)?", ":""), i);
}

static void rbudiff_one_table(const char *zTab, FILE *out){
  int bOtaRowid;                  /* True to use an ota_rowid column */
  int nPK;                        /* Number of primary key columns in table */
  char **azCol;                   /* NULL terminated array of col names */
  int i;
  int nCol;
  Str ct = {0, 0, 0};             /* The "CREATE TABLE data_xxx" statement */
  Str sql = {0, 0, 0};            /* Query to find differences */
  Str insert = {0, 0, 0};         /* First part of output INSERT statement */
  sqlite3_stmt *pStmt = 0;

  /* --rbu mode must use real primary keys. */
  g.bSchemaPK = 1;

  /* Check that the schemas of the two tables match. Exit early otherwise. */
  checkSchemasMatch(zTab);

  /* Grab the column names and PK details for the table(s). If no usable PK
  ** columns are found, bail out early.  */
  azCol = columnNames("main", zTab, &nPK, &bOtaRowid);
  if( azCol==0 ){
    runtimeError("table %s has no usable PK columns", zTab);
  }

  /* Build and output the CREATE TABLE statement for the data_xxx table */
  strPrintf(&ct, "CREATE TABLE IF NOT EXISTS 'data_%q'(", zTab);
  if( bOtaRowid ) strPrintf(&ct, "rbu_rowid, ");
  strPrintfArray(&ct, ", ", "%s", &azCol[bOtaRowid], -1);
  strPrintf(&ct, ", rbu_control);");


  /* Get the SQL for the query to retrieve data from the two databases */
  getRbudiffQuery(zTab, azCol, nPK, bOtaRowid, &sql);

  /* Build the first part of the INSERT statement output for each row
  ** in the data_xxx table. */
  strPrintf(&insert, "INSERT INTO 'data_%q' (", zTab);
  if( bOtaRowid ) strPrintf(&insert, "rbu_rowid, ");
  strPrintfArray(&insert, ", ", "%s", &azCol[bOtaRowid], -1);
  strPrintf(&insert, ", rbu_control) VALUES(");

  pStmt = db_prepare("%s", sql.z);
  nCol = sqlite3_column_count(pStmt);
  while( sqlite3_step(pStmt)==SQLITE_ROW ){
    if( ct.z ){
      fprintf(out, "%s\n", ct.z);
      strFree(&ct);
    }

    fprintf(out, "%s", insert.z);
    for(i=0; i<nCol; i++){
      if( i>0 ) fprintf(out, ", ");
      printQuoted(out, sqlite3_column_value(pStmt, i));
    }
    fprintf(out, ");\n");
  }

  sqlite3_finalize(pStmt);

  strFree(&ct);
  strFree(&sql);
  strFree(&insert);
}

/*
** Display a summary of differences between two versions of the same
** table table.
**
**   *  Number of rows changed
**   *  Number of rows added

  if( sqlite3_table_column_metadata(g.db,"main",zTab,0,0,0,0,0,0) ){
    /* Table missing from source */
    fprintf(out, "%s: missing from first database\n", zTab);
    goto end_summarize_one_table;
  }

  az = columnNames("main", zTab, &nPk, 0);
  az2 = columnNames("aux", zTab, &nPk2, 0);
  if( az && az2 ){
    for(n=0; az[n]; n++){
      if( sqlite3_stricmp(az[n],az2[n])!=0 ) break;
    }
  }
  if( az==0
   || az2==0

  int *aiFlg = 0;               /* 0 if column is not part of PK */
  int *aiPk = 0;                /* Column numbers for each PK column */
  int nPk = 0;                  /* Number of PRIMARY KEY columns */
  Str sql;                      /* SQL for the diff query */
  int i, k;                     /* Loop counters */
  const char *zSep;             /* List separator */

  /* Check that the schemas of the two tables match. Exit early otherwise. */


  checkSchemasMatch(zTab);








  pStmt = db_prepare("PRAGMA main.table_info=%Q", zTab);
  while( SQLITE_ROW==sqlite3_step(pStmt) ){
    nCol++;
    azCol = sqlite3_realloc(azCol, sizeof(char*)*nCol);
    if( azCol==0 ) runtimeError("out of memory");
    aiFlg = sqlite3_realloc(aiFlg, sizeof(int)*nCol);
    if( aiFlg==0 ) runtimeError("out of memory");

  printf("Usage: %s [options] DB1 DB2\n", g.zArgv0);
  printf(
"Output SQL text that would transform DB1 into DB2.\n"
"Options:\n"
"  --changeset FILE      Write a CHANGESET into FILE\n"
"  -L|--lib LIBRARY      Load an SQLite extension library\n"
"  --primarykey          Use schema-defined PRIMARY KEYs\n"
"  --rbu                 Output SQL to create/populate RBU table(s)\n"
"  --schema              Show only differences in the schema\n"
"  --summary             Show only a summary of the differences\n"
"  --table TAB           Show only differences in table TAB\n"
  );
}

int main(int argc, char **argv){

        if( azExt==0 ) cmdlineError("out of memory");
        azExt[nExt++] = argv[++i];
      }else
#endif
      if( strcmp(z,"primarykey")==0 ){
        g.bSchemaPK = 1;
      }else
      if( strcmp(z,"rbu")==0 ){
        xDiff = rbudiff_one_table;
      }else
      if( strcmp(z,"schema")==0 ){
        g.bSchemaOnly = 1;
      }else
      if( strcmp(z,"summary")==0 ){
        xDiff = summarize_one_table;
      }else
      if( strcmp(z,"table")==0 ){