SQLite

/*
** 2017 July 15
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
**
** This file contains the implementation of the "unionvtab" and "swarmvtab"
** virtual tables. These modules provide read-only access to multiple tables,
** possibly in multiple database files, via a single database object.
** The source tables must have the following characteristics:
**
**   * They must all be rowid tables (not VIRTUAL or WITHOUT ROWID
**     tables or views).
**
**   * Each table must have the same set of columns, declared in
**     the same order and with the same declared types.
**
**   * The tables must not feature a user-defined column named "_rowid_".
**
**   * Each table must contain a distinct range of rowid values.
**
** The difference between the two virtual table modules is that for 
** "unionvtab", all source tables must be located in the main database or
** in databases ATTACHed to the main database by the user. For "swarmvtab",
** the tables may be located in any database file on disk. The "swarmvtab"
** implementation takes care of opening and closing database files
** automatically.
**
** UNIONVTAB
**
**   A "unionvtab" virtual table is created as follows:
**
**     CREATE VIRTUAL TABLE <name> USING unionvtab(<sql statement>);
**
**   The implementation evalutes <sql statement> whenever a unionvtab virtual
**   table is created or opened. It should return one row for each source
**   database table. The four columns required of each row are:
**
**     1. The name of the database containing the table ("main" or "temp" or
**        the name of an attached database). Or NULL to indicate that all
**        databases should be searched for the table in the usual fashion.
**
**     2. The name of the database table.
**
**     3. The smallest rowid in the range of rowids that may be stored in the
**        database table (an integer).
**
**     4. The largest rowid in the range of rowids that may be stored in the
**        database table (an integer).
**
** SWARMVTAB
**
**   A "swarmvtab" virtual table is created similarly to a unionvtab table:
**
**     CREATE VIRTUAL TABLE <name> USING swarmvtab(<sql statement>);
**
**   The difference is that for a swarmvtab table, the first column returned
**   by the <sql statement> must return a path or URI that can be used to open
**   the database file containing the source table.
**
*/

#include "sqlite3ext.h"
SQLITE_EXTENSION_INIT1
#include <assert.h>
#include <string.h>

#ifndef LARGEST_INT64
# define LARGEST_INT64  (0xffffffff|(((sqlite3_int64)0x7fffffff)<<32))
#endif
#ifndef SMALLEST_INT64
# define SMALLEST_INT64 (((sqlite3_int64)-1) - LARGEST_INT64)
#endif

/*
** The swarmvtab module attempts to keep the number of open database files
** at or below this limit. This may not be possible if there are too many
** simultaneous queries.
*/
#define SWARMVTAB_MAX_OPEN 9

typedef struct UnionCsr UnionCsr;
typedef struct UnionTab UnionTab;
typedef struct UnionSrc UnionSrc;

/*
** Each source table (row returned by the initialization query) is 
** represented by an instance of the following structure stored in the
** UnionTab.aSrc[] array.
*/
struct UnionSrc {
  char *zDb;                      /* Database containing source table */
  char *zTab;                     /* Source table name */
  sqlite3_int64 iMin;             /* Minimum rowid */
  sqlite3_int64 iMax;             /* Maximum rowid */

  /* Fields used by swarmvtab only */
  char *zFile;                    /* Database file containing table zTab */
  int nUser;                      /* Current number of users */
  sqlite3 *db;                    /* Database handle */
  UnionSrc *pNextClosable;        /* Next in list of closable sources */
};

/*
** Virtual table  type for union vtab.

  sqlite3_stmt *pStmt;            /* SQL statement to run */

  /* Used by swarmvtab only */
  sqlite3_int64 iMaxRowid;        /* Last rowid to visit */
  int iTab;                       /* Index of table read by pStmt */
};

/*
** Given UnionTab table pTab and UnionSrc object pSrc, return the database
** handle that should be used to access the table identified by pSrc. This
** is the main db handle for "unionvtab" tables, or the source-specific 
** handle for "swarmvtab".
*/
#define unionGetDb(pTab, pSrc) ((pTab)->bSwarm ? (pSrc)->db : (pTab)->db)

/*
** If *pRc is other than SQLITE_OK when this function is called, it
** always returns NULL. Otherwise, it attempts to allocate and return
** a pointer to nByte bytes of zeroed memory. If the memory allocation
** is attempted but fails, NULL is returned and *pRc is set to 

    if( rc ){
      *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db));
    }
  }
}

/*
** This function is a no-op for unionvtab. For swarmvtab, it attempts to
** close open database files until at most nMax are open. An SQLite error
** code is returned if an error occurs, or SQLITE_OK otherwise.
*/
static int unionCloseSources(UnionTab *pTab, int nMax){
  int rc = SQLITE_OK;

  if( pTab->bSwarm ){
    while( rc==SQLITE_OK && pTab->pClosable && pTab->nOpen>nMax ){
      UnionSrc **pp;

      for(pp=&pTab->pClosable; (*pp)->pNextClosable; pp=&(*pp)->pNextClosable);



      assert( (*pp)->db );
      rc = sqlite3_close((*pp)->db);
      (*pp)->db = 0;
      *pp = 0;
      pTab->nOpen--;
    }
  }
  return rc;
}

/*
** xDisconnect method.
*/
static int unionDisconnect(sqlite3_vtab *pVtab){

    sqlite3_free(z);
  }
  sqlite3_free(z0);

  return rc;
}

/*
** This function may only be called for swarmvtab tables. The results of
** calling it on a unionvtab table are undefined.
**
** For a swarmvtab table, this function ensures that source database iSrc
** is open. If the database is opened successfully and the schema is as
** expected, or if it is already open when this function is called, SQLITE_OK
** is returned.
**
** Alternatively If an error occurs while opening the databases, or if the
** database schema is unsuitable, an SQLite error code is returned and (*pzErr)
** may be set to point to an English language error message. In this case it is
** the responsibility of the caller to eventually free the error message buffer
** using sqlite3_free(). 
*/
static int unionOpenDatabase(UnionTab *pTab, int iSrc, char **pzErr){
  int rc = SQLITE_OK;
  UnionSrc *pSrc = &pTab->aSrc[iSrc];

  assert( pTab->bSwarm && iSrc<pTab->nSrc );
  if( pSrc->db==0 ){

    rc = unionCloseSources(pTab, pTab->nMaxOpen-1);


    if( rc==SQLITE_OK ){
      rc = sqlite3_open_v2(pSrc->zFile, &pSrc->db, SQLITE_OPEN_READONLY, 0);
      if( rc!=SQLITE_OK ){
        *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(pSrc->db));
      }else{
        char *z = unionSourceToStr(&rc, pTab, pSrc, pzErr);
        if( rc==SQLITE_OK ){
          if( pTab->zSourceStr==0 ){
            pTab->zSourceStr = z;

        pSrc->pNextClosable = pTab->pClosable;
        pTab->pClosable = pSrc;
      }
    }
  }
  return rc;
}



/* 
** xConnect/xCreate method.
**
** The argv[] array contains the following:
**
**   argv[0]   -> module name  ("unionvtab")

    /* For unionvtab, verify that all source tables exist and have 
    ** compatible schemas. For swarmvtab, attach the first database and
    ** check that the first table is a rowid table only.  */
    if( rc==SQLITE_OK ){
      pTab->db = db;
      pTab->bSwarm = bSwarm;
      pTab->nMaxOpen = SWARMVTAB_MAX_OPEN;
      if( bSwarm ){
        rc = unionOpenDatabase(pTab, 0, pzErr);
      }else{
        rc = unionSourceCheck(pTab, pzErr);
      }
    }

    pTab = 0;
  }

  *ppVtab = (sqlite3_vtab*)pTab;
  return rc;
}


/*
** xOpen
*/
static int unionOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
  UnionCsr *pCsr;
  int rc = SQLITE_OK;
  (void)p;  /* Suppress harmless warning */

static int unionClose(sqlite3_vtab_cursor *cur){
  UnionCsr *pCsr = (UnionCsr*)cur;
  unionFinalizeCsrStmt(pCsr);
  sqlite3_free(pCsr);
  return SQLITE_OK;
}


/*
** This function does the work of the xNext() method. Except that, if it
** returns SQLITE_ROW, it should be called again within the same xNext()
** method call. See unionNext() for details.
*/
static int doUnionNext(UnionCsr *pCsr){
  int rc = SQLITE_OK;
  assert( pCsr->pStmt );
  if( sqlite3_step(pCsr->pStmt)!=SQLITE_ROW ){
    UnionTab *pTab = (UnionTab*)pCsr->base.pVtab;
    rc = unionFinalizeCsrStmt(pCsr);

      }
    }
  }

  return rc;
}

/*
** xNext
*/
static int unionNext(sqlite3_vtab_cursor *cur){
  int rc;
  do {
    rc = doUnionNext((UnionCsr*)cur);
  }while( rc==SQLITE_ROW );
  return rc;
}

    if( pTab->bSwarm ){
      pCsr->iTab = i;
      pCsr->iMaxRowid = iMax;
      rc = unionOpenDatabase(pTab, i, &pTab->base.zErrMsg);
      break;
    }
  }


  if( zSql==0 ){
    return rc;
  }else{
    sqlite3 *db = unionGetDb(pTab, &pTab->aSrc[pCsr->iTab]);
    pCsr->pStmt = unionPrepare(&rc, db, zSql, &pTab->base.zErrMsg);
    if( pCsr->pStmt ){

do_execsql_test 1.3 {
  CREATE VIRTUAL TABLE temp.s1 USING swarmvtab('SELECT * FROM dir');
  SELECT count(*) FROM s1 WHERE rowid<50;
} {49}

proc do_compare_test {tn where} {
  set sql [subst {

    SELECT (SELECT group_concat(a || ',' || b, ',') FROM t0 WHERE $where) 
           IS 
           (SELECT group_concat(a || ',' || b, ',') FROM s1 WHERE $where)

  }]

  uplevel [list do_execsql_test $tn $sql 1]
}

do_compare_test 1.4.1 "rowid = 700"
do_compare_test 1.4.2 "rowid = -1"
do_compare_test 1.4.3 "rowid = 0"
do_compare_test 1.4.4 "rowid = 55"
do_compare_test 1.4.5 "rowid BETWEEN 20 AND 100"
do_compare_test 1.4.6 "rowid > 350"
do_compare_test 1.4.7 "rowid >= 350"
do_compare_test 1.4.8 "rowid >= 200"
do_compare_test 1.4.9 "1"

# Multiple simultaneous cursors.
#
do_execsql_test 1.5.1.(5-seconds-or-so) {
  SELECT count(*) FROM s1 a, s1 b WHERE b.rowid<=200;
} {80000}
do_execsql_test 1.5.2 {
  SELECT count(*) FROM s1 a, s1 b, s1 c 
  WHERE a.rowid=b.rowid AND b.rowid=c.rowid;
} {400}

# Empty source tables.
#
do_test 1.6.0 {
  for {set i 0} {$i < 20} {incr i} {
    sqlite3 db2 test.db$i
    db2 eval " DELETE FROM t$i "
    db2 close
  }
  db eval { DELETE FROM t0 WHERE rowid<=200 }
} {}

do_compare_test 1.6.1 "rowid = 700"
do_compare_test 1.6.2 "rowid = -1"
do_compare_test 1.6.3 "rowid = 0"
do_compare_test 1.6.4 "rowid = 55"
do_compare_test 1.6.5 "rowid BETWEEN 20 AND 100"
do_compare_test 1.6.6 "rowid > 350"
do_compare_test 1.6.7 "rowid >= 350"
do_compare_test 1.6.8 "rowid >= 200"
do_compare_test 1.6.9 "1"
do_compare_test 1.6.10 "rowid >= 5"

do_test 1.x {
  set sqlite_open_file_count
} [expr $nFile+9]

do_test 1.y { db close } {}

# Delete all the database files created above.
#
for {set i 0} {$i < 40} {incr i} { forcedelete "test.db$i" }

#-------------------------------------------------------------------------
# Test some error conditions:
#
#   2.1: Database file does not exist.
#   2.2: Table does not exist.
#   2.3: Table schema does not match.
#
reset_db
load_static_extension db unionvtab
do_test 2.0.1 {
  db eval {
    CREATE TABLE t0(a INTEGER PRIMARY KEY, b TEXT);
    WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<400) 
      INSERT INTO t0 SELECT i, hex(randomblob(50)) FROM s;
    CREATE TABLE dir(f, t, imin, imax);
  }

  for {set i 0} {$i < 40} {incr i} {
    set iMin [expr $i*10 + 1]
    set iMax [expr $iMin+9]

    forcedelete "test.db$i"
    db eval [subst {
      ATTACH 'test.db$i' AS aux;
      CREATE TABLE aux.t$i (a INTEGER PRIMARY KEY, b TEXT);
      INSERT INTO aux.t$i SELECT * FROM t0 WHERE a BETWEEN $iMin AND $iMax;
      DETACH aux;
      INSERT INTO dir VALUES('test.db$i', 't$i', $iMin, $iMax);
    }]
  }
  execsql {
    CREATE VIRTUAL TABLE temp.s1 USING swarmvtab('SELECT * FROM dir');
  }
} {}

do_test 2.0.2 {
  forcedelete test.db5

  sqlite3 db2 test.db15
  db2 eval { DROP TABLE t15 }
  db2 close

  sqlite3 db2 test.db25
  db2 eval { 
    DROP TABLE t25;
    CREATE TABLE t25(x, y, z PRIMARY KEY);
  }
  db2 close
} {}

do_catchsql_test 2.1 {
  SELECT * FROM s1 WHERE rowid BETWEEN 1 AND 100;
} {1 {unable to open database file}}
do_catchsql_test 2.2 {
  SELECT * FROM s1 WHERE rowid BETWEEN 101 AND 200;
} {1 {no such rowid table: t15}}
do_catchsql_test 2.3 {
  SELECT * FROM s1 WHERE rowid BETWEEN 201 AND 300;
} {1 {source table schema mismatch}}

finish_test