SQLite

**
**   CREATE TABLE idx(id INTEGER PRIMARY KEY, path TEXT);
**   INSERT INTO idx VALUES(4, '/etc/passwd');
**
** Adding the row to the idx table automatically creates a row in the 
** virtual table with rowid=4, path=/etc/passwd and a text field that 
** contains data read from file /etc/passwd on disk.
**
*************************************************************************
** Virtual table module "fsdir"
**
** This module is designed to be used as a read-only eponymous virtual table.
** Its schema is as follows:
**
**   CREATE TABLE fsdir(dir TEXT, name TEXT);
**
** When queried, a WHERE term of the form "dir = $dir" must be provided. The
** virtual table then appears to have one row for each entry in file-system
** directory $dir. Column dir contains a copy of $dir, and column "name"
** contains the name of the directory entry.
**
** If the specified $dir cannot be opened or is not a directory, it is not
** an error. The virtual table appears to be empty in this case.
**
*************************************************************************
** Virtual table module "fstree"
**
** This module is also a read-only eponymous virtual table with the 
** following schema:
**
**   CREATE TABLE fstree(path TEXT, size INT, data BLOB);
**
** Running a "SELECT * FROM fstree" query on this table returns the entire
** contents of the file-system, starting at "/". To restrict the search
** space, the virtual table supports LIKE and GLOB constraints on the
** 'path' column. For example:
**
**   SELECT * FROM fstree WHERE path LIKE '/home/dan/sqlite/%'
*/
#include "sqliteInt.h"
#include "tcl.h"

#include <stdlib.h>
#include <string.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>

#if SQLITE_OS_UNIX
# include <unistd.h>
# include <dirent.h>
#endif
#if SQLITE_OS_WIN
# include <io.h>
#endif

#ifndef SQLITE_OMIT_VIRTUALTABLE

struct fs_cursor {
  sqlite3_vtab_cursor base;
  sqlite3_stmt *pStmt;
  char *zBuf;
  int nBuf;
  int nAlloc;
};

/*************************************************************************
** Start of fsdir implementation.
*/
typedef struct FsdirVtab FsdirVtab;
typedef struct FsdirCsr FsdirCsr;
struct FsdirVtab {
  sqlite3_vtab base;
};

struct FsdirCsr {
  sqlite3_vtab_cursor base;
  char *zDir;                     /* Buffer containing directory scanned */
  DIR *pDir;                      /* Open directory */
  sqlite3_int64 iRowid;
  struct dirent entry;            /* Current entry */
};

/*
** This function is the implementation of both the xConnect and xCreate
** methods of the fsdir virtual table.
**
** The argv[] array contains the following:
**
**   argv[0]   -> module name  ("fs")
**   argv[1]   -> database name
**   argv[2]   -> table name
**   argv[...] -> other module argument fields.
*/
static int fsdirConnect(
  sqlite3 *db,
  void *pAux,
  int argc, const char *const*argv,
  sqlite3_vtab **ppVtab,
  char **pzErr
){
  FsdirVtab *pTab;

  if( argc!=3 ){
    *pzErr = sqlite3_mprintf("wrong number of arguments");
    return SQLITE_ERROR;
  }

  pTab = (FsdirVtab *)sqlite3_malloc(sizeof(FsdirVtab));
  if( !pTab ) return SQLITE_NOMEM;
  memset(pTab, 0, sizeof(FsdirVtab));

  *ppVtab = &pTab->base;
  sqlite3_declare_vtab(db, "CREATE TABLE xyz(dir, name);");

  return SQLITE_OK;
}

/*
** xDestroy/xDisconnect implementation.
*/
static int fsdirDisconnect(sqlite3_vtab *pVtab){
  sqlite3_free(pVtab);
  return SQLITE_OK;
}

/*
** xBestIndex implementation. The only constraint supported is:
**
**   (dir = ?)
*/
static int fsdirBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){
  int ii;

  pIdxInfo->estimatedCost = 1000000000.0;

  for(ii=0; ii<pIdxInfo->nConstraint; ii++){
    struct sqlite3_index_constraint const *p = &pIdxInfo->aConstraint[ii];
    if( p->iColumn==0 && p->usable && p->op==SQLITE_INDEX_CONSTRAINT_EQ ){
      struct sqlite3_index_constraint_usage *pUsage;
      pUsage = &pIdxInfo->aConstraintUsage[ii];
      pUsage->omit = 1;
      pUsage->argvIndex = 1;
      pIdxInfo->idxNum = 1;
      pIdxInfo->estimatedCost = 1.0;
      break;
    }
  }

  return SQLITE_OK;
}

/*
** xOpen implementation.
**
** Open a new fsdir cursor.
*/
static int fsdirOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
  FsdirCsr *pCur;
  pCur = (FsdirCsr*)sqlite3_malloc(sizeof(FsdirCsr));
  if( pCur==0 ) return SQLITE_NOMEM;
  memset(pCur, 0, sizeof(FsdirCsr));
  *ppCursor = &pCur->base;
  return SQLITE_OK;
}

/*
** Close a fsdir cursor.
*/
static int fsdirClose(sqlite3_vtab_cursor *cur){
  FsdirCsr *pCur = (FsdirCsr*)cur;
  if( pCur->pDir ) closedir(pCur->pDir);
  sqlite3_free(pCur->zDir);
  sqlite3_free(pCur);
  return SQLITE_OK;
}

/*
** Skip the cursor to the next entry.
*/
static int fsdirNext(sqlite3_vtab_cursor *cur){
  FsdirCsr *pCsr = (FsdirCsr*)cur;

  if( pCsr->pDir ){
    struct dirent *pRes = 0;
    readdir_r(pCsr->pDir, &pCsr->entry, &pRes);
    if( pRes==0 ){
      closedir(pCsr->pDir);
      pCsr->pDir = 0;
    }
    pCsr->iRowid++;
  }

  return SQLITE_OK;
}

/*
** xFilter method implementation.
*/
static int fsdirFilter(
  sqlite3_vtab_cursor *pVtabCursor, 
  int idxNum, const char *idxStr,
  int argc, sqlite3_value **argv
){
  FsdirCsr *pCsr = (FsdirCsr*)pVtabCursor;
  int rc;
  const char *zDir;
  int nDir;


  if( idxNum!=1 || argc!=1 ){
    return SQLITE_ERROR;
  }

  pCsr->iRowid = 0;
  sqlite3_free(pCsr->zDir);
  if( pCsr->pDir ){
    closedir(pCsr->pDir);
    pCsr->pDir = 0;
  }

  zDir = (const char*)sqlite3_value_text(argv[0]);
  nDir = sqlite3_value_bytes(argv[0]);
  pCsr->zDir = sqlite3_malloc(nDir+1);
  if( pCsr->zDir==0 ) return SQLITE_NOMEM;
  memcpy(pCsr->zDir, zDir, nDir+1);

  pCsr->pDir = opendir(pCsr->zDir);
  return fsdirNext(pVtabCursor); 
}

/*
** xEof method implementation.
*/
static int fsdirEof(sqlite3_vtab_cursor *cur){
  FsdirCsr *pCsr = (FsdirCsr*)cur;
  return pCsr->pDir==0;
}

/*
** xColumn method implementation.
*/
static int fsdirColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){
  FsdirCsr *pCsr = (FsdirCsr*)cur;
  switch( i ){
    case 0: /* dir */
      sqlite3_result_text(ctx, pCsr->zDir, -1, SQLITE_STATIC);
      break;

    case 1: /* name */
      sqlite3_result_text(ctx, pCsr->entry.d_name, -1, SQLITE_TRANSIENT);
      break;

    default:
      assert( 0 );
  }

  return SQLITE_OK;
}

/*
** xRowid method implementation.
*/
static int fsdirRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
  FsdirCsr *pCsr = (FsdirCsr*)cur;
  *pRowid = pCsr->iRowid;
  return SQLITE_OK;
}
/*
** End of fsdir implementation.
*************************************************************************/

/*************************************************************************
** Start of fstree implementation.
*/
typedef struct FstreeVtab FstreeVtab;
typedef struct FstreeCsr FstreeCsr;
struct FstreeVtab {
  sqlite3_vtab base;
  sqlite3 *db;
};

struct FstreeCsr {
  sqlite3_vtab_cursor base;
  sqlite3_stmt *pStmt;            /* Statement to list paths */
  int fd;                         /* File descriptor open on current path */
};

/*
** This function is the implementation of both the xConnect and xCreate
** methods of the fstree virtual table.
**
** The argv[] array contains the following:
**
**   argv[0]   -> module name  ("fs")
**   argv[1]   -> database name
**   argv[2]   -> table name
**   argv[...] -> other module argument fields.
*/
static int fstreeConnect(
  sqlite3 *db,
  void *pAux,
  int argc, const char *const*argv,
  sqlite3_vtab **ppVtab,
  char **pzErr
){
  FstreeVtab *pTab;

  if( argc!=3 ){
    *pzErr = sqlite3_mprintf("wrong number of arguments");
    return SQLITE_ERROR;
  }

  pTab = (FstreeVtab *)sqlite3_malloc(sizeof(FstreeVtab));
  if( !pTab ) return SQLITE_NOMEM;
  memset(pTab, 0, sizeof(FstreeVtab));
  pTab->db = db;

  *ppVtab = &pTab->base;
  sqlite3_declare_vtab(db, "CREATE TABLE xyz(path, size, data);");

  return SQLITE_OK;
}

/*
** xDestroy/xDisconnect implementation.
*/
static int fstreeDisconnect(sqlite3_vtab *pVtab){
  sqlite3_free(pVtab);
  return SQLITE_OK;
}

/*
** xBestIndex implementation. The only constraint supported is:
**
**   (dir = ?)
*/
static int fstreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){
  int ii;

  for(ii=0; ii<pIdxInfo->nConstraint; ii++){
    struct sqlite3_index_constraint const *p = &pIdxInfo->aConstraint[ii];
    if( p->iColumn==0 && p->usable && (
          p->op==SQLITE_INDEX_CONSTRAINT_GLOB
       || p->op==SQLITE_INDEX_CONSTRAINT_LIKE
       || p->op==SQLITE_INDEX_CONSTRAINT_EQ
    )){
      struct sqlite3_index_constraint_usage *pUsage;
      pUsage = &pIdxInfo->aConstraintUsage[ii];
      pIdxInfo->idxNum = p->op;
      pUsage->argvIndex = 1;
      pIdxInfo->estimatedCost = 100000.0;
      return SQLITE_OK;
    }
  }

  pIdxInfo->estimatedCost = 1000000000.0;
  return SQLITE_OK;
}

/*
** xOpen implementation.
**
** Open a new fstree cursor.
*/
static int fstreeOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
  FstreeCsr *pCur;
  pCur = (FstreeCsr*)sqlite3_malloc(sizeof(FstreeCsr));
  if( pCur==0 ) return SQLITE_NOMEM;
  memset(pCur, 0, sizeof(FstreeCsr));
  pCur->fd = -1;
  *ppCursor = &pCur->base;
  return SQLITE_OK;
}

static void fstreeCloseFd(FstreeCsr *pCsr){
  if( pCsr->fd>=0 ){
    close(pCsr->fd);
    pCsr->fd = -1;
  }
}

/*
** Close a fstree cursor.
*/
static int fstreeClose(sqlite3_vtab_cursor *cur){
  FstreeCsr *pCsr = (FstreeCsr*)cur;
  sqlite3_finalize(pCsr->pStmt);
  fstreeCloseFd(pCsr);
  sqlite3_free(pCsr);
  return SQLITE_OK;
}

/*
** Skip the cursor to the next entry.
*/
static int fstreeNext(sqlite3_vtab_cursor *cur){
  FstreeCsr *pCsr = (FstreeCsr*)cur;
  int rc;

  fstreeCloseFd(pCsr);
  rc = sqlite3_step(pCsr->pStmt);
  if( rc!=SQLITE_ROW ){
    rc = sqlite3_finalize(pCsr->pStmt);
    pCsr->pStmt = 0;
  }else{
    rc = SQLITE_OK;
    pCsr->fd = open(sqlite3_column_text(pCsr->pStmt, 0), O_RDONLY);
  }

  return rc;
}

/*
** xFilter method implementation.
*/
static int fstreeFilter(
  sqlite3_vtab_cursor *pVtabCursor, 
  int idxNum, const char *idxStr,
  int argc, sqlite3_value **argv
){
  FstreeCsr *pCsr = (FstreeCsr*)pVtabCursor;
  FstreeVtab *pTab = (FstreeVtab*)(pCsr->base.pVtab);
  int rc;
  const char *zSql = 
"WITH r(d) AS ("
"  SELECT CASE WHEN dir='/' THEN '' ELSE dir END || '/' || name "
"    FROM fsdir WHERE dir=? AND name NOT LIKE '.%'"
"  UNION ALL"
"  SELECT dir || '/' || name FROM r, fsdir WHERE dir=d AND name NOT LIKE '.%'"
") SELECT d FROM r;";

  const char *zDir = "/";
  int nDir = 1;
  char aWild[2] = {'\0', '\0' };

  fstreeCloseFd(pCsr);
  sqlite3_finalize(pCsr->pStmt);
  pCsr->pStmt = 0;
  rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pCsr->pStmt, 0);
  if( rc!=SQLITE_OK ) return rc;

  if( idxNum ){
    const char *zQuery = (const char*)sqlite3_value_text(argv[0]);
    switch( idxNum ){
      case SQLITE_INDEX_CONSTRAINT_GLOB:
        aWild[0] = '*';
        aWild[1] = '?';
        break;
      case SQLITE_INDEX_CONSTRAINT_LIKE:
        aWild[0] = '_';
        aWild[1] = '%';
        break;
    }

    if( zQuery[0]=='/' ){
      int i;
      for(i=1; zQuery[i]; i++){
        if( zQuery[i]==aWild[0] || zQuery[i]==aWild[1] ) break;
        if( zQuery[i]=='/' ) nDir = i;
      }
      zDir = zQuery;
    }
  }

  sqlite3_bind_text(pCsr->pStmt, 1, zDir, nDir, SQLITE_TRANSIENT);

  return fstreeNext(pVtabCursor); 
}

/*
** xEof method implementation.
*/
static int fstreeEof(sqlite3_vtab_cursor *cur){
  FstreeCsr *pCsr = (FstreeCsr*)cur;
  return pCsr->pStmt==0;
}

/*
** xColumn method implementation.
*/
static int fstreeColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){
  FstreeCsr *pCsr = (FstreeCsr*)cur;
  if( i==0 ){      /* path */
    sqlite3_result_value(ctx, sqlite3_column_value(pCsr->pStmt, 0));
  }else{
    struct stat sBuf;
    fstat(pCsr->fd, &sBuf);

    if( S_ISREG(sBuf.st_mode) ){
      if( i==1 ){
        sqlite3_result_int64(ctx, sBuf.st_size);
      }else{
        int nRead;
        char *aBuf = sqlite3_malloc(sBuf.st_mode+1);
        if( !aBuf ) return SQLITE_NOMEM;
        nRead = read(pCsr->fd, aBuf, sBuf.st_mode);
        if( nRead!=sBuf.st_mode ){
          return SQLITE_IOERR;
        }
        sqlite3_result_blob(ctx, aBuf, nRead, SQLITE_TRANSIENT);
        sqlite3_free(aBuf);
      }
    }
  }

  return SQLITE_OK;
}

/*
** xRowid method implementation.
*/
static int fstreeRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
  *pRowid = 0;
  return SQLITE_OK;
}
/*
** End of fstree implementation.
*************************************************************************/




/*
** This function is the implementation of both the xConnect and xCreate
** methods of the fs virtual table.
**
** The argv[] array contains the following:
**

  pVtab->zTbl = (char *)&pVtab[1];
  pVtab->zDb = &pVtab->zTbl[strlen(zTbl)+1];
  pVtab->db = db;
  memcpy(pVtab->zTbl, zTbl, strlen(zTbl));
  memcpy(pVtab->zDb, zDb, strlen(zDb));
  *ppVtab = &pVtab->base;
  sqlite3_declare_vtab(db, "CREATE TABLE x(path TEXT, data TEXT)");

  return SQLITE_OK;
}
/* Note that for this virtual table, the xCreate and xConnect
** methods are identical. */

static int fsDisconnect(sqlite3_vtab *pVtab){

  }
  return rc;
}

static int fsColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){
  fs_cursor *pCur = (fs_cursor*)cur;

  assert( i==0 || i==1 || i==2 );
  if( i==0 ){
    sqlite3_result_value(ctx, sqlite3_column_value(pCur->pStmt, 0));
  }else{
    const char *zFile = (const char *)sqlite3_column_text(pCur->pStmt, 1);
    struct stat sbuf;
    int fd;


    int n;
    fd = open(zFile, O_RDONLY);
    if( fd<0 ) return SQLITE_IOERR;
    fstat(fd, &sbuf);

    if( sbuf.st_size>=pCur->nAlloc ){
      int nNew = sbuf.st_size*2;
      char *zNew;

  0,                           /* xSync */
  0,                           /* xCommit */
  0,                           /* xRollback */
  0,                           /* xFindMethod */
  0,                           /* xRename */
};

static sqlite3_module fsdirModule = {
  0,                              /* iVersion */
  fsdirConnect,                   /* xCreate */
  fsdirConnect,                   /* xConnect */
  fsdirBestIndex,                 /* xBestIndex */
  fsdirDisconnect,                /* xDisconnect */
  fsdirDisconnect,                /* xDestroy */
  fsdirOpen,                      /* xOpen - open a cursor */
  fsdirClose,                     /* xClose - close a cursor */
  fsdirFilter,                    /* xFilter - configure scan constraints */
  fsdirNext,                      /* xNext - advance a cursor */
  fsdirEof,                       /* xEof - check for end of scan */
  fsdirColumn,                    /* xColumn - read data */
  fsdirRowid,                     /* xRowid - read data */
  0,                              /* xUpdate */
  0,                              /* xBegin */
  0,                              /* xSync */
  0,                              /* xCommit */
  0,                              /* xRollback */
  0,                              /* xFindMethod */
  0,                              /* xRename */
};

static sqlite3_module fstreeModule = {
  0,                              /* iVersion */
  fstreeConnect,                  /* xCreate */
  fstreeConnect,                  /* xConnect */
  fstreeBestIndex,                /* xBestIndex */
  fstreeDisconnect,               /* xDisconnect */
  fstreeDisconnect,               /* xDestroy */
  fstreeOpen,                     /* xOpen - open a cursor */
  fstreeClose,                    /* xClose - close a cursor */
  fstreeFilter,                   /* xFilter - configure scan constraints */
  fstreeNext,                     /* xNext - advance a cursor */
  fstreeEof,                      /* xEof - check for end of scan */
  fstreeColumn,                   /* xColumn - read data */
  fstreeRowid,                    /* xRowid - read data */
  0,                              /* xUpdate */
  0,                              /* xBegin */
  0,                              /* xSync */
  0,                              /* xCommit */
  0,                              /* xRollback */
  0,                              /* xFindMethod */
  0,                              /* xRename */
};

/*
** Decode a pointer to an sqlite3 object.
*/
extern int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb);

/*
** Register the echo virtual table module.

  if( objc!=2 ){
    Tcl_WrongNumArgs(interp, 1, objv, "DB");
    return TCL_ERROR;
  }
  if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR;
#ifndef SQLITE_OMIT_VIRTUALTABLE
  sqlite3_create_module(db, "fs", &fsModule, (void *)interp);
  sqlite3_create_module(db, "fsdir", &fsdirModule, 0);
  sqlite3_create_module(db, "fstree", &fstreeModule, 0);
#endif
  return TCL_OK;
}

#endif

do_execsql_test 10.1 {
  CREATE TABLE idx(id INTEGER PRIMARY KEY, path TEXT);
  INSERT INTO idx VALUES (1, 't1.txt');
  INSERT INTO idx VALUES (2, 't2.txt');
  INSERT INTO idx VALUES (3, 't3.txt');

  CREATE VIRTUAL TABLE vt USING fs(idx);
  SELECT path, data FROM vt;
} {
  1 {a b c d e f g h i j k l m n o p q r s t u v w x y z} 
  2 {a b c d e f g h i j k l m a b c d e f g h i j k l m}
  3 {n o p q r s t u v w x y z n o p q r s t u v w x y z}
}

do_execsql_test 10.2 {
  SELECT path, data FROM vt WHERE rowid = 2;
} {
  2 {a b c d e f g h i j k l m a b c d e f g h i j k l m}
}

do_execsql_test 10.3 {
  CREATE VIRTUAL TABLE ft USING fts4(content=vt);
  INSERT INTO ft(ft) VALUES('rebuild');

foreach ::tclvar_set_omit {0 1} {
  foreach {tn expr res cnt} {
    1 {value GLOB 'aban*'} {x3 abandon x4 abandonint} 2
    2 {value LIKE '%ac%'}  {x1 aback x7 backbone x8 backarrow} 300
    3 {value REGEXP '^......$'}  {x5 babble x6 baboon x9 castle} 30000
  } {

    db cache flush
    set ::gfunc 0
    if {$::tclvar_set_omit} {set cnt 0}

    do_test 2.$tclvar_set_omit.$tn.1 {
      execsql "SELECT name, value FROM vars WHERE name MATCH 'x*' AND $expr"
    } $res

    do_test 2.$tclvar_set_omit.$tn.2 {
      set ::gfunc
    } $cnt
  }
}

#-------------------------------------------------------------------------
#
if {$::tcl_platform(platform)=="unix"} {
  reset_db
  register_fs_module db
  do_execsql_test 3.0 {
    SELECT name FROM fsdir WHERE dir = '.' AND name = 'test.db';
    SELECT name FROM fsdir WHERE dir = '.' AND name = '.'
  } {test.db .}
  
  # Read the first 5 entries from the root directory.
  #
  set res [list]
  foreach p [lrange [exec ls -U /] 0 4] {
    lappend res "/$p"
  }
  do_execsql_test 3.1 {
    SELECT path FROM fstree LIMIT 5;
  } $res
  
  # Read all entries in the current directory.
  #
  proc contents {pattern} {
    set res [list]
    foreach f [glob -nocomplain $pattern] {
      lappend res $f
      if {[file isdir $f]} {
        set res [concat $res [contents "$f/*"]]
      }
    }
    set res
  }
  set pwd "[pwd]/*"
  set res [contents $pwd]
  do_execsql_test 3.2 {
    SELECT path FROM fstree WHERE path GLOB $pwd ORDER BY 1
  } [lsort $res]
  
  # Add some sub-directories and files to the current directory.
  #
  do_test 3.3 {
    catch { file delete -force subdir }
    foreach {path sz} {
      subdir/x1.txt     143
      subdir/x2.txt     153
    } { 
      set dir [file dirname $path]
      catch { file mkdir $dir }
      set fd [open $path w]
      puts -nonewline $fd [string repeat 1 $sz]
      close $fd
    }
  } {}
  
  set pwd [pwd]
  do_execsql_test 3.5 {
    SELECT path, size FROM fstree WHERE path GLOB $pwd || '/subdir/*' ORDER BY 1
  } [list \
    "$pwd/subdir/x1.txt" 143 \
    "$pwd/subdir/x2.txt" 153 \
  ]
  do_execsql_test 3.6 {
    SELECT path, size FROM fstree WHERE path LIKE $pwd || '/subdir/%' ORDER BY 1
  } [list \
    "$pwd/subdir/x1.txt" 143 \
    "$pwd/subdir/x2.txt" 153 \
  ]
  do_execsql_test 3.7 {
    SELECT sum(size) FROM fstree WHERE path LIKE $pwd || '/subdir/%'
  } 296
  do_execsql_test 3.8 {
    SELECT size FROM fstree WHERE path = $pwd || '/subdir/x1.txt'
  } 143

}


finish_test