/ Check-in [0394889a]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Enhance virtual table "fsdir" in ext/misc/fileio.c. Add support for "-C" to the shell command's ".ar c" command.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | sqlar-shell-support
Files: files | file ages | folders
SHA3-256: 0394889afed2479773af594e2d9659cf58b8959004ebcdeaff8e08e5dae684ef
User & Date: dan 2017-12-11 20:22:02
Context
2017-12-12
20:04
Add support for parsing options in non-traditional tar form to the ".ar" command. Have writefile() attempt to create any missing path components. And not to throw an exception if it is called to create a directory that already exists. check-in: 38dbeb1e user: dan tags: sqlar-shell-support
2017-12-11
20:22
Enhance virtual table "fsdir" in ext/misc/fileio.c. Add support for "-C" to the shell command's ".ar c" command. check-in: 0394889a user: dan tags: sqlar-shell-support
2017-12-09
18:28
Add support for -C to ".ar x". check-in: 8cd70960 user: dan tags: sqlar-shell-support
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/misc/fileio.c.

    38     38   #include <fcntl.h>
    39     39   #include <unistd.h>
    40     40   #include <dirent.h>
    41     41   #include <time.h>
    42     42   #include <utime.h>
    43     43   
    44     44   
    45         -#define FSDIR_SCHEMA "CREATE TABLE x(name,mode,mtime,data,dir HIDDEN)"
           45  +#define FSDIR_SCHEMA "(name,mode,mtime,data,path HIDDEN,dir HIDDEN)"
    46     46   
    47     47   static void readFileContents(sqlite3_context *ctx, const char *zName){
    48     48     FILE *in;
    49     49     long nIn;
    50     50     void *pBuf;
    51     51   
    52     52     in = fopen(zName, "rb");
................................................................................
   162    162       }
   163    163     }
   164    164   }
   165    165   
   166    166   #ifndef SQLITE_OMIT_VIRTUALTABLE
   167    167   
   168    168   /* 
          169  +** Cursor type for recursively iterating through a directory structure.
   169    170   */
   170    171   typedef struct fsdir_cursor fsdir_cursor;
          172  +typedef struct FsdirLevel FsdirLevel;
          173  +
          174  +struct FsdirLevel {
          175  +  DIR *pDir;                 /* From opendir() */
          176  +  char *zDir;                /* Name of directory (nul-terminated) */
          177  +};
          178  +
   171    179   struct fsdir_cursor {
   172    180     sqlite3_vtab_cursor base;  /* Base class - must be first */
   173         -  int eType;                 /* One of FSDIR_DIR or FSDIR_ENTRY */
   174         -  DIR *pDir;                 /* From opendir() */
          181  +
          182  +  int nLvl;                  /* Number of entries in aLvl[] array */
          183  +  int iLvl;                  /* Index of current entry */
          184  +  FsdirLevel *aLvl;          /* Hierarchy of directories being traversed */
          185  +
          186  +  const char *zBase;
          187  +  int nBase;
          188  +
   175    189     struct stat sStat;         /* Current lstat() results */
   176         -  char *zDir;                /* Directory to read */
   177         -  int nDir;                  /* Value of strlen(zDir) */
   178    190     char *zPath;               /* Path to current entry */
   179         -  int bEof;
   180    191     sqlite3_int64 iRowid;      /* Current rowid */
   181    192   };
   182    193   
   183    194   typedef struct fsdir_tab fsdir_tab;
   184    195   struct fsdir_tab {
   185    196     sqlite3_vtab base;         /* Base class - must be first */
   186         -  int eType;                 /* One of FSDIR_DIR or FSDIR_ENTRY */
   187    197   };
   188    198   
   189         -#define FSDIR_DIR   0
   190         -#define FSDIR_ENTRY 1
   191         -
   192    199   /*
   193    200   ** Construct a new fsdir virtual table object.
   194    201   */
   195    202   static int fsdirConnect(
   196    203     sqlite3 *db,
   197    204     void *pAux,
   198    205     int argc, const char *const*argv,
   199    206     sqlite3_vtab **ppVtab,
   200    207     char **pzErr
   201    208   ){
   202    209     fsdir_tab *pNew = 0;
   203    210     int rc;
   204    211   
   205         -  rc = sqlite3_declare_vtab(db, FSDIR_SCHEMA);
          212  +  rc = sqlite3_declare_vtab(db, "CREATE TABLE x" FSDIR_SCHEMA);
   206    213     if( rc==SQLITE_OK ){
   207    214       pNew = (fsdir_tab*)sqlite3_malloc( sizeof(*pNew) );
   208    215       if( pNew==0 ) return SQLITE_NOMEM;
   209    216       memset(pNew, 0, sizeof(*pNew));
   210         -    pNew->eType = (pAux==0 ? FSDIR_DIR : FSDIR_ENTRY);
   211    217     }
   212    218     *ppVtab = (sqlite3_vtab*)pNew;
   213    219     return rc;
   214    220   }
   215    221   
   216    222   /*
   217    223   ** This method is the destructor for fsdir vtab objects.
................................................................................
   225    231   ** Constructor for a new fsdir_cursor object.
   226    232   */
   227    233   static int fsdirOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
   228    234     fsdir_cursor *pCur;
   229    235     pCur = sqlite3_malloc( sizeof(*pCur) );
   230    236     if( pCur==0 ) return SQLITE_NOMEM;
   231    237     memset(pCur, 0, sizeof(*pCur));
   232         -  pCur->eType = ((fsdir_tab*)p)->eType;
          238  +  pCur->iLvl = -1;
   233    239     *ppCursor = &pCur->base;
   234    240     return SQLITE_OK;
   235    241   }
          242  +
          243  +static void fsdirResetCursor(fsdir_cursor *pCur){
          244  +  int i;
          245  +  for(i=0; i<=pCur->iLvl; i++){
          246  +    FsdirLevel *pLvl = &pCur->aLvl[i];
          247  +    if( pLvl->pDir ) closedir(pLvl->pDir);
          248  +    sqlite3_free(pLvl->zDir);
          249  +  }
          250  +  sqlite3_free(pCur->zPath);
          251  +  pCur->aLvl = 0;
          252  +  pCur->zPath = 0;
          253  +  pCur->zBase = 0;
          254  +  pCur->nBase = 0;
          255  +  pCur->iLvl = -1;
          256  +  pCur->iRowid = 1;
          257  +}
   236    258   
   237    259   /*
   238    260   ** Destructor for an fsdir_cursor.
   239    261   */
   240    262   static int fsdirClose(sqlite3_vtab_cursor *cur){
   241    263     fsdir_cursor *pCur = (fsdir_cursor*)cur;
   242         -  if( pCur->pDir ) closedir(pCur->pDir);
   243         -  sqlite3_free(pCur->zDir);
   244         -  sqlite3_free(pCur->zPath);
          264  +
          265  +  fsdirResetCursor(pCur);
          266  +  sqlite3_free(pCur->aLvl);
   245    267     sqlite3_free(pCur);
   246    268     return SQLITE_OK;
   247    269   }
          270  +
          271  +static void fsdirSetErrmsg(fsdir_cursor *pCur, const char *zFmt, ...){
          272  +  va_list ap;
          273  +  va_start(ap, zFmt);
          274  +  pCur->base.pVtab->zErrMsg = sqlite3_vmprintf(zFmt, ap);
          275  +  va_end(ap);
          276  +}
          277  +
   248    278   
   249    279   /*
   250    280   ** Advance an fsdir_cursor to its next row of output.
   251    281   */
   252    282   static int fsdirNext(sqlite3_vtab_cursor *cur){
   253    283     fsdir_cursor *pCur = (fsdir_cursor*)cur;
   254         -  struct dirent *pEntry;
   255         -
   256         -  if( pCur->eType==FSDIR_ENTRY ){
   257         -    pCur->bEof = 1;
   258         -    return SQLITE_OK;
   259         -  }
   260         -
   261         -  sqlite3_free(pCur->zPath);
   262         -  pCur->zPath = 0;
   263         -
   264         -  while( 1 ){
   265         -    pEntry = readdir(pCur->pDir);
   266         -    if( pEntry ){
   267         -      if( strcmp(pEntry->d_name, ".") 
   268         -       && strcmp(pEntry->d_name, "..") 
   269         -      ){
   270         -        pCur->zPath = sqlite3_mprintf("%s/%s", pCur->zDir, pEntry->d_name);
   271         -        if( pCur->zPath==0 ) return SQLITE_NOMEM;
   272         -        lstat(pCur->zPath, &pCur->sStat);
   273         -        break;
   274         -      }
   275         -    }else{
   276         -      pCur->bEof = 1;
   277         -      break;
   278         -    }
   279         -  }
          284  +  mode_t m = pCur->sStat.st_mode;
   280    285   
   281    286     pCur->iRowid++;
          287  +  if( S_ISDIR(m) ){
          288  +    /* Descend into this directory */
          289  +    int iNew = pCur->iLvl + 1;
          290  +    FsdirLevel *pLvl;
          291  +    if( iNew>=pCur->nLvl ){
          292  +      int nNew = iNew+1;
          293  +      int nByte = nNew*sizeof(FsdirLevel);
          294  +      FsdirLevel *aNew = (FsdirLevel*)sqlite3_realloc(pCur->aLvl, nByte);
          295  +      if( aNew==0 ) return SQLITE_NOMEM;
          296  +      memset(&aNew[pCur->nLvl], 0, sizeof(FsdirLevel)*(nNew-pCur->nLvl));
          297  +      pCur->aLvl = aNew;
          298  +      pCur->nLvl = nNew;
          299  +    }
          300  +    pCur->iLvl = iNew;
          301  +    pLvl = &pCur->aLvl[iNew];
          302  +    
          303  +    pLvl->zDir = pCur->zPath;
          304  +    pCur->zPath = 0;
          305  +    pLvl->pDir = opendir(pLvl->zDir);
          306  +    if( pLvl->pDir==0 ){
          307  +      fsdirSetErrmsg(pCur, "cannot read directory: %s", pCur->zPath);
          308  +      return SQLITE_ERROR;
          309  +    }
          310  +  }
          311  +
          312  +  while( pCur->iLvl>=0 ){
          313  +    FsdirLevel *pLvl = &pCur->aLvl[pCur->iLvl];
          314  +    struct dirent *pEntry = readdir(pLvl->pDir);
          315  +    if( pEntry ){
          316  +      if( pEntry->d_name[0]=='.' ){
          317  +       if( pEntry->d_name[1]=='.' && pEntry->d_name[2]=='\0' ) continue;
          318  +       if( pEntry->d_name[1]=='\0' ) continue;
          319  +      }
          320  +      sqlite3_free(pCur->zPath);
          321  +      pCur->zPath = sqlite3_mprintf("%s/%s", pLvl->zDir, pEntry->d_name);
          322  +      if( pCur->zPath==0 ) return SQLITE_NOMEM;
          323  +      if( lstat(pCur->zPath, &pCur->sStat) ){
          324  +        fsdirSetErrmsg(pCur, "cannot stat file: %s", pCur->zPath);
          325  +        return SQLITE_ERROR;
          326  +      }
          327  +      return SQLITE_OK;
          328  +    }
          329  +    closedir(pLvl->pDir);
          330  +    sqlite3_free(pLvl->zDir);
          331  +    pLvl->pDir = 0;
          332  +    pLvl->zDir = 0;
          333  +    pCur->iLvl--;
          334  +  }
          335  +
          336  +  /* EOF */
          337  +  sqlite3_free(pCur->zPath);
          338  +  pCur->zPath = 0;
   282    339     return SQLITE_OK;
   283    340   }
   284    341   
   285    342   /*
   286    343   ** Return values of columns for the row at which the series_cursor
   287    344   ** is currently pointing.
   288    345   */
................................................................................
   290    347     sqlite3_vtab_cursor *cur,   /* The cursor */
   291    348     sqlite3_context *ctx,       /* First argument to sqlite3_result_...() */
   292    349     int i                       /* Which column to return */
   293    350   ){
   294    351     fsdir_cursor *pCur = (fsdir_cursor*)cur;
   295    352     switch( i ){
   296    353       case 0: { /* name */
   297         -      const char *zName;
   298         -      if( pCur->eType==FSDIR_DIR ){
   299         -        zName = &pCur->zPath[pCur->nDir+1];
   300         -      }else{
   301         -        zName = pCur->zPath;
   302         -      }
   303         -      sqlite3_result_text(ctx, zName, -1, SQLITE_TRANSIENT);
          354  +      sqlite3_result_text(ctx, &pCur->zPath[pCur->nBase], -1, SQLITE_TRANSIENT);
   304    355         break;
   305    356       }
   306    357   
   307    358       case 1: /* mode */
   308    359         sqlite3_result_int64(ctx, pCur->sStat.st_mode);
   309    360         break;
   310    361   
   311         -    case 2: /* mode */
          362  +    case 2: /* mtime */
   312    363         sqlite3_result_int64(ctx, pCur->sStat.st_mtime);
   313    364         break;
   314    365   
   315         -    case 3: {
          366  +    case 3: { /* data */
   316    367         mode_t m = pCur->sStat.st_mode;
   317    368         if( S_ISDIR(m) ){
   318    369           sqlite3_result_null(ctx);
   319    370         }else if( S_ISLNK(m) ){
   320    371           char aStatic[64];
   321    372           char *aBuf = aStatic;
   322    373           int nBuf = 64;
................................................................................
   357    408   
   358    409   /*
   359    410   ** Return TRUE if the cursor has been moved off of the last
   360    411   ** row of output.
   361    412   */
   362    413   static int fsdirEof(sqlite3_vtab_cursor *cur){
   363    414     fsdir_cursor *pCur = (fsdir_cursor*)cur;
   364         -  return pCur->bEof;
   365         -}
   366         -
   367         -static void fsdirSetErrmsg(fsdir_cursor *pCur, const char *zFmt, ...){
   368         -  va_list ap;
   369         -  va_start(ap, zFmt);
   370         -  pCur->base.pVtab->zErrMsg = sqlite3_vmprintf(zFmt, ap);
   371         -  va_end(ap);
          415  +  return (pCur->zPath==0);
   372    416   }
   373    417   
   374    418   /*
   375    419   ** xFilter callback.
   376    420   */
   377    421   static int fsdirFilter(
   378    422     sqlite3_vtab_cursor *cur, 
   379    423     int idxNum, const char *idxStr,
   380    424     int argc, sqlite3_value **argv
   381    425   ){
   382    426     const char *zDir = 0;
   383    427     fsdir_cursor *pCur = (fsdir_cursor*)cur;
   384    428   
   385         -  sqlite3_free(pCur->zDir);
   386         -  pCur->iRowid = 0;
   387         -  pCur->zDir = 0;
   388         -  pCur->bEof = 0;
   389         -  if( pCur->pDir ){
   390         -    closedir(pCur->pDir);
   391         -    pCur->pDir = 0;
   392         -  }
          429  +  fsdirResetCursor(pCur);
   393    430   
   394    431     if( idxNum==0 ){
   395    432       fsdirSetErrmsg(pCur, "table function fsdir requires an argument");
   396    433       return SQLITE_ERROR;
   397    434     }
   398    435   
   399         -  assert( argc==1 );
          436  +  assert( argc==idxNum && (argc==1 || argc==2) );
   400    437     zDir = (const char*)sqlite3_value_text(argv[0]);
   401    438     if( zDir==0 ){
   402    439       fsdirSetErrmsg(pCur, "table function fsdir requires a non-NULL argument");
   403    440       return SQLITE_ERROR;
   404    441     }
          442  +  if( argc==2 ){
          443  +    pCur->zBase = (const char*)sqlite3_value_text(argv[1]);
          444  +  }
          445  +  if( pCur->zBase ){
          446  +    pCur->nBase = strlen(pCur->zBase)+1;
          447  +    pCur->zPath = sqlite3_mprintf("%s/%s", pCur->zBase, zDir);
          448  +  }else{
          449  +    pCur->zPath = sqlite3_mprintf("%s", zDir);
          450  +  }
   405    451   
   406         -  pCur->zDir = sqlite3_mprintf("%s", zDir);
   407         -  if( pCur->zDir==0 ){
          452  +  if( pCur->zPath==0 ){
   408    453       return SQLITE_NOMEM;
   409    454     }
          455  +  if( lstat(pCur->zPath, &pCur->sStat) ){
          456  +    fsdirSetErrmsg(pCur, "cannot stat file: %s", pCur->zPath);
          457  +    return SQLITE_ERROR;
          458  +  }
   410    459   
   411         -  if( pCur->eType==FSDIR_ENTRY ){
   412         -    int rc = lstat(pCur->zDir, &pCur->sStat);
   413         -    if( rc ){
   414         -      fsdirSetErrmsg(pCur, "cannot stat file: %s", pCur->zDir);
   415         -    }else{
   416         -      pCur->zPath = sqlite3_mprintf("%s", pCur->zDir);
   417         -      if( pCur->zPath==0 ) return SQLITE_NOMEM;
   418         -    }
   419         -    return SQLITE_OK;
   420         -  }else{
   421         -    pCur->nDir = strlen(pCur->zDir);
   422         -    pCur->pDir = opendir(zDir);
   423         -    if( pCur->pDir==0 ){
   424         -      fsdirSetErrmsg(pCur, "error in opendir(\"%s\")", zDir);
   425         -      return SQLITE_ERROR;
   426         -    }
   427         -
   428         -    return fsdirNext(cur);
   429         -  }
          460  +  return SQLITE_OK;
   430    461   }
   431    462   
   432    463   /*
   433    464   ** SQLite will invoke this method one or more times while planning a query
   434    465   ** that uses the generate_series virtual table.  This routine needs to create
   435    466   ** a query plan for each invocation and compute an estimated cost for that
   436    467   ** plan.
................................................................................
   446    477   **  (8)  output in descending order
   447    478   */
   448    479   static int fsdirBestIndex(
   449    480     sqlite3_vtab *tab,
   450    481     sqlite3_index_info *pIdxInfo
   451    482   ){
   452    483     int i;                 /* Loop over constraints */
          484  +  int idx4 = -1;
          485  +  int idx5 = -1;
   453    486   
   454    487     const struct sqlite3_index_constraint *pConstraint;
   455    488     pConstraint = pIdxInfo->aConstraint;
   456    489     for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){
   457    490       if( pConstraint->usable==0 ) continue;
   458    491       if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue;
   459         -    if( pConstraint->iColumn!=4 ) continue;
   460         -    break;
          492  +    if( pConstraint->iColumn==4 ) idx4 = i;
          493  +    if( pConstraint->iColumn==5 ) idx5 = i;
   461    494     }
   462    495   
   463         -  if( i<pIdxInfo->nConstraint ){
   464         -    pIdxInfo->aConstraintUsage[i].omit = 1;
   465         -    pIdxInfo->aConstraintUsage[i].argvIndex = 1;
   466         -    pIdxInfo->idxNum = 1;
   467         -    pIdxInfo->estimatedCost = 10.0;
   468         -  }else{
          496  +  if( idx4<0 ){
   469    497       pIdxInfo->idxNum = 0;
   470    498       pIdxInfo->estimatedCost = (double)(((sqlite3_int64)1) << 50);
          499  +  }else{
          500  +    pIdxInfo->aConstraintUsage[idx4].omit = 1;
          501  +    pIdxInfo->aConstraintUsage[idx4].argvIndex = 1;
          502  +    if( idx5>=0 ){
          503  +      pIdxInfo->aConstraintUsage[idx5].omit = 1;
          504  +      pIdxInfo->aConstraintUsage[idx5].argvIndex = 2;
          505  +      pIdxInfo->idxNum = 2;
          506  +      pIdxInfo->estimatedCost = 10.0;
          507  +    }else{
          508  +      pIdxInfo->idxNum = 1;
          509  +      pIdxInfo->estimatedCost = 100.0;
          510  +    }
   471    511     }
   472    512   
   473    513     return SQLITE_OK;
   474    514   }
   475    515   
   476    516   static int fsdirRegister(sqlite3 *db){
   477    517     static sqlite3_module fsdirModule = {

Changes to src/shell.c.in.

  4280   4280       }
  4281   4281     }
  4282   4282     shellFinalize(&rc, pSql);
  4283   4283   
  4284   4284     sqlite3_free(zDir);
  4285   4285     return rc;
  4286   4286   }
         4287  +
  4287   4288   
  4288   4289   /*
  4289   4290   ** Implementation of .ar "Create" command. 
  4290   4291   **
  4291   4292   ** Create the "sqlar" table in the database if it does not already exist.
  4292   4293   ** Then add each file in the azFile[] array to the archive. Directories
  4293   4294   ** are added recursively. If argument bVerbose is non-zero, a message is
................................................................................
  4294   4295   ** printed on stdout for each file archived.
  4295   4296   */
  4296   4297   static int arCreateCommand(
  4297   4298     ShellState *p,                  /* Shell state pointer */
  4298   4299     ArCommand *pAr                  /* Command arguments and options */
  4299   4300   ){
  4300   4301     const char *zSql = 
  4301         -    "WITH f(n, m, t, d) AS ("
  4302         -    "  SELECT name, mode, mtime, data FROM fsentry(:1) UNION ALL "
  4303         -    "  SELECT n || '/' || name, mode, mtime, data "
  4304         -    "      FROM f, fsdir(n) WHERE (m&?)"
  4305         -    ") SELECT * FROM f";
         4302  +    "SELECT name, mode, mtime, data FROM fsdir(?, ?)";
  4306   4303   
  4307   4304     const char *zSqlar = 
  4308   4305         "CREATE TABLE IF NOT EXISTS sqlar("
  4309   4306         "name TEXT PRIMARY KEY,  -- name of the file\n"
  4310   4307         "mode INT,               -- access permissions\n"
  4311   4308         "mtime INT,              -- last modification time\n"
  4312   4309         "sz INT,                 -- original file size\n"
................................................................................
  4318   4315     sqlite3_stmt *pStmt = 0;        /* Directory traverser */
  4319   4316     sqlite3_stmt *pInsert = 0;      /* Compilation of zInsert */
  4320   4317     int i;                          /* For iterating through azFile[] */
  4321   4318     int rc;                         /* Return code */
  4322   4319   
  4323   4320     Bytef *aCompress = 0;           /* Compression buffer */
  4324   4321     int nCompress = 0;              /* Size of compression buffer */
  4325         -  
         4322  +
  4326   4323     rc = sqlite3_exec(p->db, "SAVEPOINT ar;", 0, 0, 0);
  4327   4324     if( rc!=SQLITE_OK ) return rc;
  4328   4325   
  4329   4326     rc = sqlite3_exec(p->db, zSqlar, 0, 0, 0);
  4330   4327     shellPrepare(p, &rc, zInsert, &pInsert);
  4331   4328     shellPrepare(p, &rc, zSql, &pStmt);
         4329  +  sqlite3_bind_text(pStmt, 2, pAr->zDir, -1, SQLITE_STATIC);
  4332   4330   
  4333   4331     for(i=0; i<pAr->nArg && rc==SQLITE_OK; i++){
  4334   4332       sqlite3_bind_text(pStmt, 1, pAr->azArg[i], -1, SQLITE_STATIC);
  4335         -    sqlite3_bind_int(pStmt, 2, S_IFDIR);
  4336   4333       while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
  4337   4334         int sz;
  4338   4335         const char *zName = (const char*)sqlite3_column_text(pStmt, 0);
  4339   4336         int mode = sqlite3_column_int(pStmt, 1);
  4340   4337         unsigned int mtime = sqlite3_column_int(pStmt, 2);
  4341   4338   
  4342   4339         if( pAr->bVerbose ){

Changes to test/shell8.test.

    82     82   
    83     83   do_test 1.2 {
    84     84     file delete -force ar3
    85     85     file mkdir ar3
    86     86     catchcmd test_ar.db ".ar xvC ar3"
    87     87     dir_to_list ar3/ar1
    88     88   } $expected
           89  +
           90  +do_test 1.3 {
           91  +  file delete -force ar3
           92  +  file mkdir ar3
           93  +
           94  +  forcedelete test_ar.db
           95  +  catchcmd test_ar.db ".ar cC ar1 file1 file2 dir1"
           96  +  catchcmd test_ar.db ".ar xC ar3"
           97  +  dir_to_list ar3
           98  +} $expected
    89     99   
    90    100   
    91    101   
    92    102   finish_test