SQLite Archiver

Check-in [4a0ed63dae]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add symlink support to sqlar.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 4a0ed63daef97c4b3380aabc6578e3056fe54bca
User & Date: dan 2017-12-02 19:15:43.286
Context
2017-12-04
16:24
Update README.md to describe how symbolic links are stored in an archive. check-in: ef2844a7e0 user: dan tags: trunk
2017-12-02
19:15
Add symlink support to sqlar. check-in: 4a0ed63dae user: dan tags: trunk
2017-05-12
15:32
Update the built-in SQLite to the first 3.19.0 beta. check-in: 84cb978770 user: drh tags: trunk
Changes
Unified Diff Ignore Whitespace Patch
Changes to sqlar.c.
42
43
44
45
46
47
48
49








50
51
52
53
54
55
56
     "   -x      Extract files from archive\n"
     "   -v      Verbose output\n"
  );
  exit(1);
}

/*
** The database schema:








*/
static const char zSchema[] =
  "CREATE TABLE IF NOT EXISTS sqlar(\n"
  "  name TEXT PRIMARY KEY,\n"
  "  mode INT,\n"
  "  mtime INT,\n"
  "  sz INT,\n"







|
>
>
>
>
>
>
>
>







42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
     "   -x      Extract files from archive\n"
     "   -v      Verbose output\n"
  );
  exit(1);
}

/*
** The database schema. Each file, directory or symlink in an archive
** is represented by a single row in this table. 
**
** name:  Path to file-system entry (text).
** mode:  Value of stat.st_mode returned by stat() call (integer).
** mtime: Value of stat.st_mtime returned by stat() call (integer).
** sz:    Size of file in bytes. Always 0 for a directory. -1 for symlink.
** data:  Blob containing file contents. NULL for a directory. Text value
**        containing linked path for a symlink.
*/
static const char zSchema[] =
  "CREATE TABLE IF NOT EXISTS sqlar(\n"
  "  name TEXT PRIMARY KEY,\n"
  "  mode INT,\n"
  "  mtime INT,\n"
  "  sz INT,\n"
405
406
407
408
409
410
411






412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429

430
431

432
433
434
435
436
437
438
439
440
441
442
443
444























445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474



475
476
477
478
479
480
481
482
  FILE *out;
  make_parent_directory(zFilename);
  if( pCompr==0 ){
    rc = mkdir(zFilename, iMode);
    if( rc ) errorMsg("cannot make directory: %s\n", zFilename);
    return;
  }






  out = fopen(zFilename, "wb");
  if( out==0 ) errorMsg("cannot open for writing: %s\n", zFilename);
  if( sz==nCompr ){
    if( sz>0 && fwrite(pCompr, sz, 1, out)!=1 ){
      errorMsg("failed to write: %s\n", zFilename);
    }
  }else{
    pOut = sqlite3_malloc( sz+1 );
    if( pOut==0 ) errorMsg("cannot allocate %d bytes\n", sz+1);
    nOut = sz;
    rc = uncompress((Bytef*)pOut, &nOut, (const Bytef*)pCompr, nCompr);
    if( rc!=Z_OK ) errorMsg("uncompress failed for %s\n", zFilename);
    if( nOut>0 && fwrite(pOut, nOut, 1, out)!=1 ){
      errorMsg("failed to write: %s\n", zFilename);
    }
    sqlite3_free(pOut);
  }
  fclose(out);

  rc = chmod(zFilename, iMode&0777);
  if( rc ) errorMsg("cannot change mode to %03o: %s\n", iMode, zFilename);

}

/*
** Error out if there are any issues with the given filename
*/
static void check_filename(const char *z){
  if( strncmp(z, "../", 3)==0 || sqlite3_strglob("*/../*", z)==0 ){
    errorMsg("Filename with '..' in its path: %s\n", z);
  }
  if( sqlite3_strglob("*\\*", z)==0 ){
    errorMsg("Filename with '\\' in its name: %s\n", z);
  }
}
























/*
** Add a file to the database.
*/
static void add_file(
  const char *zFilename,     /* Name of file to add */
  int verboseFlag,           /* If true, show each file added */
  int noCompress             /* If true, always omit compression */
){
  int rc;
  struct stat x;
  int szOrig;
  int szCompr;
  const char *zName;

  check_filename(zFilename);
  rc = stat(zFilename, &x);
  if( rc ) errorMsg("no such file or directory: %s\n", zFilename);
  if( x.st_size>1000000000 ){
    errorMsg("file too big: %s\n", zFilename);
  }
  if( pStmt==0 ){
    db_prepare("REPLACE INTO sqlar(name,mode,mtime,sz,data)"
               " VALUES(?1,?2,?3,?4,?5)");
  }
  zName = zFilename;
  while( zName[0]=='/' ) zName++;
  sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
  sqlite3_bind_int(pStmt, 2, x.st_mode);
  sqlite3_bind_int64(pStmt, 3, x.st_mtime);



  if( S_ISREG(x.st_mode) ){
    char *zContent = read_file(zFilename, &szOrig, &szCompr, noCompress);
    sqlite3_bind_int(pStmt, 4, szOrig);
    sqlite3_bind_blob(pStmt, 5, zContent, szCompr, sqlite3_free);
    if( verboseFlag ){
      if( szCompr<szOrig ){
        int pct = szOrig ? (100*(sqlite3_int64)szCompr)/szOrig : 0;
        printf("  added: %s (deflate %d%%)\n", zFilename, 100-pct);







>
>
>
>
>
>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
>
|
|
>













>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
















|













>
>
>
|







413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
  FILE *out;
  make_parent_directory(zFilename);
  if( pCompr==0 ){
    rc = mkdir(zFilename, iMode);
    if( rc ) errorMsg("cannot make directory: %s\n", zFilename);
    return;
  }
  if( sz<0 ){
    /* This is a symlink. */
    if( symlink(pCompr, zFilename)<0 ){
      errorMsg("failed to create symlink: %s -> %s\n", zFilename, pCompr);
    }
  }else{
    out = fopen(zFilename, "wb");
    if( out==0 ) errorMsg("cannot open for writing: %s\n", zFilename);
    if( sz==nCompr ){
      if( sz>0 && fwrite(pCompr, sz, 1, out)!=1 ){
        errorMsg("failed to write: %s\n", zFilename);
      }
    }else{
      pOut = sqlite3_malloc( sz+1 );
      if( pOut==0 ) errorMsg("cannot allocate %d bytes\n", sz+1);
      nOut = sz;
      rc = uncompress((Bytef*)pOut, &nOut, (const Bytef*)pCompr, nCompr);
      if( rc!=Z_OK ) errorMsg("uncompress failed for %s\n", zFilename);
      if( nOut>0 && fwrite(pOut, nOut, 1, out)!=1 ){
        errorMsg("failed to write: %s\n", zFilename);
      }
      sqlite3_free(pOut);
    }
    fclose(out);

    rc = chmod(zFilename, iMode&0777);
    if( rc ) errorMsg("cannot change mode to %03o: %s\n", iMode&0777,zFilename);
  }
}

/*
** Error out if there are any issues with the given filename
*/
static void check_filename(const char *z){
  if( strncmp(z, "../", 3)==0 || sqlite3_strglob("*/../*", z)==0 ){
    errorMsg("Filename with '..' in its path: %s\n", z);
  }
  if( sqlite3_strglob("*\\*", z)==0 ){
    errorMsg("Filename with '\\' in its name: %s\n", z);
  }
}

/*
** File zSymlink is a symbolic link. Figure out the path that the link 
** points to and bind it (as text) to parameter iVar of statement pStmt.
*/
static void bind_symlink_path(
  sqlite3_stmt *pStmt,            /* Statement to bind to */
  int iVar,                       /* Variable within pStmt to bind to */
  const char *zSymlink,           /* Path to symlink */
  int verboseFlag                 /* If true, show file added */
){
  char buf[4096];
  int n = readlink(zSymlink, buf, sizeof(buf));
  if( n<0 ){
    errorMsg("readlink(%s) failed\n", zSymlink);
  }
  if( n>sizeof(buf) ){
    errorMsg("symlinked path is too long (max %d bytes)\n", sizeof(buf));
  }

  sqlite3_bind_text(pStmt, iVar, buf, n, SQLITE_TRANSIENT);
  if( verboseFlag ) printf("  added: %s -> %.*s\n", zSymlink, n, buf);
}

/*
** Add a file to the database.
*/
static void add_file(
  const char *zFilename,     /* Name of file to add */
  int verboseFlag,           /* If true, show each file added */
  int noCompress             /* If true, always omit compression */
){
  int rc;
  struct stat x;
  int szOrig;
  int szCompr;
  const char *zName;

  check_filename(zFilename);
  rc = lstat(zFilename, &x);
  if( rc ) errorMsg("no such file or directory: %s\n", zFilename);
  if( x.st_size>1000000000 ){
    errorMsg("file too big: %s\n", zFilename);
  }
  if( pStmt==0 ){
    db_prepare("REPLACE INTO sqlar(name,mode,mtime,sz,data)"
               " VALUES(?1,?2,?3,?4,?5)");
  }
  zName = zFilename;
  while( zName[0]=='/' ) zName++;
  sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
  sqlite3_bind_int(pStmt, 2, x.st_mode);
  sqlite3_bind_int64(pStmt, 3, x.st_mtime);
  if( S_ISLNK(x.st_mode) ){
    sqlite3_bind_int(pStmt, 4, -1);
    bind_symlink_path(pStmt, 5, zFilename, verboseFlag);
  }else if( S_ISREG(x.st_mode) ){
    char *zContent = read_file(zFilename, &szOrig, &szCompr, noCompress);
    sqlite3_bind_int(pStmt, 4, szOrig);
    sqlite3_bind_blob(pStmt, 5, zContent, szCompr, sqlite3_free);
    if( verboseFlag ){
      if( szCompr<szOrig ){
        int pct = szOrig ? (100*(sqlite3_int64)szCompr)/szOrig : 0;
        printf("  added: %s (deflate %d%%)\n", zFilename, 100-pct);
586
587
588
589
590
591
592


593
594
595
596
597
598
599
600
601


602

603








604
605

606
607
608
609
610
611
612
  }else if( extractFlag ){
    const char *zSql;
    db_open(zArchive, 0, seeFlag, azFiles, nFiles);
    zSql = "SELECT name, mode, mtime, sz, data FROM sqlar"
           " WHERE name_on_list(name)";
    db_prepare(zSql);
    while( sqlite3_step(pStmt)==SQLITE_ROW ){


      const char *zFN = (const char*)sqlite3_column_text(pStmt, 0);
      check_filename(zFN);
      if( zFN[0]=='/' ){
        errorMsg("absolute pathname: %s\n", zFN);
      }
      if( sqlite3_column_type(pStmt,4)==SQLITE_BLOB && access(zFN, F_OK)==0 ){
        errorMsg("file already exists: %s\n", zFN);
      }
      if( verboseFlag ) printf("%s\n", zFN);


      write_file(zFN, sqlite3_column_int(pStmt,1),

                 sqlite3_column_int64(pStmt,2),








                 sqlite3_column_int(pStmt,3),
                 sqlite3_column_blob(pStmt,4),

                 sqlite3_column_bytes(pStmt,4));
    }
    db_close(1);
  }else{
    if( azFiles==0 ){
      errorMsg("Specify one or more files to add on the command-line");
    }







>
>









>
>
|
>
|
>
>
>
>
>
>
>
>
|
|
>







628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
  }else if( extractFlag ){
    const char *zSql;
    db_open(zArchive, 0, seeFlag, azFiles, nFiles);
    zSql = "SELECT name, mode, mtime, sz, data FROM sqlar"
           " WHERE name_on_list(name)";
    db_prepare(zSql);
    while( sqlite3_step(pStmt)==SQLITE_ROW ){
      int sz = sqlite3_column_int(pStmt, 3);     /* Size of file on disk */
      const char *pData;                         /* Data for this file */
      const char *zFN = (const char*)sqlite3_column_text(pStmt, 0);
      check_filename(zFN);
      if( zFN[0]=='/' ){
        errorMsg("absolute pathname: %s\n", zFN);
      }
      if( sqlite3_column_type(pStmt,4)==SQLITE_BLOB && access(zFN, F_OK)==0 ){
        errorMsg("file already exists: %s\n", zFN);
      }
      if( verboseFlag ) printf("%s\n", zFN);
      if( sz<0 ){
        /* symlink */
        pData = (const char*)sqlite3_column_text(pStmt, 4);
      }else{
        pData = (const char*)sqlite3_column_blob(pStmt,4);
        if( pData==0 && sqlite3_column_type(pStmt, 4)!=SQLITE_NULL ){
          /* If the file is zero bytes in size, then column "data" will
          ** contain a zero-byte blob value. In this case, column_blob()
          ** returns NULL, which confuses the write_file() call into
          ** thinking this is a directory, not a zero-byte file.  */
          pData = "";
        }
      }
      write_file(zFN, sqlite3_column_int(pStmt,1),
                 sqlite3_column_int64(pStmt,2),
                 sz, pData,
                 sqlite3_column_bytes(pStmt,4));
    }
    db_close(1);
  }else{
    if( azFiles==0 ){
      errorMsg("Specify one or more files to add on the command-line");
    }
Changes to sqlarfs.c.
132
133
134
135
136
137
138









139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176


177
178
179
180
181
182
183
  sqlite3_bind_text(g.pExists, 1, &path[1], -1, SQLITE_STATIC);
  rc = sqlite3_step(g.pExists);
  sqlite3_reset(g.pExists);
  if( rc==SQLITE_DONE ) return -ENOENT;
  return 0;
}











/*
** Load the file named path[] into the cache, if it is not there already.
**
** Return 0 on success.  Return an error code if the file could not be loaded.
*/
static int loadCache(const char *path){
  unsigned long int nIn;
  const char *zIn;
  int rc;
  if( g.zCacheName ){
    if( strcmp(path, g.zCacheName)==0 ) return 0;
    sqlite3_free(g.zCacheName); g.zCacheName = 0;
    sqlite3_free(g.zCacheData); g.zCacheData = 0;
  }
  if( g.pRead==0 ){
    rc = sqlite3_prepare_v2(g.db,
               "SELECT sz, data FROM sqlar WHERE name=?1",
               -1, &g.pRead, 0);
    if( rc!=SQLITE_OK ){
      return -EIO;
    }
  }
  sqlite3_bind_text(g.pRead, 1, path, -1, SQLITE_STATIC);
  if( sqlite3_step(g.pRead)==SQLITE_ROW ){
    g.szCache = sqlite3_column_int64(g.pRead, 0);
    zIn = (const char*)sqlite3_column_blob(g.pRead, 1);
    nIn = (unsigned long int)sqlite3_column_bytes(g.pRead, 1);
    g.zCacheData = sqlite3_malloc( g.szCache );
    if( g.zCacheData==0 ){
      rc = -EIO;
    }else{
      rc = uncompress((Bytef*)g.zCacheData, &g.szCache, (const Bytef*)zIn, nIn);
      if( rc!=Z_OK ){
        sqlite3_free(g.zCacheData);
        g.zCacheData = 0;
        rc = -EIO;
      }


    }
    if( g.zCacheData ){
      g.zCacheName = sqlite3_mprintf("%s", path);
      if( g.zCacheName==0 ){
        rc = -EIO;
        sqlite3_free(g.zCacheData);
        g.zCacheData = 0;







>
>
>
>
>
>
>
>
>









|





<
|
<
<
<
|
<






|


|






>
>







132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162

163



164

165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
  sqlite3_bind_text(g.pExists, 1, &path[1], -1, SQLITE_STATIC);
  rc = sqlite3_step(g.pExists);
  sqlite3_reset(g.pExists);
  if( rc==SQLITE_DONE ) return -ENOENT;
  return 0;
}

static int prepare_read_stmt(void){
  int rc = SQLITE_OK;
  if( g.pRead==0 ){
    rc = sqlite3_prepare_v2(g.db,
               "SELECT sz, data FROM sqlar WHERE name=?1",
               -1, &g.pRead, 0);
  }
  return rc;
}

/*
** Load the file named path[] into the cache, if it is not there already.
**
** Return 0 on success.  Return an error code if the file could not be loaded.
*/
static int loadCache(const char *path){
  unsigned long int nIn;
  const char *zIn;
  int rc = 0;
  if( g.zCacheName ){
    if( strcmp(path, g.zCacheName)==0 ) return 0;
    sqlite3_free(g.zCacheName); g.zCacheName = 0;
    sqlite3_free(g.zCacheData); g.zCacheData = 0;
  }

  if( prepare_read_stmt() ){



    return -EIO;

  }
  sqlite3_bind_text(g.pRead, 1, path, -1, SQLITE_STATIC);
  if( sqlite3_step(g.pRead)==SQLITE_ROW ){
    g.szCache = sqlite3_column_int64(g.pRead, 0);
    zIn = (const char*)sqlite3_column_blob(g.pRead, 1);
    nIn = (unsigned long int)sqlite3_column_bytes(g.pRead, 1);
    g.zCacheData = sqlite3_malloc( g.szCache+1 );
    if( g.zCacheData==0 ){
      rc = -EIO;
    }else if( g.szCache!=nIn ){
      rc = uncompress((Bytef*)g.zCacheData, &g.szCache, (const Bytef*)zIn, nIn);
      if( rc!=Z_OK ){
        sqlite3_free(g.zCacheData);
        g.zCacheData = 0;
        rc = -EIO;
      }
    }else{
      memcpy(g.zCacheData, zIn, nIn);
    }
    if( g.zCacheData ){
      g.zCacheName = sqlite3_mprintf("%s", path);
      if( g.zCacheName==0 ){
        rc = -EIO;
        sqlite3_free(g.zCacheData);
        g.zCacheData = 0;
208
209
210
211
212
213
214
215






















216
217
218
219
220

221
222
223
224
225
226
227
    if( offset+size>g.szCache ) size = g.szCache - offset;
    memcpy(buf, g.zCacheData + offset, size);
    return size;
  }else{
    return rc;
  }
}  























static struct fuse_operations sqlarfs_methods = {
  .getattr = sqlarfs_getattr,
  .readdir = sqlarfs_readdir,
  .open   	= sqlarfs_open,
  .read    = sqlarfs_read,

};

/*
** Show a help message and quit.
*/
static void showHelp(const char *argv0){
  fprintf(stderr, "Usage: %s [options] archive mount-point\n", argv0);








>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>

|
|
|
|
>







214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
    if( offset+size>g.szCache ) size = g.szCache - offset;
    memcpy(buf, g.zCacheData + offset, size);
    return size;
  }else{
    return rc;
  }
}  

static int sqlarfs_readlink(const char *zSymlink, char *pBuf, size_t nBuf){
  int rc = 0;
  if( prepare_read_stmt() ){
    rc = -EIO;
  }else{
    sqlite3_bind_text(g.pRead, 1, &zSymlink[1], -1, SQLITE_STATIC);
    if( sqlite3_step(g.pRead)==SQLITE_ROW ){
      int n = sqlite3_column_bytes(g.pRead, 1);
      if( n<nBuf ){
        memcpy(pBuf, sqlite3_column_text(g.pRead, 1), n);
        pBuf[n] = '\0';
      }else{
        rc = -ENAMETOOLONG;
      }
    }else{
      rc = -ENOENT;
    }
    sqlite3_reset(g.pRead);
  }
  return rc;
}

static struct fuse_operations sqlarfs_methods = {
  .getattr  = sqlarfs_getattr,
  .readdir  = sqlarfs_readdir,
  .open     = sqlarfs_open,
  .read     = sqlarfs_read,
  .readlink = sqlarfs_readlink,
};

/*
** Show a help message and quit.
*/
static void showHelp(const char *argv0){
  fprintf(stderr, "Usage: %s [options] archive mount-point\n", argv0);