/ Check-in [7652b3c2]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Do not use the compress() and uncompress() functions in ext/misc/compress.c - they are not quite compatible with the spec. Instead use new functions in ext/misc/sqlar.c.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | sqlar-shell-support
Files: files | file ages | folders
SHA3-256: 7652b3c2374084047b6c1da3e525e0cac34fe220597f81e793bc4fd9f33358da
User & Date: dan 2017-12-16 19:11:26
Context
2017-12-23
18:34
Merge enhancements from trunk. check-in: 150f07fe user: drh tags: sqlar-shell-support
2017-12-16
19:11
Do not use the compress() and uncompress() functions in ext/misc/compress.c - they are not quite compatible with the spec. Instead use new functions in ext/misc/sqlar.c. check-in: 7652b3c2 user: dan tags: sqlar-shell-support
2017-12-14
19:15
Have the writefile() function optionally set the modification-time of the files it writes or creates. And many small fixes to the new code on this branch. check-in: 7b51269c user: dan tags: sqlar-shell-support
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Added ext/misc/sqlar.c.

            1  +/*
            2  +** 2017-12-17
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +******************************************************************************
           12  +**
           13  +** Utility functions sqlar_compress() and sqlar_uncompress(). Useful
           14  +** for working with sqlar archives and used by the shell tool's built-in
           15  +** sqlar support.
           16  +*/
           17  +#include "sqlite3ext.h"
           18  +SQLITE_EXTENSION_INIT1
           19  +#include <zlib.h>
           20  +
           21  +/*
           22  +** Implementation of the "sqlar_compress(X)" SQL function.
           23  +**
           24  +** If the type of X is SQLITE_BLOB, and compressing that blob using
           25  +** zlib utility function compress() yields a smaller blob, return the
           26  +** compressed blob. Otherwise, return a copy of X.
           27  +*/
           28  +static void sqlarCompressFunc(
           29  +  sqlite3_context *context,
           30  +  int argc,
           31  +  sqlite3_value **argv
           32  +){
           33  +  assert( argc==1 );
           34  +  if( sqlite3_value_type(argv[0])==SQLITE_BLOB ){
           35  +    const Bytef *pData = sqlite3_value_blob(argv[0]);
           36  +    uLong nData = sqlite3_value_bytes(argv[0]);
           37  +    uLongf nOut = compressBound(nData);
           38  +    Bytef *pOut;
           39  +
           40  +    pOut = (Bytef*)sqlite3_malloc(nOut);
           41  +    if( pOut==0 ){
           42  +      sqlite3_result_error_nomem(context);
           43  +      return;
           44  +    }else{
           45  +      if( Z_OK!=compress(pOut, &nOut, pData, nData) ){
           46  +        sqlite3_result_error(context, "error in compress()", -1);
           47  +      }else if( nOut<nData ){
           48  +        sqlite3_result_blob(context, pOut, nOut, SQLITE_TRANSIENT);
           49  +      }else{
           50  +        sqlite3_result_value(context, argv[0]);
           51  +      }
           52  +      sqlite3_free(pOut);
           53  +    }
           54  +  }else{
           55  +    sqlite3_result_value(context, argv[0]);
           56  +  }
           57  +}
           58  +
           59  +/*
           60  +** Implementation of the "sqlar_uncompress(X,SZ)" SQL function
           61  +**
           62  +** Parameter SZ is interpreted as an integer. If it is less than or
           63  +** equal to zero, then this function returns a copy of X. Or, if
           64  +** SZ is equal to the size of X when interpreted as a blob, also
           65  +** return a copy of X. Otherwise, decompress blob X using zlib
           66  +** utility function uncompress() and return the results (another
           67  +** blob).
           68  +*/
           69  +static void sqlarUncompressFunc(
           70  +  sqlite3_context *context,
           71  +  int argc,
           72  +  sqlite3_value **argv
           73  +){
           74  +  uLong nData;
           75  +  uLongf sz;
           76  +
           77  +  assert( argc==2 );
           78  +  sz = sqlite3_value_int(argv[1]);
           79  +
           80  +  if( sz<=0 || sz==(nData = sqlite3_value_bytes(argv[0])) ){
           81  +    sqlite3_result_value(context, argv[0]);
           82  +  }else{
           83  +    const Bytef *pData= sqlite3_value_blob(argv[0]);
           84  +    Bytef *pOut = sqlite3_malloc(sz);
           85  +    if( Z_OK!=uncompress(pOut, &sz, pData, nData) ){
           86  +      sqlite3_result_error(context, "error in uncompress()", -1);
           87  +    }else{
           88  +      sqlite3_result_blob(context, pOut, sz, SQLITE_TRANSIENT);
           89  +    }
           90  +    sqlite3_free(pOut);
           91  +  }
           92  +}
           93  +
           94  +
           95  +#ifdef _WIN32
           96  +__declspec(dllexport)
           97  +#endif
           98  +int sqlite3_sqlar_init(
           99  +  sqlite3 *db, 
          100  +  char **pzErrMsg, 
          101  +  const sqlite3_api_routines *pApi
          102  +){
          103  +  int rc = SQLITE_OK;
          104  +  SQLITE_EXTENSION_INIT2(pApi);
          105  +  (void)pzErrMsg;  /* Unused parameter */
          106  +  rc = sqlite3_create_function(db, "sqlar_compress", 1, SQLITE_UTF8, 0,
          107  +                               sqlarCompressFunc, 0, 0);
          108  +  if( rc==SQLITE_OK ){
          109  +    rc = sqlite3_create_function(db, "sqlar_uncompress", 2, SQLITE_UTF8, 0,
          110  +                                 sqlarUncompressFunc, 0, 0);
          111  +  }
          112  +  return rc;
          113  +}

Changes to main.mk.

   686    686   	./mkkeywordhash >keywordhash.h
   687    687   
   688    688   # Source files that go into making shell.c
   689    689   SHELL_SRC = \
   690    690   	$(TOP)/src/shell.c.in \
   691    691   	$(TOP)/ext/misc/shathree.c \
   692    692   	$(TOP)/ext/misc/fileio.c \
   693         -	$(TOP)/ext/misc/completion.c
          693  +	$(TOP)/ext/misc/completion.c \
          694  +	$(TOP)/ext/misc/sqlar.c
   694    695   
   695    696   shell.c:	$(SHELL_SRC) $(TOP)/tool/mkshellc.tcl
   696    697   	tclsh $(TOP)/tool/mkshellc.tcl >shell.c
   697    698   
   698    699   
   699    700   
   700    701   # Rules to build the extension objects.

Changes to src/shell.c.in.

   793    793   #define SQLITE_EXTENSION_INIT1
   794    794   #define SQLITE_EXTENSION_INIT2(X) (void)(X)
   795    795   
   796    796   INCLUDE ../ext/misc/shathree.c
   797    797   INCLUDE ../ext/misc/fileio.c
   798    798   INCLUDE ../ext/misc/completion.c
   799    799   #ifdef SQLITE_HAVE_ZLIB
   800         -INCLUDE ../ext/misc/compress.c
          800  +INCLUDE ../ext/misc/sqlar.c
   801    801   #endif
   802    802   
   803    803   #if defined(SQLITE_ENABLE_SESSION)
   804    804   /*
   805    805   ** State information for a single open session
   806    806   */
   807    807   typedef struct OpenSession OpenSession;
................................................................................
  2897   2897   #ifndef SQLITE_OMIT_LOAD_EXTENSION
  2898   2898       sqlite3_enable_load_extension(p->db, 1);
  2899   2899   #endif
  2900   2900       sqlite3_fileio_init(p->db, 0, 0);
  2901   2901       sqlite3_shathree_init(p->db, 0, 0);
  2902   2902       sqlite3_completion_init(p->db, 0, 0);
  2903   2903   #ifdef SQLITE_HAVE_ZLIB
  2904         -    sqlite3_compress_init(p->db, 0, 0);
         2904  +    sqlite3_sqlar_init(p->db, 0, 0);
  2905   2905   #endif
  2906   2906       sqlite3_create_function(p->db, "shell_add_schema", 2, SQLITE_UTF8, 0,
  2907   2907                               shellAddSchemaName, 0, 0);
  2908   2908     }
  2909   2909   }
  2910   2910   
  2911   2911   #if HAVE_READLINE || HAVE_EDITLINE
................................................................................
  4478   4478   
  4479   4479   
  4480   4480   /*
  4481   4481   ** Implementation of .ar "eXtract" command. 
  4482   4482   */
  4483   4483   static int arExtractCommand(ShellState *p, sqlite3 *db, ArCommand *pAr){
  4484   4484     const char *zSql1 = 
  4485         -    "SELECT :1 || name, writefile(:1 || name, "
  4486         -    "CASE WHEN (data AND sz>=0 AND sz!=length(data)) THEN "
  4487         -    "    uncompress(data) "
  4488         -    "ELSE"
  4489         -    "    data "
  4490         -    "END, "
  4491         -    "mode, mtime) FROM sqlar WHERE (%s) AND (data IS NULL OR :2 = 0)";
         4485  +    "SELECT "
         4486  +    "  :1 || name, "
         4487  +    "  writefile(:1 || name, sqlar_uncompress(data, sz), mode, mtime) "
         4488  +    "FROM sqlar WHERE (%s) AND (data IS NULL OR :2 = 0)";
  4492   4489   
  4493   4490     struct timespec times[2];
  4494   4491     sqlite3_stmt *pSql = 0;
  4495   4492     int rc = SQLITE_OK;
  4496   4493     char *zDir = 0;
  4497   4494     char *zWhere = 0;
  4498   4495     int i;
................................................................................
  4565   4562         "name TEXT PRIMARY KEY,  -- name of the file\n"
  4566   4563         "mode INT,               -- access permissions\n"
  4567   4564         "mtime INT,              -- last modification time\n"
  4568   4565         "sz INT,                 -- original file size\n"
  4569   4566         "data BLOB               -- compressed content\n"
  4570   4567     ")";
  4571   4568     const char *zDrop = "DROP TABLE IF EXISTS sqlar";
  4572         -  const char *zInsert = "REPLACE INTO sqlar VALUES(?, ?, ?, ?, ?)";
         4569  +  const char *zInsert = "REPLACE INTO sqlar VALUES(?,?,?,?,sqlar_compress(?))";
  4573   4570   
  4574   4571     sqlite3_stmt *pStmt = 0;        /* Directory traverser */
  4575   4572     sqlite3_stmt *pInsert = 0;      /* Compilation of zInsert */
  4576   4573     int i;                          /* For iterating through azFile[] */
  4577   4574     int rc;                         /* Return code */
  4578   4575   
  4579         -  Bytef *aCompress = 0;           /* Compression buffer */
  4580         -  int nCompress = 0;              /* Size of compression buffer */
  4581         -
  4582   4576     rc = sqlite3_exec(db, "SAVEPOINT ar;", 0, 0, 0);
  4583   4577     if( rc!=SQLITE_OK ) return rc;
  4584   4578   
  4585   4579     if( bUpdate==0 ){
  4586   4580       rc = sqlite3_exec(db, zDrop, 0, 0, 0);
  4587   4581       if( rc!=SQLITE_OK ) return rc;
  4588   4582     }
................................................................................
  4607   4601         sqlite3_bind_text(pInsert, 1, zName, -1, SQLITE_STATIC);
  4608   4602         sqlite3_bind_int(pInsert, 2, mode);
  4609   4603         sqlite3_bind_int64(pInsert, 3, (sqlite3_int64)mtime);
  4610   4604   
  4611   4605         if( S_ISDIR(mode) ){
  4612   4606           sz = 0;
  4613   4607           sqlite3_bind_null(pInsert, 5);
  4614         -      }else if( S_ISLNK(mode) ){
  4615         -        sz = -1;
         4608  +      }else{
  4616   4609           sqlite3_bind_value(pInsert, 5, sqlite3_column_value(pStmt, 3));
  4617         -      }else{
  4618         -        uLongf nReq;              /* Required size of compression buffer */
  4619         -        const Bytef *pData = (const Bytef*)sqlite3_column_blob(pStmt, 3);
  4620         -        sz = sqlite3_column_bytes(pStmt, 3);
  4621         -        nReq = compressBound(sz);
  4622         -        if( aCompress==0 || nReq>nCompress ){
  4623         -          Bytef *aNew = sqlite3_realloc(aCompress, nReq);
  4624         -          if( aNew==0 ){
  4625         -            rc = SQLITE_NOMEM;
  4626         -          }else{
  4627         -            aCompress = aNew;
  4628         -            nCompress = nReq;
  4629         -          }
  4630         -        }
  4631         -
  4632         -        if( Z_OK!=compress(aCompress, &nReq, pData, sz) ){
  4633         -          rc = SQLITE_ERROR;
  4634         -        }
  4635         -        if( nReq<sz ){
  4636         -          sqlite3_bind_blob(pInsert, 5, aCompress, nReq, SQLITE_STATIC);
         4610  +        if( S_ISLNK(mode) ){
         4611  +          sz = -1;
  4637   4612           }else{
  4638         -          sqlite3_bind_blob(pInsert, 5, pData, sz, SQLITE_STATIC);
         4613  +          sz = sqlite3_column_bytes(pStmt, 3);
  4639   4614           }
  4640   4615         }
  4641   4616   
  4642         -      if( rc==SQLITE_OK ){
  4643         -        sqlite3_bind_int(pInsert, 4, sz);
  4644         -        sqlite3_step(pInsert);
  4645         -        rc = sqlite3_reset(pInsert);
  4646         -      }
         4617  +      sqlite3_bind_int(pInsert, 4, sz);
         4618  +      sqlite3_step(pInsert);
         4619  +      rc = sqlite3_reset(pInsert);
  4647   4620       }
  4648   4621       shellReset(&rc, pStmt);
  4649   4622     }
  4650   4623   
  4651   4624     if( rc!=SQLITE_OK ){
  4652   4625       sqlite3_exec(db, "ROLLBACK TO ar; RELEASE ar;", 0, 0, 0);
  4653   4626     }else{
  4654   4627       rc = sqlite3_exec(db, "RELEASE ar;", 0, 0, 0);
  4655   4628     }
  4656   4629     shellFinalize(&rc, pStmt);
  4657   4630     shellFinalize(&rc, pInsert);
  4658         -  sqlite3_free(aCompress);
  4659   4631     return rc;
  4660   4632   }
  4661   4633   
  4662   4634   /*
  4663   4635   ** Implementation of .ar "Create" command. 
  4664   4636   **
  4665   4637   ** Create the "sqlar" table in the database if it does not already exist.
................................................................................
  4709   4681           raw_printf(stderr, "cannot open file: %s (%s)\n", 
  4710   4682               cmd.zFile, sqlite3_errmsg(db)
  4711   4683           );
  4712   4684           sqlite3_close(db);
  4713   4685           return rc;
  4714   4686         }
  4715   4687         sqlite3_fileio_init(db, 0, 0);
  4716         -      sqlite3_compress_init(db, 0, 0);
         4688  +      sqlite3_sqlar_init(db, 0, 0);
  4717   4689       }else{
  4718   4690         db = pState->db;
  4719   4691       }
  4720   4692   
  4721   4693       switch( cmd.eCmd ){
  4722   4694         case AR_CMD_CREATE:
  4723   4695           rc = arCreateCommand(pState, db, &cmd);