SQLite

Check-in [f69e8194bf]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Rearrange things a bit so that writing to a zipfile does not invert the order of objects it contains.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | sqlar-shell-support
Files: files | file ages | folders
SHA3-256: f69e8194bfa7de436c96028730ebd57f186d2e6207792e172e1aa38c7f4211c9
User & Date: dan 2017-12-30 14:26:29.195
Context
2017-12-30
18:32
Have zipfile support DELETE commands. (check-in: 01d4e866fb user: dan tags: sqlar-shell-support)
14:26
Rearrange things a bit so that writing to a zipfile does not invert the order of objects it contains. (check-in: f69e8194bf user: dan tags: sqlar-shell-support)
2017-12-29
20:19
Update ext/misc/zipfile.c to support creating and adding entries to existing zip archives. (check-in: 2dec2dec59 user: dan tags: sqlar-shell-support)
Changes
Unified Diff Ignore Whitespace Patch
Changes to ext/misc/zipfile.c.
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231

232
233
234
235
236
237
238
239
240
241
242
243
244

245
246
247
248
249
250
251
  int bEof;                  /* True when at EOF */
  i64 iNextOff;              /* Offset of next record in central directory */
  ZipfileEOCD eocd;          /* Parse of central directory record */
  ZipfileCDS cds;            /* Central Directory Structure */
  ZipfileLFH lfh;            /* Local File Header for current entry */
  i64 iDataOff;              /* Offset in zipfile to data */
  u32 mTime;                 /* Extended mtime value */
  int flags;
};

/*
** Values for ZipfileCsr.flags.
*/
#define ZIPFILE_MTIME_VALID 0x0001

typedef struct ZipfileEntry ZipfileEntry;
struct ZipfileEntry {
  char *zPath;               /* Path of zipfile entry */

  u8 *aCdsEntry;             /* Buffer containing entire CDS entry */
  int nCdsEntry;             /* Size of buffer aCdsEntry[] in bytes */
  ZipfileEntry *pNext;
};

typedef struct ZipfileTab ZipfileTab;
struct ZipfileTab {
  sqlite3_vtab base;         /* Base class - must be first */
  char *zFile;               /* Zip file this table accesses (may be NULL) */
  u8 *aBuffer;               /* Temporary buffer used for various tasks */

  /* The following are used by write transactions only */
  ZipfileEntry *pEntry;      /* Linked list of all files (if pWriteFd!=0) */

  FILE *pWriteFd;            /* File handle open on zip archive */
  i64 szCurrent;             /* Current size of zip archive */
  i64 szOrig;                /* Size of archive at start of transaction */
};

static void zipfileDequote(char *zIn){
  char q = zIn[0];







|










>


|









|
>







214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
  int bEof;                  /* True when at EOF */
  i64 iNextOff;              /* Offset of next record in central directory */
  ZipfileEOCD eocd;          /* Parse of central directory record */
  ZipfileCDS cds;            /* Central Directory Structure */
  ZipfileLFH lfh;            /* Local File Header for current entry */
  i64 iDataOff;              /* Offset in zipfile to data */
  u32 mTime;                 /* Extended mtime value */
  int flags;                 /* Flags byte (see below for bits) */
};

/*
** Values for ZipfileCsr.flags.
*/
#define ZIPFILE_MTIME_VALID 0x0001

typedef struct ZipfileEntry ZipfileEntry;
struct ZipfileEntry {
  char *zPath;               /* Path of zipfile entry */
  i64 iRowid;                /* Rowid for this value if queried */
  u8 *aCdsEntry;             /* Buffer containing entire CDS entry */
  int nCdsEntry;             /* Size of buffer aCdsEntry[] in bytes */
  ZipfileEntry *pNext;       /* Next element in in-memory CDS */
};

typedef struct ZipfileTab ZipfileTab;
struct ZipfileTab {
  sqlite3_vtab base;         /* Base class - must be first */
  char *zFile;               /* Zip file this table accesses (may be NULL) */
  u8 *aBuffer;               /* Temporary buffer used for various tasks */

  /* The following are used by write transactions only */
  ZipfileEntry *pFirstEntry; /* Linked list of all files (if pWriteFd!=0) */
  ZipfileEntry *pLastEntry;  /* Last element in pFirstEntry list */
  FILE *pWriteFd;            /* File handle open on zip archive */
  i64 szCurrent;             /* Current size of zip archive */
  i64 szOrig;                /* Size of archive at start of transaction */
};

static void zipfileDequote(char *zIn){
  char q = zIn[0];
825
826
827
828
829
830
831
















832
833
834
835
836
837
838
  }else{
    pIdxInfo->estimatedCost = (double)(((sqlite3_int64)1) << 50);
    pIdxInfo->idxNum = 0;
  }

  return SQLITE_OK;
}

















static int zipfileLoadDirectory(ZipfileTab *pTab){
  ZipfileEOCD eocd;
  int rc;

  rc = zipfileReadEOCD(pTab, pTab->pWriteFd, &eocd);
  if( rc==SQLITE_OK ){







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
  }else{
    pIdxInfo->estimatedCost = (double)(((sqlite3_int64)1) << 50);
    pIdxInfo->idxNum = 0;
  }

  return SQLITE_OK;
}

/*
** Add object pNew to the end of the linked list that begins at
** ZipfileTab.pFirstEntry and ends with pLastEntry.
*/
static void zipfileAddEntry(ZipfileTab *pTab, ZipfileEntry *pNew){
  assert( (pTab->pFirstEntry==0)==(pTab->pLastEntry==0) );
  assert( pNew->pNext==0 );
  if( pTab->pFirstEntry==0 ){
    pTab->pFirstEntry = pTab->pLastEntry = pNew;
  }else{
    assert( pTab->pLastEntry->pNext==0 );
    pTab->pLastEntry->pNext = pNew;
    pTab->pLastEntry = pNew;
  }
}

static int zipfileLoadDirectory(ZipfileTab *pTab){
  ZipfileEOCD eocd;
  int rc;

  rc = zipfileReadEOCD(pTab, pTab->pWriteFd, &eocd);
  if( rc==SQLITE_OK ){
867
868
869
870
871
872
873

874
875
876
877
878
879
880
881
882
883
884
        rc = SQLITE_NOMEM;
      }else{
        pNew->zPath = (char*)&pNew[1];
        memcpy(pNew->zPath, &aBuf[ZIPFILE_CDS_FIXED_SZ], nFile);
        pNew->zPath[nFile] = '\0';
        pNew->aCdsEntry = (u8*)&pNew->zPath[nFile+1];
        pNew->nCdsEntry = ZIPFILE_CDS_FIXED_SZ+nFile+nExtra+nComment;

        memcpy(pNew->aCdsEntry, aRec, pNew->nCdsEntry);

        pNew->pNext = pTab->pEntry;
        pTab->pEntry = pNew;
      }

      iOff += ZIPFILE_CDS_FIXED_SZ+nFile+nExtra+nComment;
    }

    sqlite3_free(aBuf);
  }else if( rc==SQLITE_EMPTY ){







>

|
<
<







885
886
887
888
889
890
891
892
893
894


895
896
897
898
899
900
901
        rc = SQLITE_NOMEM;
      }else{
        pNew->zPath = (char*)&pNew[1];
        memcpy(pNew->zPath, &aBuf[ZIPFILE_CDS_FIXED_SZ], nFile);
        pNew->zPath[nFile] = '\0';
        pNew->aCdsEntry = (u8*)&pNew->zPath[nFile+1];
        pNew->nCdsEntry = ZIPFILE_CDS_FIXED_SZ+nFile+nExtra+nComment;
        pNew->pNext = 0;
        memcpy(pNew->aCdsEntry, aRec, pNew->nCdsEntry);
        zipfileAddEntry(pTab, pNew);


      }

      iOff += ZIPFILE_CDS_FIXED_SZ+nFile+nExtra+nComment;
    }

    sqlite3_free(aBuf);
  }else if( rc==SQLITE_EMPTY ){
904
905
906
907
908
909
910

911
912
913
914
915
916
917
    ZIPFILE_CDS_FIXED_SZ + nPath + pCds->nExtra
  );

  if( pNew ){
    pNew->zPath = (char*)&pNew[1];
    pNew->aCdsEntry = (u8*)&pNew->zPath[nPath+1];
    pNew->nCdsEntry = ZIPFILE_CDS_FIXED_SZ + nPath + pCds->nExtra;

    memcpy(pNew->zPath, zPath, nPath+1);

    aWrite = pNew->aCdsEntry;
    zipfileWrite32(aWrite, ZIPFILE_SIGNATURE_CDS);
    zipfileWrite16(aWrite, pCds->iVersionMadeBy);
    zipfileWrite16(aWrite, pCds->iVersionExtract);
    zipfileWrite16(aWrite, pCds->flags);







>







921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
    ZIPFILE_CDS_FIXED_SZ + nPath + pCds->nExtra
  );

  if( pNew ){
    pNew->zPath = (char*)&pNew[1];
    pNew->aCdsEntry = (u8*)&pNew->zPath[nPath+1];
    pNew->nCdsEntry = ZIPFILE_CDS_FIXED_SZ + nPath + pCds->nExtra;
    pNew->pNext = 0;
    memcpy(pNew->zPath, zPath, nPath+1);

    aWrite = pNew->aCdsEntry;
    zipfileWrite32(aWrite, ZIPFILE_SIGNATURE_CDS);
    zipfileWrite16(aWrite, pCds->iVersionMadeBy);
    zipfileWrite16(aWrite, pCds->iVersionExtract);
    zipfileWrite16(aWrite, pCds->flags);
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
    cds.szUncompressed = sz;
    cds.iExternalAttr = (mode<<16);
    cds.iOffset = pTab->szCurrent;
    pNew = zipfileNewEntry(&cds, zPath, nPath, (u32)mTime);
    if( pNew==0 ){
      rc = SQLITE_NOMEM;
    }else{
      pNew->pNext = pTab->pEntry;
      pTab->pEntry = pNew;
    }
  }

  /* Append the new header+file to the archive */
  if( rc==SQLITE_OK ){
    rc = zipfileAppendEntry(pTab, &cds, zPath, nPath, pData, nData, (u32)mTime);
  }







<
|







1149
1150
1151
1152
1153
1154
1155

1156
1157
1158
1159
1160
1161
1162
1163
    cds.szUncompressed = sz;
    cds.iExternalAttr = (mode<<16);
    cds.iOffset = pTab->szCurrent;
    pNew = zipfileNewEntry(&cds, zPath, nPath, (u32)mTime);
    if( pNew==0 ){
      rc = SQLITE_NOMEM;
    }else{

      zipfileAddEntry(pTab, pNew);
    }
  }

  /* Append the new header+file to the archive */
  if( rc==SQLITE_OK ){
    rc = zipfileAppendEntry(pTab, &cds, zPath, nPath, pData, nData, (u32)mTime);
  }
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176

1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
  return zipfileAppendData(pTab, pTab->aBuffer, aBuf - pTab->aBuffer);
}

static void zipfileCleanupTransaction(ZipfileTab *pTab){
  ZipfileEntry *pEntry;
  ZipfileEntry *pNext;

  for(pEntry=pTab->pEntry; pEntry; pEntry=pNext){
    pNext = pEntry->pNext;
    sqlite3_free(pEntry);
  }
  pTab->pEntry = 0;

  fclose(pTab->pWriteFd);
  pTab->pWriteFd = 0;
  pTab->szCurrent = 0;
  pTab->szOrig = 0;
}

static int zipfileBegin(sqlite3_vtab *pVtab){
  return SQLITE_OK;
}

static int zipfileCommit(sqlite3_vtab *pVtab){
  ZipfileTab *pTab = (ZipfileTab*)pVtab;
  int rc = SQLITE_OK;
  if( pTab->pWriteFd ){
    i64 iOffset = pTab->szCurrent;
    ZipfileEntry *pEntry;
    ZipfileEOCD eocd;
    int nEntry = 0;

    /* Write out all entries */
    for(pEntry=pTab->pEntry; rc==SQLITE_OK && pEntry; pEntry=pEntry->pNext){
      rc = zipfileAppendData(pTab, pEntry->aCdsEntry, pEntry->nCdsEntry);
      nEntry++;
    }

    /* Write out the EOCD record */
    eocd.iDisk = 0;
    eocd.iFirstDisk = 0;
    eocd.nEntry = nEntry;







|



|
>















|




|
|







1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
  return zipfileAppendData(pTab, pTab->aBuffer, aBuf - pTab->aBuffer);
}

static void zipfileCleanupTransaction(ZipfileTab *pTab){
  ZipfileEntry *pEntry;
  ZipfileEntry *pNext;

  for(pEntry=pTab->pFirstEntry; pEntry; pEntry=pNext){
    pNext = pEntry->pNext;
    sqlite3_free(pEntry);
  }
  pTab->pFirstEntry = 0;
  pTab->pLastEntry = 0;
  fclose(pTab->pWriteFd);
  pTab->pWriteFd = 0;
  pTab->szCurrent = 0;
  pTab->szOrig = 0;
}

static int zipfileBegin(sqlite3_vtab *pVtab){
  return SQLITE_OK;
}

static int zipfileCommit(sqlite3_vtab *pVtab){
  ZipfileTab *pTab = (ZipfileTab*)pVtab;
  int rc = SQLITE_OK;
  if( pTab->pWriteFd ){
    i64 iOffset = pTab->szCurrent;
    ZipfileEntry *p;
    ZipfileEOCD eocd;
    int nEntry = 0;

    /* Write out all entries */
    for(p=pTab->pFirstEntry; rc==SQLITE_OK && p; p=p->pNext){
      rc = zipfileAppendData(pTab, p->aCdsEntry, p->nCdsEntry);
      nEntry++;
    }

    /* Write out the EOCD record */
    eocd.iDisk = 0;
    eocd.iFirstDisk = 0;
    eocd.nEntry = nEntry;
Changes to test/zipfile.test.
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56

57
58
59
60
  INSERT INTO zz VALUES('f.txt', '-rw-r--r--', 1000000000, 5, 'abcde', 0);
  INSERT INTO zz VALUES('g.txt', '-rw-r--r--', 1000000002, 5, '12345', 0);
}

do_execsql_test 1.2 {
  SELECT name, mtime, data FROM zipfile('test.zip');
} {
  g.txt 1000000002 12345
  f.txt 1000000000 abcde 
}

do_execsql_test 1.3 {
  INSERT INTO zz VALUES('h.txt', 
    '-rw-r--r--', 1000000004, 20, 'aaaaaaaaaabbbbbbbbbb', NULL
  );
}

do_execsql_test 1.4 {
  SELECT name, mtime, zipfile_uncompress(data, sz, method), method
  FROM zipfile('test.zip');
} {
  h.txt 1000000004 aaaaaaaaaabbbbbbbbbb 8
  f.txt 1000000000 abcde 0
  g.txt 1000000002 12345 0

}

finish_test








|
|












<


>




33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53

54
55
56
57
58
59
60
  INSERT INTO zz VALUES('f.txt', '-rw-r--r--', 1000000000, 5, 'abcde', 0);
  INSERT INTO zz VALUES('g.txt', '-rw-r--r--', 1000000002, 5, '12345', 0);
}

do_execsql_test 1.2 {
  SELECT name, mtime, data FROM zipfile('test.zip');
} {
  f.txt 1000000000 abcde 
  g.txt 1000000002 12345
}

do_execsql_test 1.3 {
  INSERT INTO zz VALUES('h.txt', 
    '-rw-r--r--', 1000000004, 20, 'aaaaaaaaaabbbbbbbbbb', NULL
  );
}

do_execsql_test 1.4 {
  SELECT name, mtime, zipfile_uncompress(data, sz, method), method
  FROM zipfile('test.zip');
} {

  f.txt 1000000000 abcde 0
  g.txt 1000000002 12345 0
  h.txt 1000000004 aaaaaaaaaabbbbbbbbbb 8
}

finish_test