SQLite

Check-in [b0b7d0363a]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix a problem in the zipfile module causing it to generate incorrect checksums. Remove the ability to insert compressed data into a zip archive.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256: b0b7d0363acf38c2178e2d3041d8ce2a0de061a51caa64670dbf539ee6d4356b
User & Date: dan 2018-01-15 19:00:35.051
Context
2018-01-16
02:38
Disable the ".archive" command tests in shell8.test if the CLI is compiled without ZLIB support. (check-in: ce8bfe6c2b user: drh tags: trunk)
2018-01-15
21:59
Merge the enhancements associated with the first 3.22.0 beta. (check-in: c9d2ec51c8 user: drh tags: apple-osx)
19:00
Fix a problem in the zipfile module causing it to generate incorrect checksums. Remove the ability to insert compressed data into a zip archive. (check-in: b0b7d0363a user: dan tags: trunk)
15:49
Fix a zipfile problem with extracting zero length files compressed using deflate. (check-in: cf64087224 user: dan tags: trunk)
Changes
Unified Diff Ignore Whitespace Patch
Changes to ext/misc/zipfile.c.
︙︙
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828

829

830
831
832

833
834
835
836
837
838
839

static int zipfileColumn(
  sqlite3_vtab_cursor *cur,   /* The cursor */
  sqlite3_context *ctx,       /* First argument to sqlite3_result_...() */
  int i                       /* Which column to return */
){
  ZipfileCsr *pCsr = (ZipfileCsr*)cur;
  int rc = SQLITE_OK;
  if( i>=3 && sqlite3_vtab_nochange(ctx) ){
    return SQLITE_OK;
  }
  switch( i ){
    case 0:   /* name */
      sqlite3_result_text(ctx, pCsr->cds.zFile, -1, SQLITE_TRANSIENT);
      break;
    case 1:   /* mode */
      /* TODO: Whether or not the following is correct surely depends on
      ** the platform on which the archive was created.  */
      sqlite3_result_int(ctx, pCsr->cds.iExternalAttr >> 16);
      break;
    case 2: { /* mtime */
      if( pCsr->flags & ZIPFILE_MTIME_VALID ){
        sqlite3_result_int64(ctx, pCsr->mTime);
      }else{
        sqlite3_result_int64(ctx, zipfileMtime(pCsr));
      }
      break;
    }
    case 3: { /* sz */

      sqlite3_result_int64(ctx, pCsr->cds.szUncompressed);

      break;
    }
    case 4:   /* rawdata */

    case 5: { /* data */
      if( i==4 || pCsr->cds.iCompression==0 || pCsr->cds.iCompression==8 ){
        int sz = pCsr->cds.szCompressed;
        int szFinal = pCsr->cds.szUncompressed;
        if( szFinal>0 ){
          u8 *aBuf = sqlite3_malloc(sz);
          if( aBuf==0 ){








<
<
<


















>
|
>



>








801
802
803
804
805
806
807



808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839

static int zipfileColumn(
  sqlite3_vtab_cursor *cur,   /* The cursor */
  sqlite3_context *ctx,       /* First argument to sqlite3_result_...() */
  int i                       /* Which column to return */
){
  ZipfileCsr *pCsr = (ZipfileCsr*)cur;
  int rc = SQLITE_OK;



  switch( i ){
    case 0:   /* name */
      sqlite3_result_text(ctx, pCsr->cds.zFile, -1, SQLITE_TRANSIENT);
      break;
    case 1:   /* mode */
      /* TODO: Whether or not the following is correct surely depends on
      ** the platform on which the archive was created.  */
      sqlite3_result_int(ctx, pCsr->cds.iExternalAttr >> 16);
      break;
    case 2: { /* mtime */
      if( pCsr->flags & ZIPFILE_MTIME_VALID ){
        sqlite3_result_int64(ctx, pCsr->mTime);
      }else{
        sqlite3_result_int64(ctx, zipfileMtime(pCsr));
      }
      break;
    }
    case 3: { /* sz */
      if( sqlite3_vtab_nochange(ctx)==0 ){
        sqlite3_result_int64(ctx, pCsr->cds.szUncompressed);
      }
      break;
    }
    case 4:   /* rawdata */
      if( sqlite3_vtab_nochange(ctx) ) break;
    case 5: { /* data */
      if( i==4 || pCsr->cds.iCompression==0 || pCsr->cds.iCompression==8 ){
        int sz = pCsr->cds.szCompressed;
        int szFinal = pCsr->cds.szUncompressed;
        if( szFinal>0 ){
          u8 *aBuf = sqlite3_malloc(sz);
          if( aBuf==0 ){
︙︙
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882

      sqlite3_result_int(ctx, pCsr->cds.iCompression);
      break;
    case 7:   /* z */
      sqlite3_result_int64(ctx, pCsr->iId);
      break;
  }

  return SQLITE_OK;
}

/*
** Return the rowid for the current row.
*/
static int zipfileRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
  assert( 0 );








|








868
869
870
871
872
873
874
875
876
877
878
879
880
881
882

      sqlite3_result_int(ctx, pCsr->cds.iCompression);
      break;
    case 7:   /* z */
      sqlite3_result_int64(ctx, pCsr->iId);
      break;
  }

  return rc;
}

/*
** Return the rowid for the current row.
*/
static int zipfileRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
  assert( 0 );
︙︙
904
905
906
907
908
909
910

911
912
913
914
915
916
917
918

  int nRead;                      /* Bytes to read from file */
  i64 iOff;                       /* Offset to read from */
  int rc;

  fseek(pFile, 0, SEEK_END);
  szFile = (i64)ftell(pFile);
  if( szFile==0 ){

    return SQLITE_EMPTY;
  }
  nRead = (int)(MIN(szFile, ZIPFILE_BUFFER_SIZE));
  iOff = szFile - nRead;

  rc = zipfileReadData(pFile, aRead, nRead, iOff, &pTab->base.zErrMsg);
  if( rc==SQLITE_OK ){
    int i;








>
|








904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919

  int nRead;                      /* Bytes to read from file */
  i64 iOff;                       /* Offset to read from */
  int rc;

  fseek(pFile, 0, SEEK_END);
  szFile = (i64)ftell(pFile);
  if( szFile==0 ){
    memset(pEOCD, 0, sizeof(ZipfileEOCD));
    return SQLITE_OK;
  }
  nRead = (int)(MIN(szFile, ZIPFILE_BUFFER_SIZE));
  iOff = szFile - nRead;

  rc = zipfileReadData(pFile, aRead, nRead, iOff, &pTab->base.zErrMsg);
  if( rc==SQLITE_OK ){
    int i;
︙︙
982
983
984
985
986
987
988



989
990
991
992
993

994
995
996
997
998
999
1000

    pCsr->pFile = fopen(zFile, "rb");
    if( pCsr->pFile==0 ){
      zipfileSetErrmsg(pCsr, "cannot open file: %s", zFile);
      rc = SQLITE_ERROR;
    }else{
      rc = zipfileReadEOCD(pTab, pCsr->pFile, &pCsr->eocd);
      if( rc==SQLITE_OK ){



        pCsr->iNextOff = pCsr->eocd.iOffset;
        rc = zipfileNext(cur);
      }else if( rc==SQLITE_EMPTY ){
        rc = SQLITE_OK;
        pCsr->bEof = 1;

      }
    }
  }else{
    ZipfileEntry e;
    memset(&e, 0, sizeof(e));
    e.pNext = pTab->pFirstEntry;
    pCsr->pCurrent = &e;








>
>
>
|
|
<
<
<
>








983
984
985
986
987
988
989
990
991
992
993
994



995
996
997
998
999
1000
1001
1002

    pCsr->pFile = fopen(zFile, "rb");
    if( pCsr->pFile==0 ){
      zipfileSetErrmsg(pCsr, "cannot open file: %s", zFile);
      rc = SQLITE_ERROR;
    }else{
      rc = zipfileReadEOCD(pTab, pCsr->pFile, &pCsr->eocd);
      if( rc==SQLITE_OK ){
        if( pCsr->eocd.nEntry==0 ){
          pCsr->bEof = 1;
        }else{
          pCsr->iNextOff = pCsr->eocd.iOffset;
          rc = zipfileNext(cur);



        }
      }
    }
  }else{
    ZipfileEntry e;
    memset(&e, 0, sizeof(e));
    e.pNext = pTab->pFirstEntry;
    pCsr->pCurrent = &e;
︙︙
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077

}

static int zipfileLoadDirectory(ZipfileTab *pTab){
  ZipfileEOCD eocd;
  int rc;

  rc = zipfileReadEOCD(pTab, pTab->pWriteFd, &eocd);
  if( rc==SQLITE_OK ){
    int i;
    int iOff = 0;
    u8 *aBuf = sqlite3_malloc(eocd.nSize);
    if( aBuf==0 ){
      rc = SQLITE_NOMEM;
    }else{
      rc = zipfileReadData(








|








1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079

}

static int zipfileLoadDirectory(ZipfileTab *pTab){
  ZipfileEOCD eocd;
  int rc;

  rc = zipfileReadEOCD(pTab, pTab->pWriteFd, &eocd);
  if( rc==SQLITE_OK && eocd.nEntry>0 ){
    int i;
    int iOff = 0;
    u8 *aBuf = sqlite3_malloc(eocd.nSize);
    if( aBuf==0 ){
      rc = SQLITE_NOMEM;
    }else{
      rc = zipfileReadData(
︙︙
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123

        zipfileAddEntry(pTab, 0, pNew);
      }

      iOff += ZIPFILE_CDS_FIXED_SZ+nFile+nExtra+nComment;
    }

    sqlite3_free(aBuf);
  }else if( rc==SQLITE_EMPTY ){
    rc = SQLITE_OK;
  }

  return rc;
}

static ZipfileEntry *zipfileNewEntry(
  ZipfileCDS *pCds,               /* Values for fixed size part of CDS */








<
<








1110
1111
1112
1113
1114
1115
1116


1117
1118
1119
1120
1121
1122
1123

        zipfileAddEntry(pTab, 0, pNew);
      }

      iOff += ZIPFILE_CDS_FIXED_SZ+nFile+nExtra+nComment;
    }

    sqlite3_free(aBuf);


  }

  return rc;
}

static ZipfileEntry *zipfileNewEntry(
  ZipfileCDS *pCds,               /* Values for fixed size part of CDS */
︙︙
1290
1291
1292
1293
1294
1295
1296

1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314

1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357

  int nData = 0;                  /* Size of pData buffer in bytes */
  int iMethod = 0;                /* Compression method for new entry */
  u8 *pFree = 0;                  /* Free this */
  char *zFree = 0;                /* Also free this */
  ZipfileCDS cds;                 /* New Central Directory Structure entry */
  ZipfileEntry *pOld = 0;
  int bIsDir = 0;


  assert( pTab->zFile );
  assert( pTab->pWriteFd );

  if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
    const char *zDelete = (const char*)sqlite3_value_text(apVal[0]);
    int nDelete = (int)strlen(zDelete);
    for(pOld=pTab->pFirstEntry; 1; pOld=pOld->pNext){
      if( pOld->bDeleted ) continue;
      if( zipfileComparePath(pOld->zPath, zDelete, nDelete)==0 ){
        pOld->bDeleted = 1;
        break;
      }
      assert( pOld->pNext );
    }
    if( nVal==1 ) return SQLITE_OK;
  }


  if( sqlite3_value_nochange(apVal[5]) && sqlite3_value_nochange(apVal[6])
   && sqlite3_value_nochange(apVal[7]) && sqlite3_value_nochange(apVal[8])
  ){
    /* Reuse the data from the existing entry. */
    FILE *pFile = pTab->pWriteFd;
    zipfileReadCDS(pOld->aCdsEntry, &cds);

    bIsDir = ((cds.iExternalAttr>>16) & S_IFDIR) ? 1 : 0;
    sz = cds.szUncompressed;
    iMethod = cds.iCompression;
    if( sz>0 ){
      char **pzErr = &pTab->base.zErrMsg;
      ZipfileLFH lfh;
      rc = zipfileReadLFH(pFile, cds.iOffset, pTab->aBuffer, &lfh, pzErr);
      if( rc==SQLITE_OK ){
        nData = lfh.szCompressed;
        pData = pFree = sqlite3_malloc(nData);
        if( pFree==NULL ){
          rc = SQLITE_NOMEM;
        }else{
          i64 iRead = cds.iOffset + ZIPFILE_LFH_FIXED_SZ + lfh.nFile+lfh.nExtra;
          rc = zipfileReadData(pFile, pFree, nData, iRead, pzErr);
        }
      }
    }
  }else{
    int mNull;
    mNull = (sqlite3_value_type(apVal[5])==SQLITE_NULL ? 0x0 : 0x8)  /* sz */
      + (sqlite3_value_type(apVal[6])==SQLITE_NULL ? 0x0 : 0x4)  /* rawdata */
      + (sqlite3_value_type(apVal[7])==SQLITE_NULL ? 0x0 : 0x2)  /* data */
      + (sqlite3_value_type(apVal[8])==SQLITE_NULL ? 0x0 : 0x1); /* method */
    if( mNull==0x00 ){     
      /* All four are NULL - this must be a directory */
      bIsDir = 1;
    }
    else if( mNull==0x2 || mNull==0x3 ){
      /* Value specified for "data", and possibly "method". This must be
      ** a regular file or a symlink. */
      const u8 *aIn = sqlite3_value_blob(apVal[7]);
      int nIn = sqlite3_value_bytes(apVal[7]);
      int bAuto = sqlite3_value_type(apVal[8])==SQLITE_NULL;

      iMethod = sqlite3_value_int(apVal[8]);








>


















>
|
|

<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
|
<
<
<
|
|
<
<
<
|
<
|
<
<
|

|
<








1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319















1320



1321
1322



1323

1324


1325
1326
1327

1328
1329
1330
1331
1332
1333
1334

  int nData = 0;                  /* Size of pData buffer in bytes */
  int iMethod = 0;                /* Compression method for new entry */
  u8 *pFree = 0;                  /* Free this */
  char *zFree = 0;                /* Also free this */
  ZipfileCDS cds;                 /* New Central Directory Structure entry */
  ZipfileEntry *pOld = 0;
  int bIsDir = 0;
  u32 iCrc32 = 0;

  assert( pTab->zFile );
  assert( pTab->pWriteFd );

  if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
    const char *zDelete = (const char*)sqlite3_value_text(apVal[0]);
    int nDelete = (int)strlen(zDelete);
    for(pOld=pTab->pFirstEntry; 1; pOld=pOld->pNext){
      if( pOld->bDeleted ) continue;
      if( zipfileComparePath(pOld->zPath, zDelete, nDelete)==0 ){
        pOld->bDeleted = 1;
        break;
      }
      assert( pOld->pNext );
    }
    if( nVal==1 ) return SQLITE_OK;
  }

  /* Check that "sz" and "rawdata" are both NULL: */
  if( sqlite3_value_type(apVal[5])!=SQLITE_NULL
   || sqlite3_value_type(apVal[6])!=SQLITE_NULL
  ){















    rc = SQLITE_CONSTRAINT;



  }




  if( rc==SQLITE_OK ){

    if( sqlite3_value_type(apVal[7])==SQLITE_NULL ){


      /* data=NULL. A directory */
      bIsDir = 1;
    }else{

      /* Value specified for "data", and possibly "method". This must be
      ** a regular file or a symlink. */
      const u8 *aIn = sqlite3_value_blob(apVal[7]);
      int nIn = sqlite3_value_bytes(apVal[7]);
      int bAuto = sqlite3_value_type(apVal[8])==SQLITE_NULL;

      iMethod = sqlite3_value_int(apVal[8]);
︙︙
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397

        if( rc==SQLITE_OK ){
          if( iMethod || nCmp<nIn ){
            iMethod = 8;
            pData = pFree;
            nData = nCmp;
          }
        }
      }
    }
    else if( mNull==0x0D ){
      /* Values specified for "sz", "rawdata" and "method". In other words,
      ** pre-compressed data is being inserted.  */
      pData = sqlite3_value_blob(apVal[6]);
      nData = sqlite3_value_bytes(apVal[6]);
      sz = sqlite3_value_int(apVal[5]);
      iMethod = sqlite3_value_int(apVal[8]);
      if( iMethod<0 || iMethod>65535 ){
        pTab->base.zErrMsg = sqlite3_mprintf(
            "zipfile: invalid compression method: %d", iMethod
            );
        rc = SQLITE_ERROR;
      }
    }
    else{
      rc = SQLITE_CONSTRAINT;
    }
  }

  if( rc==SQLITE_OK ){
    rc = zipfileGetMode(pTab, apVal[3], 
        (bIsDir ? (S_IFDIR + 0755) : (S_IFREG + 0644)), &mode
    );








<
<
<
<
<
<
<
<
<
<
<
<
|
<

<
<
<








1343
1344
1345
1346
1347
1348
1349












1350

1351



1352
1353
1354
1355
1356
1357
1358

        if( rc==SQLITE_OK ){
          if( iMethod || nCmp<nIn ){
            iMethod = 8;
            pData = pFree;
            nData = nCmp;
          }
        }












        iCrc32 = crc32(0, aIn, nIn);

      }



    }
  }

  if( rc==SQLITE_OK ){
    rc = zipfileGetMode(pTab, apVal[3], 
        (bIsDir ? (S_IFDIR + 0755) : (S_IFREG + 0644)), &mode
    );
︙︙
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455

    /* Create the new CDS record. */
    memset(&cds, 0, sizeof(cds));
    cds.iVersionMadeBy = ZIPFILE_NEWENTRY_MADEBY;
    cds.iVersionExtract = ZIPFILE_NEWENTRY_REQUIRED;
    cds.flags = ZIPFILE_NEWENTRY_FLAGS;
    cds.iCompression = (u16)iMethod;
    zipfileMtimeToDos(&cds, (u32)mTime);
    cds.crc32 = crc32(0, pData, nData);
    cds.szCompressed = nData;
    cds.szUncompressed = (u32)sz;
    cds.iExternalAttr = (mode<<16);
    cds.iOffset = (u32)pTab->szCurrent;
    pNew = zipfileNewEntry(&cds, zPath, nPath, (u32)mTime);
    if( pNew==0 ){
      rc = SQLITE_NOMEM;








|








1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416

    /* Create the new CDS record. */
    memset(&cds, 0, sizeof(cds));
    cds.iVersionMadeBy = ZIPFILE_NEWENTRY_MADEBY;
    cds.iVersionExtract = ZIPFILE_NEWENTRY_REQUIRED;
    cds.flags = ZIPFILE_NEWENTRY_FLAGS;
    cds.iCompression = (u16)iMethod;
    zipfileMtimeToDos(&cds, (u32)mTime);
    cds.crc32 = iCrc32;
    cds.szCompressed = nData;
    cds.szUncompressed = (u32)sz;
    cds.iExternalAttr = (mode<<16);
    cds.iOffset = (u32)pTab->szCurrent;
    pNew = zipfileNewEntry(&cds, zPath, nPath, (u32)mTime);
    if( pNew==0 ){
      rc = SQLITE_NOMEM;
︙︙
Changes to test/zipfile.test.
︙︙
32
33
34
35
36
37
38









39
40
41
42
43
44
45
46
47
48
49
50
51
52

  2 mtime {} 0 {} 0 
  3 sz {} 0 {} 0 
  4 rawdata {} 0 {} 0
  5 data {} 0 {} 0
  6 method {} 0 {} 0
}










do_execsql_test 1.1.1 {
  INSERT INTO zz(name, mode, mtime, sz, rawdata, method) 
  VALUES('f.txt', '-rw-r--r--', 1000000000, 5, 'abcde', 0);
}
do_execsql_test 1.1.2 {
  INSERT INTO zz(name, mtime, sz, rawdata, method) 
  VALUES('g.txt', 1000000002, 5, '12345', 0);
}

do_execsql_test 1.2 {
  SELECT name, mtime, data FROM zipfile('test.zip')
} {
  f.txt 1000000000 abcde 
  g.txt 1000000002 12345








>
>
>
>
>
>
>
>
>

|
|


|
|








32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61

  2 mtime {} 0 {} 0 
  3 sz {} 0 {} 0 
  4 rawdata {} 0 {} 0
  5 data {} 0 {} 0
  6 method {} 0 {} 0
}

do_catchsql_test 1.1.0.1 {
  INSERT INTO zz(name, mode, mtime, sz, rawdata, method) 
  VALUES('f.txt', '-rw-r--r--', 1000000000, 5, 'abcde', 0);
} {1 {constraint failed}}
do_catchsql_test 1.1.0.1 {
  INSERT INTO zz(name, mtime, sz, rawdata, method) 
  VALUES('g.txt', 1000000002, 5, '12345', 0);
} {1 {constraint failed}}

do_execsql_test 1.1.1 {
  INSERT INTO zz(name, mode, mtime, data, method) 
  VALUES('f.txt', '-rw-r--r--', 1000000000, 'abcde', 0);
}
do_execsql_test 1.1.2 {
  INSERT INTO zz(name, mode, mtime, data, method) 
  VALUES('g.txt', NULL, 1000000002, '12345', 0);
}

do_execsql_test 1.2 {
  SELECT name, mtime, data FROM zipfile('test.zip')
} {
  f.txt 1000000000 abcde 
  g.txt 1000000002 12345
︙︙