/ Check-in [e2114df1]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:When creating a new archive entry, have zipfile store UTC instead of local time in the legacy MS-DOS format timestamp field.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | zipfile-timestamp-fix
Files: files | file ages | folders
SHA3-256: e2114df18383d111dd5fbac902e08b42a7f4b2b2d6f7bf29574a3722e4a4dad5
User & Date: dan 2018-01-31 19:13:31
Context
2018-01-31
19:45
Fix a test case in zipfile.test. Closed-Leaf check-in: 4eb5b24c user: dan tags: zipfile-timestamp-fix
19:13
When creating a new archive entry, have zipfile store UTC instead of local time in the legacy MS-DOS format timestamp field. check-in: e2114df1 user: dan tags: zipfile-timestamp-fix
16:50
Improve the omit-left-join optimization so that it works in some cases when the RHS is subject to a UNIQUE but not NOT NULL constraint. check-in: 02ba8a7b user: drh tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/misc/zipfile.c.

   682    682   **   File modification date:
   683    683   **     Bits 00-04: day
   684    684   **     Bits 05-08: month (1-12)
   685    685   **     Bits 09-15: years from 1980 
   686    686   **
   687    687   ** https://msdn.microsoft.com/en-us/library/9kkf9tah.aspx
   688    688   */
   689         -static time_t zipfileMtime(ZipfileCDS *pCDS){
   690         -  struct tm t;
   691         -  memset(&t, 0, sizeof(t));
   692         -  t.tm_sec = (pCDS->mTime & 0x1F)*2;
   693         -  t.tm_min = (pCDS->mTime >> 5) & 0x2F;
   694         -  t.tm_hour = (pCDS->mTime >> 11) & 0x1F;
          689  +static u32 zipfileMtime(ZipfileCDS *pCDS){
          690  +  int Y = (1980 + ((pCDS->mDate >> 9) & 0x7F));
          691  +  int M = ((pCDS->mDate >> 5) & 0x0F);
          692  +  int D = (pCDS->mDate & 0x1F);
          693  +  int B = -13;
   695    694   
   696         -  t.tm_mday = (pCDS->mDate & 0x1F);
   697         -  t.tm_mon = ((pCDS->mDate >> 5) & 0x0F) - 1;
   698         -  t.tm_year = 80 + ((pCDS->mDate >> 9) & 0x7F);
          695  +  int sec = (pCDS->mTime & 0x1F)*2;
          696  +  int min = (pCDS->mTime >> 5) & 0x3F;
          697  +  int hr = (pCDS->mTime >> 11) & 0x1F;
   699    698   
   700         -  return mktime(&t);
          699  +  /* JD = INT(365.25 * (Y+4716)) + INT(30.6001 * (M+1)) + D + B - 1524.5 */
          700  +
          701  +  /* Calculate the JD in seconds for noon on the day in question */
          702  +  if( M<3 ){
          703  +    Y = Y-1;
          704  +    M = M+12;
          705  +  }
          706  +  i64 JD = (i64)(24*60*60) * (
          707  +      (int)(365.25 * (Y + 4716))
          708  +    + (int)(30.6001 * (M + 1))
          709  +    + D + B - 1524
          710  +  );
          711  +
          712  +  /* Correct the JD for the time within the day */
          713  +  JD += (hr-12) * 3600 + min * 60 + sec;
          714  +
          715  +  /* Convert JD to unix timestamp (the JD epoch is 2440587.5) */
          716  +  return (u32)(JD - (i64)(24405875) * 24*60*6);
   701    717   }
   702    718   
   703    719   /*
   704    720   ** The opposite of zipfileMtime(). This function populates the mTime and
   705    721   ** mDate fields of the CDS structure passed as the first argument according
   706    722   ** to the UNIX timestamp value passed as the second.
   707    723   */
   708    724   static void zipfileMtimeToDos(ZipfileCDS *pCds, u32 mUnixTime){
   709         -  time_t t = (time_t)mUnixTime;
   710         -  struct tm res;
   711         -
   712         -#if !defined(_WIN32) && !defined(WIN32)
   713         -  localtime_r(&t, &res);
   714         -#else
   715         -  memcpy(&res, localtime(&t), sizeof(struct tm));
   716         -#endif
   717         -
   718         -  pCds->mTime = (u16)(
   719         -    (res.tm_sec / 2) + 
   720         -    (res.tm_min << 5) +
   721         -    (res.tm_hour << 11));
   722         -
   723         -  pCds->mDate = (u16)(
   724         -    (res.tm_mday-1) +
   725         -    ((res.tm_mon+1) << 5) +
   726         -    ((res.tm_year-80) << 9));
          725  +  /* Convert unix timestamp to JD (2440588 is noon on 1/1/1970) */
          726  +  i64 JD = (i64)2440588 + mUnixTime / (24*60*60);
          727  +
          728  +  int A, B, C, D, E;
          729  +  int yr, mon, day;
          730  +  int hr, min, sec;
          731  +
          732  +  A = (int)((JD - 1867216.25)/36524.25);
          733  +  A = JD + 1 + A - (A/4);
          734  +  B = A + 1524;
          735  +  C = (int)((B - 122.1)/365.25);
          736  +  D = (36525*(C&32767))/100;
          737  +  E = (int)((B-D)/30.6001);
          738  +
          739  +  day = B - D - (int)(30.6001*E);
          740  +  mon = (E<14 ? E-1 : E-13);
          741  +  yr = mon>2 ? C-4716 : C-4715;
          742  +
          743  +  hr = (mUnixTime % (24*60*60)) / (60*60);
          744  +  min = (mUnixTime % (60*60)) / 60;
          745  +  sec = (mUnixTime % 60);
          746  +
          747  +  pCds->mDate = (u16)(day + (mon << 5) + ((yr-1980) << 9));
          748  +  pCds->mTime = (u16)(sec/2 + (min<<5) + (hr<<11));
          749  +
          750  +  assert( mUnixTime<315507600 
          751  +       || mUnixTime==zipfileMtime(pCds) 
          752  +       || ((mUnixTime % 2) && mUnixTime-1==zipfileMtime(pCds)) 
          753  +       /* || (mUnixTime % 2) */
          754  +  );
   727    755   }
   728    756   
   729    757   /*
   730    758   ** If aBlob is not NULL, then it is a pointer to a buffer (nBlob bytes in
   731    759   ** size) containing an entire zip archive image. Or, if aBlob is NULL,
   732    760   ** then pFile is a file-handle open on a zip file. In either case, this
   733    761   ** function creates a ZipfileEntry object based on the zip archive entry

Changes to test/zipfile.test.

    43     43         FROM fsdir('test_unzip') 
    44     44         WHERE name!='test_unzip'
    45     45         ORDER BY name
    46     46       }]
    47     47       set res
    48     48     }
    49     49   }
           50  +
           51  +
           52  +# The argument is a blob (not a hex string) containing a zip archive.
           53  +# This proc removes the extended timestamp fields from the archive
           54  +# and returns the result.
           55  +#
           56  +proc remove_timestamps {blob} {
           57  +  set hex [binary encode hex $blob]
           58  +  set hex [string map {55540500 00000500} $hex]
           59  +  binary decode hex $hex
           60  +}
           61  +
    50     62   
    51     63   # Argument $file is the name of a zip archive on disk. This function
    52     64   # executes test cases to check that the results of each of the following 
    53     65   # are the same:
    54     66   #
    55     67   #         SELECT * FROM zipfile($file)
    56     68   #         SELECT * FROM zipfile( readfile($file) )
................................................................................
   363    375   do_catchsql_test 4.1 {
   364    376     CREATE VIRTUAL TABLE yyy USING zipfile();
   365    377   } {1 {zipfile constructor requires one argument}}
   366    378   do_catchsql_test 4.2 {
   367    379     CREATE VIRTUAL TABLE yyy USING zipfile('test.zip', 'test.zip');
   368    380   } {1 {zipfile constructor requires one argument}}
   369    381   
          382  +#--------------------------------------------------------------------------
          383  +
          384  +db func rt remove_timestamps
          385  +do_execsql_test 5.0 {
          386  +  WITH c(name,mtime,data) AS (
          387  +    SELECT 'a.txt', 946684800, 'abc'
          388  +  )
          389  +  SELECT name,mtime,data FROM zipfile(
          390  +    ( SELECT rt( zipfile(name,NULL,mtime,data) ) FROM c )
          391  +  )
          392  +} {
          393  +  a.txt 946684800 abc
          394  +}
          395  +
          396  +if {[info vars ::UNZIP]!=""} { 
          397  +  load_static_extension db fileio
          398  +  forcedelete test.zip
          399  +  do_test 6.0 {
          400  +    execsql {
          401  +      WITH c(name,mtime,data) AS (
          402  +        SELECT 'a.txt', 946684800, 'abc' UNION ALL
          403  +        SELECT 'b.txt', 1000000000, 'abc' UNION ALL
          404  +        SELECT 'c.txt', 1111111000, 'abc'
          405  +      )
          406  +      SELECT writefile('test.zip',
          407  +          ( SELECT rt ( zipfile(name,NULL,mtime,data) ) FROM c )
          408  +      );
          409  +    }
          410  +    forcedelete test_unzip
          411  +    file mkdir test_unzip
          412  +    exec unzip -d test_unzip test.zip
          413  +
          414  +    db eval {
          415  +      SELECT name, mtime FROM fsdir('test_unzip') WHERE name!='test_unzip'
          416  +      ORDER BY name
          417  +    }
          418  +  } [list {*}{
          419  +    test_unzip/a.txt 946684800
          420  +    test_unzip/b.txt 1000000000 
          421  +    test_unzip/c.txt 1111111000 
          422  +  }]
          423  +}
   370    424   
   371    425   
   372    426   finish_test
   373    427   

Changes to test/zipfile2.test.

    66     66     140000080000D4A52BEC09F3B6E0110000001100000005000900000000000000
    67     67     0000A48100000000612E747874555405000140420F00504B01021E0314000008
    68     68     0000D4A52BECD98916A71100000011000000050009000000000000000000A481
    69     69     3D000000622E747874555405000140420F00504B050600000000020002007800
    70     70     00007A0000000000
    71     71   }
    72     72   
    73         -do_execsql_test 3.1 {
    74         -  WITH contents(name,mtime,data) AS (
    75         -    VALUES('a.txt', 1000000, 'contents of a.txt') UNION ALL
    76         -    VALUES('b.txt', 1000000, 'contents of b.txt')
    77         -  ) SELECT quote( zipfile(name,NULL,mtime,data) ) FROM contents;
    78         -} [blobliteral $archive]
           73  +if 0 {
           74  +  # This test is broken - the archive generated is slightly different
           75  +  # depending on the zlib version used.
           76  +  do_execsql_test 3.1 {
           77  +    WITH contents(name,mtime,data) AS (
           78  +        VALUES('a.txt', 1000000, 'contents of a.txt') UNION ALL
           79  +        VALUES('b.txt', 1000000, 'contents of b.txt')
           80  +    ) SELECT quote( zipfile(name,NULL,mtime,data) ) FROM contents;
           81  +  } [blobliteral $archive]
           82  +}
           83  +
    79     84   
    80     85   set blob [blob $archive]
    81     86   do_execsql_test 3.2 {
    82     87     SELECT name,mtime,data FROM zipfile($blob)
    83     88   } {
    84     89     a.txt 1000000 {contents of a.txt} 
    85     90     b.txt 1000000 {contents of b.txt}
................................................................................
    94     99     set a [string replace $archive $idx [expr $idx+3] 0000]
    95    100     set blob [blob $a]
    96    101     do_catchsql_test 3.3.$i {
    97    102       SELECT name,mtime,data FROM zipfile($blob)
    98    103     } {/1 .*/}
    99    104   }
   100    105   
          106  +# Change the "extra info id" for all extended-timestamp fields.
   101    107   set L [findall 5554 $archive]
   102    108   for {set i 0} {$i < [llength $L]} {incr i} {
   103    109     set idx [lindex $L $i]
   104    110     set a [string replace $archive $idx [expr $idx+3] 1234]
   105    111     set blob [blob $a]
   106    112     do_execsql_test 3.4.$i {
   107    113       SELECT name,data FROM zipfile($blob)
................................................................................
   119    125       SELECT name,data FROM zipfile($blob)
   120    126     } {
   121    127       a.txt {contents of a.txt} 
   122    128       b.txt {contents of b.txt}
   123    129     }
   124    130   }
   125    131   
   126         -if 0 {
   127         -set blob [db one {
   128         -  WITH contents(name,mtime,data) AS (
   129         -    VALUES('a.txt', 1000000, 'aaaaaaaaaaaaaaaaaaaaaaa')
   130         -  ) SELECT quote( zipfile(name,NULL,mtime,data) ) FROM contents;
   131         -}]
   132         -set blob [string range $blob 2 end]
   133         -set blob [string range $blob 0 end-1]
   134         -while {[string length $blob]>0} {
   135         -  puts [string range $blob 0 63]
   136         -  set blob [string range $blob 64 end]
   137         -}
   138         -exit
   139         -}
          132  +# set blob [db one {
          133  +#   WITH contents(name,mtime,data) AS (
          134  +#     VALUES('a.txt', 1000000, 'aaaaaaaaaaaaaaaaaaaaaaa')
          135  +#   ) SELECT quote( zipfile(name,NULL,mtime,data) ) FROM contents;
          136  +# }]
          137  +# set blob [string range $blob 2 end]
          138  +# set blob [string range $blob 0 end-1]
          139  +# while {[string length $blob]>0} {
          140  +#   puts [string range $blob 0 63]
          141  +#   set blob [string range $blob 64 end]
          142  +# }
          143  +# exit
   140    144   
   141    145   set archive2 {
   142    146     504B0304140000080800D4A52BEC08F54C6E050000001700000005000900612E
   143    147     747874555405000140420F004B4CC40A00504B01021E03140000080800D4A52B
   144    148     EC08F54C6E0500000017000000050009000000000000000000A4810000000061
   145    149     2E747874555405000140420F00504B050600000000010001003C000000310000
   146    150     000000
................................................................................
   155    159   set L [findall 17000000 $archive2]
   156    160   set a $archive2
   157    161   foreach i $L { set a [string replace $a $i [expr $i+7] 16000000] }
   158    162   set blob [blob $a]
   159    163   do_catchsql_test 4.1 {
   160    164     SELECT name,mtime,data,method FROM zipfile($blob)
   161    165   } {1 {inflate() failed (0)}}
          166  +
   162    167   
   163    168   
   164    169   finish_test
   165    170