feat(zip): Support deflate compression (bring your own)

Author: kriskowal

packages/compartment-mapper/test/integrity.test.js

@@ -64,7 +64,7 @@ test('extracting an archive with a missing file', async t => {
       }),
     {
       message:
-        'Failed to load module "./main.js" in package "app-v1.0.0" (1 underlying failures: Cannot find file app-v1.0.0/main.js in Zip file missing.zip',
+        'Failed to load module "./main.js" in package "app-v1.0.0" (1 underlying failures: Cannot find file app-v1.0.0/main.js in ZIP file missing.zip',
     },
   );
 
@@ -88,6 +88,8 @@ test('extracting an archive with an inconsistent hash', async t => {
   const content = new Uint8Array(node.content.byteLength + 1);
   content.set(node.content, 0);
   node.content = content;
+  node.uncompressedLength += 1;
+  node.compressedLength += 1;
 
   const invalidBytes = writer.snapshot();
 
@@ -136,6 +138,8 @@ test('extracting an archive with an inconsistent compartment map hash', async t
   const content = new Uint8Array(node.content.byteLength + 1);
   content.fill(' '.charCodeAt(0));
   content.set(node.content, 0);
+  node.uncompressedLength += 1;
+  node.compressedLength += 1;
   node.content = content;
 
   const invalidBytes = writer.snapshot();
@@ -176,6 +180,8 @@ test('extracting an archive with an inconsistent compartment map hash with expec
   const content = new Uint8Array(node.content.byteLength + 1);
   content.fill(' '.charCodeAt(0));
   content.set(node.content, 0);
+  node.uncompressedLength += 1;
+  node.compressedLength += 1;
   node.content = content;
 
   const invalidBytes = writer.snapshot();

packages/compartment-mapper/test/retained.test.js

@@ -21,7 +21,8 @@ test('archives only contain compartments retained by modules', async t => {
   });
 
   const reader = new ZipReader(bytes);
-  const compartmentMapBytes = reader.files.get('compartment-map.json').content;
+  const compartmentMapBytes = reader.files.get('compartment-map.json')?.content;
+  t.assert(compartmentMapBytes);
   const compartmentMapText = new TextDecoder().decode(compartmentMapBytes);
   const compartmentMap = JSON.parse(compartmentMapText);
   t.deepEqual(Object.keys(compartmentMap.compartments), [

packages/compartment-mapper/test/stability.test.js

@@ -10,7 +10,8 @@ test('order of duplicate name/version packages', async t => {
   const bytes = await makeArchive(readPowers, fixture);
 
   const reader = new ZipReader(bytes);
-  const compartmentMapBytes = reader.files.get('compartment-map.json').content;
+  const compartmentMapBytes = reader.files.get('compartment-map.json')?.content;
+  t.assert(compartmentMapBytes);
   const compartmentMapText = new TextDecoder().decode(compartmentMapBytes);
   const compartmentMap = JSON.parse(compartmentMapText);
 

packages/zip/NEWS.md

@@ -1,4 +1,8 @@
-User-visible changes in Zip:
+User-visible changes in ZIP:
+
+# Next release
+
+- Adds support for DEFLATE compression and decompression.
 
 # 0.2.0 (2021-06-01)
 

packages/zip/README.md

@@ -1,4 +1,137 @@
-# Endo Zip
+# Endo ZIP
+
+This is a lightweight JavaScript implementation of ZIP.
+The implementation operates on whole ZIP archives in memory and operates
+exclusively on `Uint8Array` file contents.
+
+The library does entrain a specific DEFLATE compressor or decompressor, but it
+will use one if you provide it, and will otherwise just archive or extract
+uncompressed files.
+
+## Usage
+
+### Writing ZIP archives
+
+Create a ZIP archive by instantiating `ZipWriter`, adding files with `write()`,
+and generating the final archive with `snapshot()`:
+
+```javascript
+import { ZipWriter } from '@endo/zip';
+
+const textEncoder = new TextEncoder();
+const writer = new ZipWriter();
+
+// Add a file to the archive
+writer.set('hello.txt', textEncoder.encode('Hello, World!\n'), {
+  mode: 0o644,
+  date: new Date(),
+});
+
+// Generate the ZIP archive as a Uint8Array
+const zipBytes = writer.snapshot();
+```
+
+#### Options for `write()`
+
+- `mode` (number, default: `0o644`): Unix file permissions
+- `date` (Date, optional): File modification date
+- `comment` (string, default: `''`): File comment
+
+#### Compression support
+
+By default, files are stored uncompressed.
+To enable DEFLATE compression, provide compression functions when creating the
+writer:
+
+```javascript
+// Using the Compression Streams API (available in modern browsers and Node.js 18+)
+const deflate = async (bytes) => {
+  const blob = new Blob([bytes]);
+  const stream = blob.stream().pipeThrough(new CompressionStream('deflate-raw'));
+  const compressed = await new Response(stream).arrayBuffer();
+  return new Uint8Array(compressed);
+};
+
+const writer = new ZipWriter({ deflate });
+await writer.set('data.txt', textEncoder.encode('Large data...'), {
+  date: new Date(),
+});
+```
+
+For synchronous compression, if available:
+
+```javascript
+const writer = new ZipWriter({ deflateNow });
+writer.write('data.txt', textEncoder.encode('Data...'));
+```
+
+### Reading ZIP archives
+
+Read files from a ZIP archive using `ZipReader`:
+
+```javascript
+import { ZipReader } from '@endo/zip';
+
+const textDecoder = new TextDecoder();
+
+// Create a reader from ZIP bytes
+const reader = new ZipReader(zipBytes);
+
+// Read a file (synchronous for uncompressed files)
+const fileBytes = reader.read('hello.txt');
+const text = textDecoder.decode(fileBytes);
+
+// Get file metadata
+const stat = reader.stat('hello.txt');
+console.log(stat.mode, stat.date, stat.comment);
+```
+
+#### Reading compressed archives
+
+To read ZIP files with DEFLATE compression, provide an inflate function:
+
+```javascript
+// Using the Compression Streams API
+const inflate = async (bytes) => {
+  const blob = new Blob([bytes]);
+  const stream = blob.stream().pipeThrough(new DecompressionStream('deflate-raw'));
+  const decompressed = await new Response(stream).arrayBuffer();
+  return new Uint8Array(decompressed);
+};
+
+const reader = new ZipReader(zipBytes, { inflate });
+
+// Decompress asynchronously
+const fileBytes = await reader.get('compressed-file.txt');
+```
+
+For synchronous decompression:
+
+```javascript
+const reader = new ZipReader(zipBytes, { inflateNow: syncDecompressFunction });
+const fileBytes = reader.getNow('compressed-file.txt');
+```
+
+### Helper functions
+
+The package also exports constructor-free adapters.
+These make the archive more like a file system by adding gratuitious
+asynchrony.
+
+```javascript
+import { writeZip, readZip } from '@endo/zip';
+
+// Writing
+const { write, snapshot } = writeZip({ deflate });
+await write('file.txt', textEncoder.encode('content'));
+const zipBytes = await snapshot();
+
+// Reading
+const { read } = await readZip(zipBytes, 'archive.zip', { inflate });
+const fileBytes = await read('file.txt');
+```
+
+## Implementation Notes
 
 This is a modernization and specialization of [JSZip][] (MIT License) that has
 no dependencies on any built-in modules and is entirely implemented with
@@ -13,33 +146,33 @@ requiring a date to be expressly provided instead of reaching for the ambient
 original Date constructor, which will pointedly be absent in constructed
 compartments in locked-down environments.
 
-Zip format allows for an arbitrary-length comment and an arbitrary number of
+ZIP format allows for an arbitrary-length comment and an arbitrary number of
 Zip64 headers in the "end of central directory block".
-Zip implementations must therefore scan backward from the end for the magic
+ZIP implementations must therefore scan backward from the end for the magic
 numbers that introduce the "EOCDB".
-However, a specially crafted Zip file may contain those magic numbers
+However, a specially crafted ZIP file may contain those magic numbers
 before the end.
 
 So, for security, this specialized library does not support Zip64 nor
 the variable width archive comment.
 With some difficulty, Zip64 might be recovered by scanning backward from the
 end of the file until we find a coherent EOCDB with no trailing bytes.
 Even careful support for the variable width comment at the end of the archive
-would always allow for the possibility of a comment that is itself a valid Zip
-file with a long prefix, since Zip files allow an arbitrary length prefix.
+would always allow for the possibility of a comment that is itself a valid ZIP
+file with a long prefix, since ZIP files allow an arbitrary length prefix.
 
-For expedience, the specialization dropped support for INFLATE compression.
-The dependency would need to be converted to ECMAScript modules, which is not
-much effort. Pursuing that intent, one should factor out the shared CRC32
-module.
+DEFLATE compression support requires providing your own compression/decompression
+functions. Modern environments can use the [Compression Streams API][] with
+`'deflate-raw'` format. The dependency would need to be converted to ECMAScript
+modules, which is not much effort.
 
 JSZip supports an asynchronous mode, that despite the name, is not concurrent.
 The mode is intended to keep the main thread lively while emitting progress
 reports.  For expedience, this mode is omitted, but could be restored using the
 same underlying utilities, and I expect async/await and async iterators would
 make the feature easier to maintain.
 
-Provided an async seekable reader, a lazy Zip reader could be built on the same
+Provided an async seekable reader, a lazy ZIP reader could be built on the same
 foundations, deferring decompression and validation until the file is opened.
 
 For expedience, support for streaming compression and the necessary data
@@ -55,3 +188,4 @@ For expedience, there is no API for enumerating the contents of the archive.
 This would be straightforward to implement.
 
  [JSZip]: https://github.com/Stuk/jszip
+ [Compression Streams API]: https://developer.mozilla.org/en-US/docs/Web/API/Compression_Streams_API

packages/zip/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@endo/zip",
   "version": "1.0.11",
-  "description": "A minimal, synchronous Zip reader and writer",
+  "description": "A minimal ZIP archive reader and writer",
   "keywords": [
     "zip",
     "ses",

packages/zip/reader.js

@@ -1 +1 @@
-export { ZipReader } from './src/reader.js';
+export { ZipReader, readZip } from './src/reader.js';

packages/zip/src/compression.js

@@ -2,3 +2,5 @@
 
 // STORE is the magic number for "not compressed".
 export const STORE = 0;
+export const DEFLATE = 8;
+// export const BZIP2 = 12;