fix(worker): read ZIP-spec spanned archives (.z01 + .zip) instead of failing silently

A `.z01`/`.z02`/…/`.zip` set is a spanned (multi-disk) archive, not one ZIP
file cut into chunks. Both ZIP listing paths assumed the latter:

  - readZipCentralDirectory fed the parts to yauzl through a concatenating
    random-access reader; yauzl hard-refuses nonzero disk numbers
    ("multi-disk zip files are not supported"), and the EOCD's directory
    pointer is volume-relative anyway.
  - readScannedZipListing (the ranged, no-download path) passed a
    whole-archive tailStart, making the computed directory offset wildly
    negative so the parser threw RangeError on every tail size.

Both failures were swallowed upstream, so every spanned set was ingested and
uploaded with an empty file list — invisible to content and keyword search.

Add a volume-aware central-directory reader: locate the EOCD in the final
volume, resolve the directory's (volume, offset) via the disk fields, and read
just the directory bytes, spilling across volumes if it straddles a boundary.
ZIP64 spanned archives are handled through the ZIP64 EOCD locator. The
existing hand-rolled walker in central-directory.ts is reused rather than
adding a third parser.

The two shapes are told apart by filename shape, not by the detector's
multipart `pattern`, so this stays independent of how detect.ts labels them.
A set named like volumes whose EOCD reports a single disk really is a byte
split, and falls back to the concatenating reader.

Byte-split (.zip.001) and single-.zip reading are unchanged; both are now
covered by regression tests. Tests build spanned archives byte-by-byte and
cross-check against real Info-ZIP `zip -s` output (including `-fz` ZIP64)
where the CLI is available.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

worker/src/archive/central-directory.ts

@@ -11,13 +11,21 @@ function extOf(name: string): string | null {
   return e === "" ? null : e;
 }
 
+/**
+ * Locate the End Of Central Directory record by scanning backward for its
+ * signature. Returns -1 when the buffer holds no EOCD.
+ */
+export function findEocdOffset(tail: Buffer): number {
+  for (let i = tail.length - 22; i >= 0; i--) {
+    if (tail.readUInt32LE(i) === EOCD_SIG) return i;
+  }
+  return -1;
+}
+
 /** Parse a ZIP central directory from the tail of an archive. */
 export function parseZipCentralDirectoryFromTail(tail: Buffer, tailStart: number): FileEntry[] {
   // 1. Find EOCD by scanning backward for its signature.
-  let eocd = -1;
-  for (let i = tail.length - 22; i >= 0; i--) {
-    if (tail.readUInt32LE(i) === EOCD_SIG) { eocd = i; break; }
-  }
+  const eocd = findEocdOffset(tail);
   if (eocd < 0) throw new RangeError("EOCD not found in tail");
 
   let cdSize = tail.readUInt32LE(eocd + 12);
@@ -45,10 +53,22 @@ export function parseZipCentralDirectoryFromTail(tail: Buffer, tailStart: number
   }
 
   // 3. Walk central-directory headers.
+  return walkCentralDirectory(tail, cdLocal, cdSize);
+}
+
+/**
+ * Walk a run of central-directory file headers and return the file entries
+ * (directory entries are skipped).
+ *
+ * `buf` must contain the whole central directory starting at `start`; for a
+ * spanned archive that means the caller has already stitched together the
+ * volumes the directory straddles.
+ */
+export function walkCentralDirectory(tail: Buffer, start: number, cdSize: number): FileEntry[] {
   const entries: FileEntry[] = [];
-  let p = cdLocal;
-  const end = cdLocal + cdSize;
-  while (p + 46 <= end && tail.readUInt32LE(p) === CD_SIG) {
+  let p = start;
+  const end = start + cdSize;
+  while (p + 46 <= end && p + 46 <= tail.length && tail.readUInt32LE(p) === CD_SIG) {
     let crc = tail.readUInt32LE(p + 16) >>> 0;
     let comp = BigInt(tail.readUInt32LE(p + 20));
     let uncomp = BigInt(tail.readUInt32LE(p + 24));

worker/src/archive/ranged/dispatch.test.ts

@@ -1,15 +1,103 @@
 import { describe, it, expect } from "vitest";
-import { readScannedListingRanged } from "./dispatch.js";
+import { readScannedListingRanged, readScannedZipListing } from "./dispatch.js";
+import type { RangeReader } from "./range-reader.js";
+import { buildSpannedStoreZip, buildStoreZip, byteSplit } from "../testing/spanned-zip-fixture.js";
+
+/** Serve ranged reads out of in-memory part buffers keyed by fileId. */
+function readerFor(parts: { fileId: string; buf: Buffer }[]): RangeReader {
+  const byId = new Map(parts.map((p) => [p.fileId, p.buf]));
+  return async (fileId, offset, length) => {
+    const buf = byId.get(fileId);
+    if (!buf) throw new Error(`unknown fileId ${fileId}`);
+    return buf.subarray(offset, offset + length);
+  };
+}
+
+const FILES = [
+  { name: "src/b.bin", data: Buffer.alloc(2048, 7), disk: 0 },
+  { name: "src/models/dragon.stl", data: Buffer.from("DRAGON"), disk: 1 },
+  { name: "readme.txt", data: Buffer.from("hello world"), disk: 2 },
+];
 
 describe("readScannedListingRanged", () => {
   it("returns null for an unknown archive type without calling the reader", async () => {
-    const read = async () => Buffer.alloc(0);
     const result = await readScannedListingRanged(
       "DOCUMENT",
       { invoke: async () => ({}) } as never,
       [{ fileId: "1", fileSize: 100n, fileName: "a.pdf" }],
     );
     expect(result).toBeNull();
-    void read; // unused placeholder kept out of the dispatch call — DOCUMENT never reaches a reader
+  });
+});
+
+describe("readScannedZipListing", () => {
+  it("lists a ZIP-spec spanned set (.z01 … .zip) from the final volume's tail", async () => {
+    const vols = buildSpannedStoreZip(FILES, 3);
+    const names = ["Pack.z01", "Pack.z02", "Pack.zip"];
+    const parts = vols.map((buf, i) => ({ fileId: String(i), fileSize: BigInt(buf.length), fileName: names[i] }));
+
+    const entries = await readScannedZipListing(
+      parts,
+      readerFor(parts.map((p, i) => ({ fileId: p.fileId, buf: vols[i] }))),
+    );
+
+    expect(entries).not.toBeNull();
+    expect(entries!.map((e) => e.path).sort()).toEqual([
+      "readme.txt",
+      "src/b.bin",
+      "src/models/dragon.stl",
+    ]);
+  });
+
+  it("only reads the final volume — the earlier volumes are never downloaded", async () => {
+    const vols = buildSpannedStoreZip(FILES, 3);
+    const names = ["Pack.z01", "Pack.z02", "Pack.zip"];
+    const parts = vols.map((buf, i) => ({ fileId: String(i), fileSize: BigInt(buf.length), fileName: names[i] }));
+    const touched: string[] = [];
+    const base = readerFor(parts.map((p, i) => ({ fileId: p.fileId, buf: vols[i] })));
+
+    await readScannedZipListing(parts, async (id, off, len, size) => {
+      touched.push(id);
+      return base(id, off, len, size);
+    });
+
+    expect([...new Set(touched)]).toEqual(["2"]);
+  });
+
+  it("still lists a 7-Zip raw byte split (.zip.001 …) using whole-archive offsets", async () => {
+    const zip = buildStoreZip([
+      { name: "models/knight.stl", data: Buffer.alloc(5000, 9) },
+      { name: "license.txt", data: Buffer.from("MIT") },
+    ]);
+    const chunks = byteSplit(zip, 3);
+    const names = ["Pack.zip.001", "Pack.zip.002", "Pack.zip.003"];
+    const parts = chunks.map((buf, i) => ({ fileId: String(i), fileSize: BigInt(buf.length), fileName: names[i] }));
+
+    const entries = await readScannedZipListing(
+      parts,
+      readerFor(parts.map((p, i) => ({ fileId: p.fileId, buf: chunks[i] }))),
+    );
+
+    expect(entries!.map((e) => e.fileName).sort()).toEqual(["knight.stl", "license.txt"]);
+  });
+
+  it("returns null for a spanned set whose central directory starts on an earlier volume", async () => {
+    const vols = buildSpannedStoreZip(
+      FILES.map((f) => ({ ...f, disk: Math.min(f.disk, 1) })),
+      3,
+      { cdStartDisk: 1 },
+    );
+    const names = ["Pack.z01", "Pack.z02", "Pack.zip"];
+    const parts = vols.map((buf, i) => ({ fileId: String(i), fileSize: BigInt(buf.length), fileName: names[i] }));
+
+    const entries = await readScannedZipListing(
+      parts,
+      readerFor(parts.map((p, i) => ({ fileId: p.fileId, buf: vols[i] }))),
+    );
+    expect(entries).toBeNull();
+  });
+
+  it("returns null when there are no parts", async () => {
+    expect(await readScannedZipListing([], readerFor([]))).toBeNull();
   });
 });

worker/src/archive/ranged/dispatch.ts

@@ -1,11 +1,11 @@
 import type { Client } from "tdl";
-import { downloadFileRange } from "../../tdlib/range-download.js";
-import { parseZipCentralDirectoryFromTail, MIN_ZIP_TAIL_BYTES } from "../central-directory.js";
+import { parseZipCentralDirectoryFromTail, findEocdOffset, MIN_ZIP_TAIL_BYTES } from "../central-directory.js";
+import { isSpannedZipPartSet } from "../zip-spanned.js";
 import { childLogger } from "../../util/logger.js";
 import type { FileEntry } from "../zip-reader.js";
 import { readSevenZListingRanged, type RangedPart } from "./sevenz-ranged.js";
 import { readRarListingRanged } from "./rar-ranged.js";
-import { tdlibRangeReader } from "./range-reader.js";
+import { tdlibRangeReader, type RangeReader } from "./range-reader.js";
 
 const log = childLogger("ranged-dispatch");
 
@@ -13,25 +13,45 @@ const log = childLogger("ranged-dispatch");
  * Read a ZIP central directory from the tail of a (possibly multipart)
  * archive. `parts` is ordered; only the LAST part carries the EOCD record.
  * `fileSize` on each part is that part's own size (NOT the whole-archive
- * total) so the download offset stays within that part's bounds, while
- * `tailStart` passed to the parser is the logical whole-archive offset
- * (preceding parts' sizes + the offset within the last part).
+ * total) so the download offset stays within that part's bounds.
+ *
+ * Which logical offset the EOCD's central-directory pointer is measured from
+ * depends on the multipart shape:
+ *
+ *  - 7-Zip raw byte split (`.zip.001`, …): the parts are one ZIP file cut into
+ *    chunks, so the pointer is a whole-archive offset → `tailStart` is the
+ *    preceding parts' sizes plus the offset within the last part.
+ *  - ZIP-spec spanned archive (`.z01`, …, `.zip`): each volume is its own unit
+ *    and the pointer is relative to the start of the volume holding the
+ *    directory → `tailStart` is just the offset within that final volume.
+ *
+ * Getting this wrong makes the computed directory offset wildly negative, and
+ * the parser then throws RangeError on every tail size — which is exactly how
+ * spanned sets ended up indexed with no file list at all.
  */
 export async function readScannedZipListing(
-  client: Client,
-  parts: { fileId: string; fileSize: bigint }[],
+  parts: RangedPart[],
+  read: RangeReader,
 ): Promise<FileEntry[] | null> {
   if (parts.length === 0) return null;
   const lastPart = parts[parts.length - 1];
-  const precedingSize = parts.slice(0, -1).reduce((sum, p) => sum + Number(p.fileSize), 0);
+  const spanned = isSpannedZipPartSet(parts.map((p) => p.fileName));
+  const precedingSize = spanned
+    ? 0
+    : parts.slice(0, -1).reduce((sum, p) => sum + Number(p.fileSize), 0);
   const lastSize = Number(lastPart.fileSize);
   for (const tailBytes of [MIN_ZIP_TAIL_BYTES, MIN_ZIP_TAIL_BYTES * 4]) {
     const partOffset = Math.max(0, lastSize - tailBytes);
     const downloadLen = Math.min(tailBytes, lastSize);
     try {
-      const buf = await downloadFileRange(client, lastPart.fileId, partOffset, downloadLen, lastPart.fileSize);
-      const tailStart = precedingSize + partOffset;
-      return parseZipCentralDirectoryFromTail(buf, tailStart);
+      const buf = await read(lastPart.fileId, partOffset, downloadLen, lastPart.fileSize);
+      if (spanned && !cdStartsOnFinalVolume(buf)) {
+        // The directory begins on an earlier volume; reaching it would mean
+        // ranged-reading that volume too. Leave it to the full-download path.
+        log.debug({ fileId: lastPart.fileId }, "spanned ZIP central directory is not on the final volume");
+        return null;
+      }
+      return parseZipCentralDirectoryFromTail(buf, partOffset + precedingSize);
     } catch (err) {
       if (err instanceof RangeError) continue; // try a larger tail
       log.warn({ err, fileId: lastPart.fileId }, "ranged ZIP listing failed");
@@ -41,6 +61,20 @@ export async function readScannedZipListing(
   return null;
 }
 
+/**
+ * For a spanned archive, whether the EOCD says the central directory starts on
+ * the very volume that EOCD lives on (the usual case). ZIP64's saturated
+ * 0xFFFF disk fields are treated as "yes" — the ZIP64 record that supersedes
+ * them is itself in this tail, and the parser resolves it there.
+ */
+function cdStartsOnFinalVolume(tail: Buffer): boolean {
+  const eocd = findEocdOffset(tail);
+  if (eocd < 0) return true; // let the parser report the real problem
+  const thisDisk = tail.readUInt16LE(eocd + 4);
+  const cdStartDisk = tail.readUInt16LE(eocd + 6);
+  return thisDisk === cdStartDisk || thisDisk === 0xffff || cdStartDisk === 0xffff;
+}
+
 /**
  * Dispatch a (no-download) inner-file listing read by archive type. Used both
  * by the provenance-backfill path (reading an already-uploaded copy) and the
@@ -55,7 +89,7 @@ export async function readScannedListingRanged(
   parts: RangedPart[],
 ): Promise<FileEntry[] | null> {
   const read = tdlibRangeReader(client);
-  if (archiveType === "ZIP") return readScannedZipListing(client, parts);
+  if (archiveType === "ZIP") return readScannedZipListing(parts, read);
   if (archiveType === "SEVEN_Z") return readSevenZListingRanged(parts, read);
   if (archiveType === "RAR") return readRarListingRanged(parts, read);
   return null;

worker/src/archive/testing/spanned-zip-fixture.ts

@@ -0,0 +1,191 @@
+import { crc32 } from "zlib"; // Node 20+ exposes zlib.crc32
+
+/**
+ * Test-only builders that emit real ZIP byte streams.
+ *
+ * Two distinct on-disk shapes are produced here, because the worker has to
+ * tell them apart:
+ *
+ *  - `buildSpannedStoreZip` → a ZIP-spec **spanned/multi-disk** archive
+ *    (`Pack.z01`, `Pack.z02`, …, `Pack.zip`). Each volume is its own file;
+ *    central-directory records carry a disk number, and the EOCD's
+ *    "offset of start of central directory" is relative to the *start of the
+ *    disk that holds it*, not to a concatenation of the volumes.
+ *
+ *  - `buildStoreZip` → an ordinary single-file ZIP. Cutting its bytes into
+ *    chunks yields the 7-Zip raw byte split shape (`Pack.zip.001`, …), where
+ *    all disk numbers are 0 and offsets are whole-archive absolute.
+ *
+ * Field layouts follow APPNOTE 6.3.x sections 4.3.12 (central directory) and
+ * 4.3.16 (EOCD). Verified against Info-ZIP `zip -s` output.
+ */
+
+const LOCAL_SIG = 0x04034b50;
+const CD_SIG = 0x02014b50;
+const EOCD_SIG = 0x06054b50;
+/** APPNOTE 8.5.3: the first volume of a spanned archive starts with this. */
+const SPANNING_SIG = 0x08074b50;
+
+export interface FixtureFile {
+  name: string;
+  data: Buffer;
+  /** 0-based volume this file's local header + data is written to. */
+  disk?: number;
+}
+
+function localHeader(name: string, data: Buffer): Buffer {
+  const nameBuf = Buffer.from(name, "utf8");
+  const local = Buffer.alloc(30);
+  local.writeUInt32LE(LOCAL_SIG, 0);
+  local.writeUInt16LE(20, 4); // version needed
+  local.writeUInt16LE(0, 6); // flags
+  local.writeUInt16LE(0, 8); // method = store
+  local.writeUInt32LE(crc32(data) >>> 0, 14);
+  local.writeUInt32LE(data.length, 18); // compressed
+  local.writeUInt32LE(data.length, 22); // uncompressed
+  local.writeUInt16LE(nameBuf.length, 26);
+  local.writeUInt16LE(0, 28); // extra len
+  return Buffer.concat([local, nameBuf, data]);
+}
+
+function centralHeader(name: string, data: Buffer, diskStart: number, relOffset: number): Buffer {
+  const nameBuf = Buffer.from(name, "utf8");
+  const cd = Buffer.alloc(46);
+  cd.writeUInt32LE(CD_SIG, 0);
+  cd.writeUInt16LE(20, 4); // version made by
+  cd.writeUInt16LE(20, 6); // version needed
+  cd.writeUInt16LE(0, 8); // flags
+  cd.writeUInt16LE(0, 10); // method = store
+  cd.writeUInt32LE(crc32(data) >>> 0, 16);
+  cd.writeUInt32LE(data.length, 20); // compressed
+  cd.writeUInt32LE(data.length, 24); // uncompressed
+  cd.writeUInt16LE(nameBuf.length, 28);
+  cd.writeUInt16LE(diskStart, 34); // disk number start
+  cd.writeUInt32LE(relOffset, 42); // offset of local header, relative to its disk
+  return Buffer.concat([cd, nameBuf]);
+}
+
+function eocd(opts: {
+  thisDisk: number;
+  cdStartDisk: number;
+  entriesThisDisk: number;
+  entriesTotal: number;
+  cdSize: number;
+  cdOffset: number;
+}): Buffer {
+  const buf = Buffer.alloc(22);
+  buf.writeUInt32LE(EOCD_SIG, 0);
+  buf.writeUInt16LE(opts.thisDisk, 4);
+  buf.writeUInt16LE(opts.cdStartDisk, 6);
+  buf.writeUInt16LE(opts.entriesThisDisk, 8);
+  buf.writeUInt16LE(opts.entriesTotal, 10);
+  buf.writeUInt32LE(opts.cdSize, 12);
+  buf.writeUInt32LE(opts.cdOffset, 16);
+  return buf;
+}
+
+/** Build an ordinary single-file STORE ZIP (all disk numbers 0). */
+export function buildStoreZip(files: FixtureFile[]): Buffer {
+  const body: Buffer[] = [];
+  const central: Buffer[] = [];
+  let offset = 0;
+  for (const f of files) {
+    const local = localHeader(f.name, f.data);
+    body.push(local);
+    central.push(centralHeader(f.name, f.data, 0, offset));
+    offset += local.length;
+  }
+  const cdBuf = Buffer.concat(central);
+  return Buffer.concat([
+    ...body,
+    cdBuf,
+    eocd({
+      thisDisk: 0,
+      cdStartDisk: 0,
+      entriesThisDisk: files.length,
+      entriesTotal: files.length,
+      cdSize: cdBuf.length,
+      cdOffset: offset,
+    }),
+  ]);
+}
+
+/**
+ * Build a ZIP-spec spanned archive as one Buffer per volume.
+ * Returned array is volume order: [z01, z02, …, zip] (last element is the
+ * final volume, which carries the central directory and EOCD).
+ *
+ * `cdStartDisk` (default: last volume) lets a test place the central
+ * directory so that it begins on an earlier volume and spills forward,
+ * exercising the cross-volume read path. As in a real writer, no file data
+ * may live on a volume after the one the directory starts on — those volumes
+ * hold directory continuation only.
+ */
+export function buildSpannedStoreZip(
+  files: FixtureFile[],
+  totalDisks: number,
+  opts: { cdStartDisk?: number } = {}
+): Buffer[] {
+  const cdStart = opts.cdStartDisk ?? totalDisks - 1;
+  for (const f of files) {
+    if ((f.disk ?? 0) > cdStart) {
+      throw new Error(`fixture misuse: ${f.name} is on volume ${f.disk} but the CD starts on ${cdStart}`);
+    }
+    if ((f.disk ?? 0) >= totalDisks) {
+      throw new Error(`fixture misuse: ${f.name} is on volume ${f.disk} of ${totalDisks}`);
+    }
+  }
+  const chunks: Buffer[][] = Array.from({ length: totalDisks }, () => []);
+  const lengths = new Array<number>(totalDisks).fill(0);
+
+  const marker = Buffer.alloc(4);
+  marker.writeUInt32LE(SPANNING_SIG, 0);
+  chunks[0].push(marker);
+  lengths[0] = 4;
+
+  const central: Buffer[] = [];
+  for (const f of files) {
+    const disk = f.disk ?? 0;
+    const local = localHeader(f.name, f.data);
+    central.push(centralHeader(f.name, f.data, disk, lengths[disk]));
+    chunks[disk].push(local);
+    lengths[disk] += local.length;
+  }
+
+  const cdBuf = Buffer.concat(central);
+  const lastDisk = totalDisks - 1;
+  const cdStartDisk = cdStart;
+  const cdOffset = lengths[cdStartDisk];
+
+  // Write the CD starting on cdStartDisk, spilling onto later volumes.
+  const spillDisks = lastDisk - cdStartDisk + 1;
+  const firstChunkLen = Math.ceil(cdBuf.length / spillDisks);
+  let written = 0;
+  for (let d = cdStartDisk; d <= lastDisk; d++) {
+    const take = d === lastDisk ? cdBuf.length - written : Math.min(firstChunkLen, cdBuf.length - written);
+    chunks[d].push(cdBuf.subarray(written, written + take));
+    lengths[d] += take;
+    written += take;
+  }
+
+  chunks[lastDisk].push(
+    eocd({
+      thisDisk: lastDisk,
+      cdStartDisk,
+      entriesThisDisk: files.length,
+      entriesTotal: files.length,
+      cdSize: cdBuf.length,
+      cdOffset,
+    })
+  );
+
+  return chunks.map((c) => Buffer.concat(c));
+}
+
+/** Cut a buffer into `count` roughly equal chunks (7-Zip raw byte split). */
+export function byteSplit(buf: Buffer, count: number): Buffer[] {
+  const size = Math.ceil(buf.length / count);
+  const out: Buffer[] = [];
+  for (let i = 0; i < buf.length; i += size) out.push(buf.subarray(i, i + size));
+  return out;
+}

worker/src/archive/zip-reader.ts

@@ -3,6 +3,7 @@ import { open as fsOpen, stat as fsStat } from "fs/promises";
 import path from "path";
 import { Readable } from "stream";
 import { childLogger } from "../util/logger.js";
+import { isSpannedZipPartSet, readSpannedZipCentralDirectory } from "./zip-spanned.js";
 
 const log = childLogger("zip-reader");
 
@@ -17,9 +18,15 @@ export interface FileEntry {
 
 /**
  * Read the central directory of a ZIP file without extracting any contents.
- * For multipart ZIPs (.zip.001, .zip.002 etc.), uses a custom random-access
- * reader that spans all parts seamlessly so yauzl can find the central
- * directory at the end of the combined data.
+ *
+ * Three shapes are handled:
+ *  - a single `.zip` → yauzl directly;
+ *  - a 7-Zip raw byte split (`.zip.001`, `.zip.002`, …), which is one ZIP file
+ *    cut into chunks → a random-access reader that spans the chunks so yauzl
+ *    sees the combined stream;
+ *  - a ZIP-spec spanned/multi-disk archive (`.z01`, `.z02`, …, `.zip`), a
+ *    different on-disk format that yauzl refuses outright → a dedicated
+ *    volume-aware central-directory reader.
  */
 export async function readZipCentralDirectory(
   filePaths: string[]
@@ -28,7 +35,22 @@ export async function readZipCentralDirectory(
     return readSingleZip(filePaths[0]);
   }
 
-  // Multipart: use a spanning random-access reader
+  if (isSpannedZipPartSet(filePaths)) {
+    try {
+      const result = await readSpannedZipCentralDirectory(filePaths);
+      if (result.kind === "entries") return result.entries;
+      if (result.kind === "failed") {
+        log.warn({ reason: result.reason, parts: filePaths.length }, "Failed to read spanned ZIP");
+        return [];
+      }
+      // "not-spanned": named like volumes but really a byte split — fall through.
+    } catch (err) {
+      log.warn({ err, parts: filePaths.length }, "Failed to read spanned ZIP");
+      return [];
+    }
+  }
+
+  // Multipart byte split: use a spanning random-access reader
   return readMultipartZip(filePaths);
 }
 

worker/src/archive/zip-spanned.test.ts

@@ -0,0 +1,214 @@
+import { describe, it, expect, beforeAll, afterAll } from "vitest";
+import { mkdir, mkdtemp, readdir, rm, writeFile } from "fs/promises";
+import { execFile } from "child_process";
+import { promisify } from "util";
+import { tmpdir } from "os";
+import path from "path";
+import { isSpannedZipPartSet } from "./zip-spanned.js";
+import { readZipCentralDirectory } from "./zip-reader.js";
+import { buildSpannedStoreZip, buildStoreZip, byteSplit } from "./testing/spanned-zip-fixture.js";
+
+const execFileAsync = promisify(execFile);
+
+let dir: string;
+
+beforeAll(async () => {
+  dir = await mkdtemp(path.join(tmpdir(), "zip-spanned-"));
+});
+
+afterAll(async () => {
+  await rm(dir, { recursive: true, force: true });
+});
+
+/** Write volume buffers out under the given names and return their paths. */
+async function writeParts(names: string[], buffers: Buffer[], sub: string): Promise<string[]> {
+  const base = path.join(dir, sub);
+  await rm(base, { recursive: true, force: true });
+  await mkdir(base, { recursive: true });
+  const paths: string[] = [];
+  for (let i = 0; i < names.length; i++) {
+    const p = path.join(base, names[i]);
+    await writeFile(p, buffers[i]);
+    paths.push(p);
+  }
+  return paths;
+}
+
+const FILES = [
+  { name: "src/b.bin", data: Buffer.alloc(2048, 7), disk: 0 },
+  { name: "src/models/dragon.stl", data: Buffer.from("DRAGON"), disk: 1 },
+  { name: "src/models/", data: Buffer.alloc(0), disk: 1 },
+  { name: "src/a.bin", data: Buffer.alloc(4096, 3), disk: 2 },
+  { name: "readme.txt", data: Buffer.from("hello world"), disk: 3 },
+];
+
+describe("isSpannedZipPartSet", () => {
+  it("recognizes a .z01 + .zip volume set", () => {
+    expect(isSpannedZipPartSet(["/t/Pack.z01", "/t/Pack.z02", "/t/Pack.zip"])).toBe(true);
+  });
+
+  it("recognizes the set regardless of the order it is handed in", () => {
+    expect(isSpannedZipPartSet(["/t/Pack.zip", "/t/Pack.z02", "/t/Pack.z01"])).toBe(true);
+  });
+
+  it("rejects a 7-Zip raw byte split (.zip.001)", () => {
+    expect(isSpannedZipPartSet(["/t/Pack.zip.001", "/t/Pack.zip.002"])).toBe(false);
+  });
+
+  it("rejects a single .zip", () => {
+    expect(isSpannedZipPartSet(["/t/Pack.zip"])).toBe(false);
+  });
+
+  it("rejects a set with no final .zip volume", () => {
+    expect(isSpannedZipPartSet(["/t/Pack.z01", "/t/Pack.z02"])).toBe(false);
+  });
+
+  it("rejects a set with two .zip volumes", () => {
+    expect(isSpannedZipPartSet(["/t/A.zip", "/t/B.zip"])).toBe(false);
+  });
+});
+
+describe("readZipCentralDirectory on a spanned (.z01 + .zip) set", () => {
+  it("lists every entry with correct paths, sizes and crc32", async () => {
+    const vols = buildSpannedStoreZip(FILES, 4);
+    const paths = await writeParts(["Pack.z01", "Pack.z02", "Pack.z03", "Pack.zip"], vols, "spanned");
+
+    const entries = await readZipCentralDirectory(paths);
+
+    expect(entries.map((e) => e.path).sort()).toEqual([
+      "readme.txt",
+      "src/a.bin",
+      "src/b.bin",
+      "src/models/dragon.stl",
+    ]);
+    const dragon = entries.find((e) => e.fileName === "dragon.stl")!;
+    expect(dragon.uncompressedSize).toBe(6n);
+    expect(dragon.extension).toBe("stl");
+    expect(dragon.crc32).toMatch(/^[0-9a-f]{8}$/);
+    const a = entries.find((e) => e.fileName === "a.bin")!;
+    expect(a.uncompressedSize).toBe(4096n);
+    expect(a.compressedSize).toBe(4096n);
+  });
+
+  it("works when the parts are handed over out of volume order", async () => {
+    const vols = buildSpannedStoreZip(FILES, 4);
+    const paths = await writeParts(["Pack.z01", "Pack.z02", "Pack.z03", "Pack.zip"], vols, "unordered");
+    const shuffled = [paths[3], paths[1], paths[0], paths[2]];
+
+    const entries = await readZipCentralDirectory(shuffled);
+    expect(entries).toHaveLength(4);
+  });
+
+  it("reads a central directory that begins on an earlier volume and spills forward", async () => {
+    // Files live on volumes 0–2; the directory starts on volume 2 and
+    // continues onto the final .zip, which holds nothing else.
+    const spillFiles = FILES.map((f) => ({ ...f, disk: Math.min(f.disk, 2) }));
+    const vols = buildSpannedStoreZip(spillFiles, 4, { cdStartDisk: 2 });
+    const paths = await writeParts(["Pack.z01", "Pack.z02", "Pack.z03", "Pack.zip"], vols, "spilled");
+
+    const entries = await readZipCentralDirectory(paths);
+    expect(entries.map((e) => e.fileName).sort()).toEqual(["a.bin", "b.bin", "dragon.stl", "readme.txt"]);
+  });
+
+  it("returns [] instead of throwing when a needed volume is missing", async () => {
+    const spillFiles = FILES.map((f) => ({ ...f, disk: Math.min(f.disk, 1) }));
+    const vols = buildSpannedStoreZip(spillFiles, 3, { cdStartDisk: 1 });
+    const paths = await writeParts(["Pack.z01", "Pack.z02", "Pack.zip"], vols, "missing");
+    // Drop Pack.z02 — the volume the central directory starts on.
+    const entries = await readZipCentralDirectory([paths[0], paths[2]]);
+    expect(entries).toEqual([]);
+  });
+
+  it("returns [] instead of throwing when the final volume is garbage", async () => {
+    const vols = buildSpannedStoreZip([{ name: "x.stl", data: Buffer.alloc(64, 1), disk: 0 }], 2);
+    const paths = await writeParts(["Pack.z01", "Pack.zip"], [vols[0], Buffer.alloc(500, 0x5a)], "garbage");
+    expect(await readZipCentralDirectory(paths)).toEqual([]);
+  });
+
+  it("falls back to concatenation semantics when .z01-named parts are really a byte split", async () => {
+    // Some producers name a raw byte split .z01/.zip. The EOCD then reports
+    // disk 0, so concatenation — not volume mapping — is the correct reading.
+    const zip = buildStoreZip([
+      { name: "one.stl", data: Buffer.alloc(3000, 1) },
+      { name: "two.stl", data: Buffer.alloc(3000, 2) },
+    ]);
+    const paths = await writeParts(["Pack.z01", "Pack.zip"], byteSplit(zip, 2), "mislabeled");
+    const entries = await readZipCentralDirectory(paths);
+    expect(entries.map((e) => e.fileName).sort()).toEqual(["one.stl", "two.stl"]);
+  });
+});
+
+describe("readZipCentralDirectory regressions for the shapes that already worked", () => {
+  it("still reads a 7-Zip raw byte split (.zip.001 …)", async () => {
+    const zip = buildStoreZip([
+      { name: "models/knight.stl", data: Buffer.alloc(5000, 9) },
+      { name: "license.txt", data: Buffer.from("MIT") },
+    ]);
+    const paths = await writeParts(
+      ["Pack.zip.001", "Pack.zip.002", "Pack.zip.003"],
+      byteSplit(zip, 3),
+      "bytesplit"
+    );
+    const entries = await readZipCentralDirectory(paths);
+    expect(entries.map((e) => e.fileName).sort()).toEqual(["knight.stl", "license.txt"]);
+    expect(entries.find((e) => e.fileName === "knight.stl")!.uncompressedSize).toBe(5000n);
+  });
+
+  it("still reads a plain single .zip", async () => {
+    const zip = buildStoreZip([{ name: "solo.stl", data: Buffer.from("SOLO") }]);
+    const paths = await writeParts(["Pack.zip"], [zip], "single");
+    const entries = await readZipCentralDirectory(paths);
+    expect(entries.map((e) => e.fileName)).toEqual(["solo.stl"]);
+  });
+});
+
+// ── Cross-checks against real Info-ZIP output ────────────────────────────
+// Skipped automatically where the `zip` CLI is unavailable; the hand-built
+// fixtures above are the authoritative, portable coverage.
+
+const HAS_ZIP_CLI = await execFileAsync("zip", ["-v"]).then(
+  () => true,
+  () => false
+);
+
+/** Collect Pack.z01 … Pack.zip from a directory, in volume order. */
+async function collectVolumes(work: string): Promise<string[]> {
+  const names = await readdir(work);
+  const vols = names.filter((n) => /^Pack\.z\d{2,}$/i.test(n)).sort();
+  const final = names.find((n) => /^Pack\.zip$/i.test(n));
+  expect(final).toBeDefined();
+  expect(vols.length).toBeGreaterThan(0);
+  return [...vols, final!].map((n) => path.join(work, n));
+}
+
+describe.skipIf(!HAS_ZIP_CLI)("readZipCentralDirectory against archives produced by Info-ZIP `zip -s`", () => {
+  it("lists a genuine spanned archive", async () => {
+    const work = path.join(dir, "real");
+    await mkdir(path.join(work, "src/models"), { recursive: true });
+    await writeFile(path.join(work, "src/big.bin"), Buffer.alloc(300_000, 4));
+    await writeFile(path.join(work, "src/models/dragon.stl"), Buffer.alloc(200_000, 5));
+    await writeFile(path.join(work, "src/readme.txt"), "hello world");
+    await execFileAsync("zip", ["-r", "-0", "-s", "100k", "Pack.zip", "src"], { cwd: work });
+
+    const entries = await readZipCentralDirectory(await collectVolumes(work));
+    expect(entries.map((e) => e.path).sort()).toEqual([
+      "src/big.bin",
+      "src/models/dragon.stl",
+      "src/readme.txt",
+    ]);
+    expect(entries.find((e) => e.fileName === "dragon.stl")!.uncompressedSize).toBe(200_000n);
+  });
+
+  it("lists a genuine ZIP64 spanned archive", async () => {
+    const work = path.join(dir, "real64");
+    await mkdir(work, { recursive: true });
+    await writeFile(path.join(work, "big.bin"), Buffer.alloc(300_000, 6));
+    await writeFile(path.join(work, "note.txt"), "zip64 spanned");
+    // -fz forces ZIP64 structures even though the payload is small.
+    await execFileAsync("zip", ["-0", "-fz", "-s", "100k", "Pack.zip", "big.bin", "note.txt"], { cwd: work });
+
+    const entries = await readZipCentralDirectory(await collectVolumes(work));
+    expect(entries.map((e) => e.fileName).sort()).toEqual(["big.bin", "note.txt"]);
+    expect(entries.find((e) => e.fileName === "big.bin")!.uncompressedSize).toBe(300_000n);
+  });
+});

worker/src/archive/zip-spanned.ts

@@ -0,0 +1,170 @@
+import { open as fsOpen, stat as fsStat } from "fs/promises";
+import path from "path";
+import { findEocdOffset, walkCentralDirectory, MIN_ZIP_TAIL_BYTES } from "./central-directory.js";
+import { childLogger } from "../util/logger.js";
+import type { FileEntry } from "./zip-reader.js";
+
+const log = childLogger("zip-spanned");
+
+const ZIP64_LOCATOR_SIG = 0x07064b50;
+const ZIP64_EOCD_SIG = 0x06064b50;
+
+/** Refuse to allocate a buffer for an absurd central-directory size. */
+const MAX_CD_BYTES = 256 * 1024 * 1024;
+
+/**
+ * A `.z01`/`.z02`/…/`.zip` set is a ZIP-spec **spanned (multi-disk)** archive:
+ * a genuinely different on-disk format from a 7-Zip raw byte split
+ * (`.zip.001`, `.zip.002`, …), which is one ZIP file cut into chunks.
+ *
+ * The distinction matters because a byte split is read by concatenating the
+ * chunks, whereas in a spanned archive each volume is its own unit: the EOCD's
+ * central-directory offset is relative to the start of the volume that holds
+ * the directory, and central-directory records carry a volume number. Feeding
+ * a spanned set to a concatenating reader yields nonsense offsets (and yauzl
+ * refuses outright: "multi-disk zip files are not supported").
+ *
+ * Detected from filename shape rather than the detector's multipart `pattern`
+ * so this stays independent of how `detect.ts` labels the two variants.
+ * Order-independent: the caller may hand the volumes over in any order.
+ */
+export function isSpannedZipPartSet(filePaths: string[]): boolean {
+  if (filePaths.length < 2) return false;
+  const names = filePaths.map((p) => path.basename(p));
+  const finals = names.filter((n) => /\.zip$/i.test(n));
+  const volumes = names.filter((n) => /\.z\d{2,}$/i.test(n));
+  return finals.length === 1 && volumes.length === names.length - 1;
+}
+
+export type SpannedZipResult =
+  /** Successfully read; `entries` may legitimately be empty for an empty archive. */
+  | { kind: "entries"; entries: FileEntry[] }
+  /** The EOCD reports a single disk — the parts are really a byte split, so the
+   *  caller should fall back to reading them as one concatenated stream. */
+  | { kind: "not-spanned" }
+  | { kind: "failed"; reason: string };
+
+/**
+ * Read the central directory of a ZIP-spec spanned archive.
+ *
+ * Only the volume holding the central directory (plus any it spills onto) is
+ * read, and only the directory bytes themselves — the file payloads are never
+ * touched, so this is cheap regardless of archive size.
+ *
+ * Coverage: STORE/DEFLATE and ZIP64 spanned archives are handled. Archives
+ * whose central directory is itself encrypted (strong encryption / "hide
+ * filenames") cannot be listed by any header-only reader and return `failed`.
+ */
+export async function readSpannedZipCentralDirectory(filePaths: string[]): Promise<SpannedZipResult> {
+  const volumes = new Map<number, string>();
+  let finalVolume: string | undefined;
+  for (const p of filePaths) {
+    const base = path.basename(p);
+    const m = base.match(/\.z(\d{2,})$/i);
+    if (m) {
+      // .z01 is volume 0, .z02 is volume 1, … (APPNOTE numbers disks from 0).
+      volumes.set(parseInt(m[1], 10) - 1, p);
+    } else if (/\.zip$/i.test(base)) {
+      finalVolume = p;
+    }
+  }
+  if (!finalVolume) return { kind: "failed", reason: "no final .zip volume" };
+
+  const finalSize = (await fsStat(finalVolume)).size;
+  const tailLen = Math.min(finalSize, MIN_ZIP_TAIL_BYTES);
+  const tailStart = finalSize - tailLen;
+  const tail = await readBytes(finalVolume, tailStart, tailLen);
+
+  const eocdPos = findEocdOffset(tail);
+  if (eocdPos < 0) return { kind: "failed", reason: "EOCD not found in final volume" };
+
+  let thisDisk = tail.readUInt16LE(eocdPos + 4);
+  let cdStartDisk = tail.readUInt16LE(eocdPos + 6);
+  let cdSize = tail.readUInt32LE(eocdPos + 12);
+  let cdOffset = tail.readUInt32LE(eocdPos + 16);
+
+  // ZIP64: any saturated field means the real values live in the ZIP64 EOCD
+  // record, which in a spanned archive may sit on a different volume.
+  if (
+    thisDisk === 0xffff ||
+    cdStartDisk === 0xffff ||
+    cdSize === 0xffffffff ||
+    cdOffset === 0xffffffff
+  ) {
+    const locPos = findZip64Locator(tail, eocdPos);
+    if (locPos < 0) return { kind: "failed", reason: "ZIP64 locator not found" };
+    const locDisk = tail.readUInt32LE(locPos + 4);
+    const locOffset = Number(tail.readBigUInt64LE(locPos + 8));
+
+    // The locator's disk number counts the final volume too, so resolve it
+    // through the same map, treating the final .zip as the highest volume.
+    const z64Path =
+      locDisk === thisDisk || locDisk === 0xffff ? finalVolume : volumes.get(locDisk) ?? finalVolume;
+    const z64 = await readBytes(z64Path, locOffset, 56);
+    if (z64.length < 56 || z64.readUInt32LE(0) !== ZIP64_EOCD_SIG) {
+      return { kind: "failed", reason: "ZIP64 EOCD record unreadable" };
+    }
+    thisDisk = z64.readUInt32LE(16);
+    cdStartDisk = z64.readUInt32LE(20);
+    cdSize = Number(z64.readBigUInt64LE(40));
+    cdOffset = Number(z64.readBigUInt64LE(48));
+  }
+
+  // A byte split named .z01/.zip still reports a single disk — concatenation,
+  // not volume mapping, is the correct reading for it.
+  if (thisDisk === 0 && cdStartDisk === 0) return { kind: "not-spanned" };
+
+  if (cdSize < 0 || cdSize > MAX_CD_BYTES) {
+    return { kind: "failed", reason: `implausible central directory size ${cdSize}` };
+  }
+
+  // The final .zip file is the highest-numbered volume.
+  volumes.set(thisDisk, finalVolume);
+
+  // Read cdSize bytes from (cdStartDisk, cdOffset), spilling onto later
+  // volumes if the directory straddles a volume boundary.
+  const chunks: Buffer[] = [];
+  let remaining = cdSize;
+  let disk = cdStartDisk;
+  let offset = cdOffset;
+  while (remaining > 0) {
+    const volPath = volumes.get(disk);
+    if (!volPath) return { kind: "failed", reason: `volume ${disk + 1} of the set is missing` };
+    const chunk = await readBytes(volPath, offset, remaining);
+    if (chunk.length === 0) {
+      return { kind: "failed", reason: `volume ${disk + 1} ended before the central directory did` };
+    }
+    chunks.push(chunk);
+    remaining -= chunk.length;
+    disk++;
+    offset = 0;
+  }
+
+  const cdBuf = Buffer.concat(chunks);
+  const entries = walkCentralDirectory(cdBuf, 0, cdSize);
+  log.debug(
+    { volumes: volumes.size, thisDisk, cdStartDisk, cdSize, entries: entries.length },
+    "Read spanned ZIP central directory"
+  );
+  return { kind: "entries", entries };
+}
+
+/** Find the ZIP64 EOCD locator, which sits just before the EOCD record. */
+function findZip64Locator(tail: Buffer, eocdPos: number): number {
+  for (let i = Math.min(eocdPos - 20, tail.length - 20); i >= 0; i--) {
+    if (tail.readUInt32LE(i) === ZIP64_LOCATOR_SIG) return i;
+  }
+  return -1;
+}
+
+/** Read up to `length` bytes at `offset`; a short read means end of file. */
+async function readBytes(filePath: string, offset: number, length: number): Promise<Buffer> {
+  const fh = await fsOpen(filePath, "r");
+  try {
+    const buf = Buffer.alloc(length);
+    const { bytesRead } = await fh.read(buf, 0, length, offset);
+    return buf.subarray(0, bytesRead);
+  } finally {
+    await fh.close();
+  }
+}

worker/src/provenance-backfill.ts

@@ -100,7 +100,7 @@ export async function resolveCandidateFingerprintEntries(
     if (destParts) {
       const read = tdlibRangeReader(client);
       destEntries =
-        candidate.archiveType === "ZIP" ? await readScannedZipListing(client, destParts)
+        candidate.archiveType === "ZIP" ? await readScannedZipListing(destParts, read)
         : candidate.archiveType === "SEVEN_Z" ? await readSevenZListingRanged(destParts, read)
         : candidate.archiveType === "RAR" ? await readRarListingRanged(destParts, read)
         : null;