merge: read ZIP-spec spanned archives instead of failing silently

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

worker/src/archive/central-directory.ts

@@ -11,13 +11,21 @@ function extOf(name: string): string | null {
   return e === "" ? null : e;
 }
 
+/**
+ * Locate the End Of Central Directory record by scanning backward for its
+ * signature. Returns -1 when the buffer holds no EOCD.
+ */
+export function findEocdOffset(tail: Buffer): number {
+  for (let i = tail.length - 22; i >= 0; i--) {
+    if (tail.readUInt32LE(i) === EOCD_SIG) return i;
+  }
+  return -1;
+}
+
 /** Parse a ZIP central directory from the tail of an archive. */
 export function parseZipCentralDirectoryFromTail(tail: Buffer, tailStart: number): FileEntry[] {
   // 1. Find EOCD by scanning backward for its signature.
-  let eocd = -1;
-  for (let i = tail.length - 22; i >= 0; i--) {
-    if (tail.readUInt32LE(i) === EOCD_SIG) { eocd = i; break; }
-  }
+  const eocd = findEocdOffset(tail);
   if (eocd < 0) throw new RangeError("EOCD not found in tail");
 
   let cdSize = tail.readUInt32LE(eocd + 12);
@@ -45,10 +53,22 @@ export function parseZipCentralDirectoryFromTail(tail: Buffer, tailStart: number
   }
 
   // 3. Walk central-directory headers.
+  return walkCentralDirectory(tail, cdLocal, cdSize);
+}
+
+/**
+ * Walk a run of central-directory file headers and return the file entries
+ * (directory entries are skipped).
+ *
+ * `buf` must contain the whole central directory starting at `start`; for a
+ * spanned archive that means the caller has already stitched together the
+ * volumes the directory straddles.
+ */
+export function walkCentralDirectory(tail: Buffer, start: number, cdSize: number): FileEntry[] {
   const entries: FileEntry[] = [];
-  let p = cdLocal;
-  const end = cdLocal + cdSize;
-  while (p + 46 <= end && tail.readUInt32LE(p) === CD_SIG) {
+  let p = start;
+  const end = start + cdSize;
+  while (p + 46 <= end && p + 46 <= tail.length && tail.readUInt32LE(p) === CD_SIG) {
     let crc = tail.readUInt32LE(p + 16) >>> 0;
     let comp = BigInt(tail.readUInt32LE(p + 20));
     let uncomp = BigInt(tail.readUInt32LE(p + 24));

worker/src/archive/ranged/dispatch.test.ts

@@ -1,15 +1,103 @@
 import { describe, it, expect } from "vitest";
-import { readScannedListingRanged } from "./dispatch.js";
+import { readScannedListingRanged, readScannedZipListing } from "./dispatch.js";
+import type { RangeReader } from "./range-reader.js";
+import { buildSpannedStoreZip, buildStoreZip, byteSplit } from "../testing/spanned-zip-fixture.js";
+
+/** Serve ranged reads out of in-memory part buffers keyed by fileId. */
+function readerFor(parts: { fileId: string; buf: Buffer }[]): RangeReader {
+  const byId = new Map(parts.map((p) => [p.fileId, p.buf]));
+  return async (fileId, offset, length) => {
+    const buf = byId.get(fileId);
+    if (!buf) throw new Error(`unknown fileId ${fileId}`);
+    return buf.subarray(offset, offset + length);
+  };
+}
+
+const FILES = [
+  { name: "src/b.bin", data: Buffer.alloc(2048, 7), disk: 0 },
+  { name: "src/models/dragon.stl", data: Buffer.from("DRAGON"), disk: 1 },
+  { name: "readme.txt", data: Buffer.from("hello world"), disk: 2 },
+];
 
 describe("readScannedListingRanged", () => {
   it("returns null for an unknown archive type without calling the reader", async () => {
-    const read = async () => Buffer.alloc(0);
     const result = await readScannedListingRanged(
       "DOCUMENT",
       { invoke: async () => ({}) } as never,
       [{ fileId: "1", fileSize: 100n, fileName: "a.pdf" }],
     );
     expect(result).toBeNull();
-    void read; // unused placeholder kept out of the dispatch call — DOCUMENT never reaches a reader
+  });
+});
+
+describe("readScannedZipListing", () => {
+  it("lists a ZIP-spec spanned set (.z01 … .zip) from the final volume's tail", async () => {
+    const vols = buildSpannedStoreZip(FILES, 3);
+    const names = ["Pack.z01", "Pack.z02", "Pack.zip"];
+    const parts = vols.map((buf, i) => ({ fileId: String(i), fileSize: BigInt(buf.length), fileName: names[i] }));
+
+    const entries = await readScannedZipListing(
+      parts,
+      readerFor(parts.map((p, i) => ({ fileId: p.fileId, buf: vols[i] }))),
+    );
+
+    expect(entries).not.toBeNull();
+    expect(entries!.map((e) => e.path).sort()).toEqual([
+      "readme.txt",
+      "src/b.bin",
+      "src/models/dragon.stl",
+    ]);
+  });
+
+  it("only reads the final volume — the earlier volumes are never downloaded", async () => {
+    const vols = buildSpannedStoreZip(FILES, 3);
+    const names = ["Pack.z01", "Pack.z02", "Pack.zip"];
+    const parts = vols.map((buf, i) => ({ fileId: String(i), fileSize: BigInt(buf.length), fileName: names[i] }));
+    const touched: string[] = [];
+    const base = readerFor(parts.map((p, i) => ({ fileId: p.fileId, buf: vols[i] })));
+
+    await readScannedZipListing(parts, async (id, off, len, size) => {
+      touched.push(id);
+      return base(id, off, len, size);
+    });
+
+    expect([...new Set(touched)]).toEqual(["2"]);
+  });
+
+  it("still lists a 7-Zip raw byte split (.zip.001 …) using whole-archive offsets", async () => {
+    const zip = buildStoreZip([
+      { name: "models/knight.stl", data: Buffer.alloc(5000, 9) },
+      { name: "license.txt", data: Buffer.from("MIT") },
+    ]);
+    const chunks = byteSplit(zip, 3);
+    const names = ["Pack.zip.001", "Pack.zip.002", "Pack.zip.003"];
+    const parts = chunks.map((buf, i) => ({ fileId: String(i), fileSize: BigInt(buf.length), fileName: names[i] }));
+
+    const entries = await readScannedZipListing(
+      parts,
+      readerFor(parts.map((p, i) => ({ fileId: p.fileId, buf: chunks[i] }))),
+    );
+
+    expect(entries!.map((e) => e.fileName).sort()).toEqual(["knight.stl", "license.txt"]);
+  });
+
+  it("returns null for a spanned set whose central directory starts on an earlier volume", async () => {
+    const vols = buildSpannedStoreZip(
+      FILES.map((f) => ({ ...f, disk: Math.min(f.disk, 1) })),
+      3,
+      { cdStartDisk: 1 },
+    );
+    const names = ["Pack.z01", "Pack.z02", "Pack.zip"];
+    const parts = vols.map((buf, i) => ({ fileId: String(i), fileSize: BigInt(buf.length), fileName: names[i] }));
+
+    const entries = await readScannedZipListing(
+      parts,
+      readerFor(parts.map((p, i) => ({ fileId: p.fileId, buf: vols[i] }))),
+    );
+    expect(entries).toBeNull();
+  });
+
+  it("returns null when there are no parts", async () => {
+    expect(await readScannedZipListing([], readerFor([]))).toBeNull();
   });
 });

worker/src/archive/ranged/dispatch.ts

@@ -1,11 +1,11 @@
 import type { Client } from "tdl";
-import { downloadFileRange } from "../../tdlib/range-download.js";
-import { parseZipCentralDirectoryFromTail, MIN_ZIP_TAIL_BYTES } from "../central-directory.js";
+import { parseZipCentralDirectoryFromTail, findEocdOffset, MIN_ZIP_TAIL_BYTES } from "../central-directory.js";
+import { isSpannedZipPartSet } from "../zip-spanned.js";
 import { childLogger } from "../../util/logger.js";
 import type { FileEntry } from "../zip-reader.js";
 import { readSevenZListingRanged, type RangedPart } from "./sevenz-ranged.js";
 import { readRarListingRanged } from "./rar-ranged.js";
-import { tdlibRangeReader } from "./range-reader.js";
+import { tdlibRangeReader, type RangeReader } from "./range-reader.js";
 
 const log = childLogger("ranged-dispatch");
 
@@ -13,25 +13,45 @@ const log = childLogger("ranged-dispatch");
  * Read a ZIP central directory from the tail of a (possibly multipart)
  * archive. `parts` is ordered; only the LAST part carries the EOCD record.
  * `fileSize` on each part is that part's own size (NOT the whole-archive
- * total) so the download offset stays within that part's bounds, while
- * `tailStart` passed to the parser is the logical whole-archive offset
- * (preceding parts' sizes + the offset within the last part).
+ * total) so the download offset stays within that part's bounds.
+ *
+ * Which logical offset the EOCD's central-directory pointer is measured from
+ * depends on the multipart shape:
+ *
+ *  - 7-Zip raw byte split (`.zip.001`, …): the parts are one ZIP file cut into
+ *    chunks, so the pointer is a whole-archive offset → `tailStart` is the
+ *    preceding parts' sizes plus the offset within the last part.
+ *  - ZIP-spec spanned archive (`.z01`, …, `.zip`): each volume is its own unit
+ *    and the pointer is relative to the start of the volume holding the
+ *    directory → `tailStart` is just the offset within that final volume.
+ *
+ * Getting this wrong makes the computed directory offset wildly negative, and
+ * the parser then throws RangeError on every tail size — which is exactly how
+ * spanned sets ended up indexed with no file list at all.
  */
 export async function readScannedZipListing(
-  client: Client,
-  parts: { fileId: string; fileSize: bigint }[],
+  parts: RangedPart[],
+  read: RangeReader,
 ): Promise<FileEntry[] | null> {
   if (parts.length === 0) return null;
   const lastPart = parts[parts.length - 1];
-  const precedingSize = parts.slice(0, -1).reduce((sum, p) => sum + Number(p.fileSize), 0);
+  const spanned = isSpannedZipPartSet(parts.map((p) => p.fileName));
+  const precedingSize = spanned
+    ? 0
+    : parts.slice(0, -1).reduce((sum, p) => sum + Number(p.fileSize), 0);
   const lastSize = Number(lastPart.fileSize);
   for (const tailBytes of [MIN_ZIP_TAIL_BYTES, MIN_ZIP_TAIL_BYTES * 4]) {
     const partOffset = Math.max(0, lastSize - tailBytes);
     const downloadLen = Math.min(tailBytes, lastSize);
     try {
-      const buf = await downloadFileRange(client, lastPart.fileId, partOffset, downloadLen, lastPart.fileSize);
-      const tailStart = precedingSize + partOffset;
-      return parseZipCentralDirectoryFromTail(buf, tailStart);
+      const buf = await read(lastPart.fileId, partOffset, downloadLen, lastPart.fileSize);
+      if (spanned && !cdStartsOnFinalVolume(buf)) {
+        // The directory begins on an earlier volume; reaching it would mean
+        // ranged-reading that volume too. Leave it to the full-download path.
+        log.debug({ fileId: lastPart.fileId }, "spanned ZIP central directory is not on the final volume");
+        return null;
+      }
+      return parseZipCentralDirectoryFromTail(buf, partOffset + precedingSize);
     } catch (err) {
       if (err instanceof RangeError) continue; // try a larger tail
       log.warn({ err, fileId: lastPart.fileId }, "ranged ZIP listing failed");
@@ -41,6 +61,20 @@ export async function readScannedZipListing(
   return null;
 }
 
+/**
+ * For a spanned archive, whether the EOCD says the central directory starts on
+ * the very volume that EOCD lives on (the usual case). ZIP64's saturated
+ * 0xFFFF disk fields are treated as "yes" — the ZIP64 record that supersedes
+ * them is itself in this tail, and the parser resolves it there.
+ */
+function cdStartsOnFinalVolume(tail: Buffer): boolean {
+  const eocd = findEocdOffset(tail);
+  if (eocd < 0) return true; // let the parser report the real problem
+  const thisDisk = tail.readUInt16LE(eocd + 4);
+  const cdStartDisk = tail.readUInt16LE(eocd + 6);
+  return thisDisk === cdStartDisk || thisDisk === 0xffff || cdStartDisk === 0xffff;
+}
+
 /**
  * Dispatch a (no-download) inner-file listing read by archive type. Used both
  * by the provenance-backfill path (reading an already-uploaded copy) and the
@@ -55,7 +89,7 @@ export async function readScannedListingRanged(
   parts: RangedPart[],
 ): Promise<FileEntry[] | null> {
   const read = tdlibRangeReader(client);
-  if (archiveType === "ZIP") return readScannedZipListing(client, parts);
+  if (archiveType === "ZIP") return readScannedZipListing(parts, read);
   if (archiveType === "SEVEN_Z") return readSevenZListingRanged(parts, read);
   if (archiveType === "RAR") return readRarListingRanged(parts, read);
   return null;

worker/src/archive/testing/spanned-zip-fixture.ts

@@ -0,0 +1,191 @@
+import { crc32 } from "zlib"; // Node 20+ exposes zlib.crc32
+
+/**
+ * Test-only builders that emit real ZIP byte streams.
+ *
+ * Two distinct on-disk shapes are produced here, because the worker has to
+ * tell them apart:
+ *
+ *  - `buildSpannedStoreZip` → a ZIP-spec **spanned/multi-disk** archive
+ *    (`Pack.z01`, `Pack.z02`, …, `Pack.zip`). Each volume is its own file;
+ *    central-directory records carry a disk number, and the EOCD's
+ *    "offset of start of central directory" is relative to the *start of the
+ *    disk that holds it*, not to a concatenation of the volumes.
+ *
+ *  - `buildStoreZip` → an ordinary single-file ZIP. Cutting its bytes into
+ *    chunks yields the 7-Zip raw byte split shape (`Pack.zip.001`, …), where
+ *    all disk numbers are 0 and offsets are whole-archive absolute.
+ *
+ * Field layouts follow APPNOTE 6.3.x sections 4.3.12 (central directory) and
+ * 4.3.16 (EOCD). Verified against Info-ZIP `zip -s` output.
+ */
+
+const LOCAL_SIG = 0x04034b50;
+const CD_SIG = 0x02014b50;
+const EOCD_SIG = 0x06054b50;
+/** APPNOTE 8.5.3: the first volume of a spanned archive starts with this. */
+const SPANNING_SIG = 0x08074b50;
+
+export interface FixtureFile {
+  name: string;
+  data: Buffer;
+  /** 0-based volume this file's local header + data is written to. */
+  disk?: number;
+}
+
+function localHeader(name: string, data: Buffer): Buffer {
+  const nameBuf = Buffer.from(name, "utf8");
+  const local = Buffer.alloc(30);
+  local.writeUInt32LE(LOCAL_SIG, 0);
+  local.writeUInt16LE(20, 4); // version needed
+  local.writeUInt16LE(0, 6); // flags
+  local.writeUInt16LE(0, 8); // method = store
+  local.writeUInt32LE(crc32(data) >>> 0, 14);
+  local.writeUInt32LE(data.length, 18); // compressed
+  local.writeUInt32LE(data.length, 22); // uncompressed
+  local.writeUInt16LE(nameBuf.length, 26);
+  local.writeUInt16LE(0, 28); // extra len
+  return Buffer.concat([local, nameBuf, data]);
+}
+
+function centralHeader(name: string, data: Buffer, diskStart: number, relOffset: number): Buffer {
+  const nameBuf = Buffer.from(name, "utf8");
+  const cd = Buffer.alloc(46);
+  cd.writeUInt32LE(CD_SIG, 0);
+  cd.writeUInt16LE(20, 4); // version made by
+  cd.writeUInt16LE(20, 6); // version needed
+  cd.writeUInt16LE(0, 8); // flags
+  cd.writeUInt16LE(0, 10); // method = store
+  cd.writeUInt32LE(crc32(data) >>> 0, 16);
+  cd.writeUInt32LE(data.length, 20); // compressed
+  cd.writeUInt32LE(data.length, 24); // uncompressed
+  cd.writeUInt16LE(nameBuf.length, 28);
+  cd.writeUInt16LE(diskStart, 34); // disk number start
+  cd.writeUInt32LE(relOffset, 42); // offset of local header, relative to its disk
+  return Buffer.concat([cd, nameBuf]);
+}
+
+function eocd(opts: {
+  thisDisk: number;
+  cdStartDisk: number;
+  entriesThisDisk: number;
+  entriesTotal: number;
+  cdSize: number;
+  cdOffset: number;
+}): Buffer {
+  const buf = Buffer.alloc(22);
+  buf.writeUInt32LE(EOCD_SIG, 0);
+  buf.writeUInt16LE(opts.thisDisk, 4);
+  buf.writeUInt16LE(opts.cdStartDisk, 6);
+  buf.writeUInt16LE(opts.entriesThisDisk, 8);
+  buf.writeUInt16LE(opts.entriesTotal, 10);
+  buf.writeUInt32LE(opts.cdSize, 12);
+  buf.writeUInt32LE(opts.cdOffset, 16);
+  return buf;
+}
+
+/** Build an ordinary single-file STORE ZIP (all disk numbers 0). */
+export function buildStoreZip(files: FixtureFile[]): Buffer {
+  const body: Buffer[] = [];
+  const central: Buffer[] = [];
+  let offset = 0;
+  for (const f of files) {
+    const local = localHeader(f.name, f.data);
+    body.push(local);
+    central.push(centralHeader(f.name, f.data, 0, offset));
+    offset += local.length;
+  }
+  const cdBuf = Buffer.concat(central);
+  return Buffer.concat([
+    ...body,
+    cdBuf,
+    eocd({
+      thisDisk: 0,
+      cdStartDisk: 0,
+      entriesThisDisk: files.length,
+      entriesTotal: files.length,
+      cdSize: cdBuf.length,
+      cdOffset: offset,
+    }),
+  ]);
+}
+
+/**
+ * Build a ZIP-spec spanned archive as one Buffer per volume.
+ * Returned array is volume order: [z01, z02, …, zip] (last element is the
+ * final volume, which carries the central directory and EOCD).
+ *
+ * `cdStartDisk` (default: last volume) lets a test place the central
+ * directory so that it begins on an earlier volume and spills forward,
+ * exercising the cross-volume read path. As in a real writer, no file data
+ * may live on a volume after the one the directory starts on — those volumes
+ * hold directory continuation only.
+ */
+export function buildSpannedStoreZip(
+  files: FixtureFile[],
+  totalDisks: number,
+  opts: { cdStartDisk?: number } = {}
+): Buffer[] {
+  const cdStart = opts.cdStartDisk ?? totalDisks - 1;
+  for (const f of files) {
+    if ((f.disk ?? 0) > cdStart) {
+      throw new Error(`fixture misuse: ${f.name} is on volume ${f.disk} but the CD starts on ${cdStart}`);
+    }
+    if ((f.disk ?? 0) >= totalDisks) {
+      throw new Error(`fixture misuse: ${f.name} is on volume ${f.disk} of ${totalDisks}`);
+    }
+  }
+  const chunks: Buffer[][] = Array.from({ length: totalDisks }, () => []);
+  const lengths = new Array<number>(totalDisks).fill(0);
+
+  const marker = Buffer.alloc(4);
+  marker.writeUInt32LE(SPANNING_SIG, 0);
+  chunks[0].push(marker);
+  lengths[0] = 4;
+
+  const central: Buffer[] = [];
+  for (const f of files) {
+    const disk = f.disk ?? 0;
+    const local = localHeader(f.name, f.data);
+    central.push(centralHeader(f.name, f.data, disk, lengths[disk]));
+    chunks[disk].push(local);
+    lengths[disk] += local.length;
+  }
+
+  const cdBuf = Buffer.concat(central);
+  const lastDisk = totalDisks - 1;
+  const cdStartDisk = cdStart;
+  const cdOffset = lengths[cdStartDisk];
+
+  // Write the CD starting on cdStartDisk, spilling onto later volumes.
+  const spillDisks = lastDisk - cdStartDisk + 1;
+  const firstChunkLen = Math.ceil(cdBuf.length / spillDisks);
+  let written = 0;
+  for (let d = cdStartDisk; d <= lastDisk; d++) {
+    const take = d === lastDisk ? cdBuf.length - written : Math.min(firstChunkLen, cdBuf.length - written);
+    chunks[d].push(cdBuf.subarray(written, written + take));
+    lengths[d] += take;
+    written += take;
+  }
+
+  chunks[lastDisk].push(
+    eocd({
+      thisDisk: lastDisk,
+      cdStartDisk,
+      entriesThisDisk: files.length,
+      entriesTotal: files.length,
+      cdSize: cdBuf.length,
+      cdOffset,
+    })
+  );
+
+  return chunks.map((c) => Buffer.concat(c));
+}
+
+/** Cut a buffer into `count` roughly equal chunks (7-Zip raw byte split). */
+export function byteSplit(buf: Buffer, count: number): Buffer[] {
+  const size = Math.ceil(buf.length / count);
+  const out: Buffer[] = [];
+  for (let i = 0; i < buf.length; i += size) out.push(buf.subarray(i, i + size));
+  return out;
+}

worker/src/archive/zip-reader.ts

@@ -3,6 +3,7 @@ import { open as fsOpen, stat as fsStat } from "fs/promises";
 import path from "path";
 import { Readable } from "stream";
 import { childLogger } from "../util/logger.js";
+import { isSpannedZipPartSet, readSpannedZipCentralDirectory } from "./zip-spanned.js";
 
 const log = childLogger("zip-reader");
 
@@ -17,9 +18,15 @@ export interface FileEntry {
 
 /**
  * Read the central directory of a ZIP file without extracting any contents.
- * For multipart ZIPs (.zip.001, .zip.002 etc.), uses a custom random-access
- * reader that spans all parts seamlessly so yauzl can find the central
- * directory at the end of the combined data.
+ *
+ * Three shapes are handled:
+ *  - a single `.zip` → yauzl directly;
+ *  - a 7-Zip raw byte split (`.zip.001`, `.zip.002`, …), which is one ZIP file
+ *    cut into chunks → a random-access reader that spans the chunks so yauzl
+ *    sees the combined stream;
+ *  - a ZIP-spec spanned/multi-disk archive (`.z01`, `.z02`, …, `.zip`), a
+ *    different on-disk format that yauzl refuses outright → a dedicated
+ *    volume-aware central-directory reader.
  */
 export async function readZipCentralDirectory(
   filePaths: string[]
@@ -28,7 +35,22 @@ export async function readZipCentralDirectory(
     return readSingleZip(filePaths[0]);
   }
 
-  // Multipart: use a spanning random-access reader
+  if (isSpannedZipPartSet(filePaths)) {
+    try {
+      const result = await readSpannedZipCentralDirectory(filePaths);
+      if (result.kind === "entries") return result.entries;
+      if (result.kind === "failed") {
+        log.warn({ reason: result.reason, parts: filePaths.length }, "Failed to read spanned ZIP");
+        return [];
+      }
+      // "not-spanned": named like volumes but really a byte split — fall through.
+    } catch (err) {
+      log.warn({ err, parts: filePaths.length }, "Failed to read spanned ZIP");
+      return [];
+    }
+  }
+
+  // Multipart byte split: use a spanning random-access reader
   return readMultipartZip(filePaths);
 }
 

worker/src/archive/zip-spanned.test.ts

@@ -0,0 +1,214 @@
+import { describe, it, expect, beforeAll, afterAll } from "vitest";
+import { mkdir, mkdtemp, readdir, rm, writeFile } from "fs/promises";
+import { execFile } from "child_process";
+import { promisify } from "util";
+import { tmpdir } from "os";
+import path from "path";
+import { isSpannedZipPartSet } from "./zip-spanned.js";
+import { readZipCentralDirectory } from "./zip-reader.js";
+import { buildSpannedStoreZip, buildStoreZip, byteSplit } from "./testing/spanned-zip-fixture.js";
+
+const execFileAsync = promisify(execFile);
+
+let dir: string;
+
+beforeAll(async () => {
+  dir = await mkdtemp(path.join(tmpdir(), "zip-spanned-"));
+});
+
+afterAll(async () => {
+  await rm(dir, { recursive: true, force: true });
+});
+
+/** Write volume buffers out under the given names and return their paths. */
+async function writeParts(names: string[], buffers: Buffer[], sub: string): Promise<string[]> {
+  const base = path.join(dir, sub);
+  await rm(base, { recursive: true, force: true });
+  await mkdir(base, { recursive: true });
+  const paths: string[] = [];
+  for (let i = 0; i < names.length; i++) {
+    const p = path.join(base, names[i]);
+    await writeFile(p, buffers[i]);
+    paths.push(p);
+  }
+  return paths;
+}
+
+const FILES = [
+  { name: "src/b.bin", data: Buffer.alloc(2048, 7), disk: 0 },
+  { name: "src/models/dragon.stl", data: Buffer.from("DRAGON"), disk: 1 },
+  { name: "src/models/", data: Buffer.alloc(0), disk: 1 },
+  { name: "src/a.bin", data: Buffer.alloc(4096, 3), disk: 2 },
+  { name: "readme.txt", data: Buffer.from("hello world"), disk: 3 },
+];
+
+describe("isSpannedZipPartSet", () => {
+  it("recognizes a .z01 + .zip volume set", () => {
+    expect(isSpannedZipPartSet(["/t/Pack.z01", "/t/Pack.z02", "/t/Pack.zip"])).toBe(true);
+  });
+
+  it("recognizes the set regardless of the order it is handed in", () => {
+    expect(isSpannedZipPartSet(["/t/Pack.zip", "/t/Pack.z02", "/t/Pack.z01"])).toBe(true);
+  });
+
+  it("rejects a 7-Zip raw byte split (.zip.001)", () => {
+    expect(isSpannedZipPartSet(["/t/Pack.zip.001", "/t/Pack.zip.002"])).toBe(false);
+  });
+
+  it("rejects a single .zip", () => {
+    expect(isSpannedZipPartSet(["/t/Pack.zip"])).toBe(false);
+  });
+
+  it("rejects a set with no final .zip volume", () => {
+    expect(isSpannedZipPartSet(["/t/Pack.z01", "/t/Pack.z02"])).toBe(false);
+  });
+
+  it("rejects a set with two .zip volumes", () => {
+    expect(isSpannedZipPartSet(["/t/A.zip", "/t/B.zip"])).toBe(false);
+  });
+});
+
+describe("readZipCentralDirectory on a spanned (.z01 + .zip) set", () => {
+  it("lists every entry with correct paths, sizes and crc32", async () => {
+    const vols = buildSpannedStoreZip(FILES, 4);
+    const paths = await writeParts(["Pack.z01", "Pack.z02", "Pack.z03", "Pack.zip"], vols, "spanned");
+
+    const entries = await readZipCentralDirectory(paths);
+
+    expect(entries.map((e) => e.path).sort()).toEqual([
+      "readme.txt",
+      "src/a.bin",
+      "src/b.bin",
+      "src/models/dragon.stl",
+    ]);
+    const dragon = entries.find((e) => e.fileName === "dragon.stl")!;
+    expect(dragon.uncompressedSize).toBe(6n);
+    expect(dragon.extension).toBe("stl");
+    expect(dragon.crc32).toMatch(/^[0-9a-f]{8}$/);
+    const a = entries.find((e) => e.fileName === "a.bin")!;
+    expect(a.uncompressedSize).toBe(4096n);
+    expect(a.compressedSize).toBe(4096n);
+  });
+
+  it("works when the parts are handed over out of volume order", async () => {
+    const vols = buildSpannedStoreZip(FILES, 4);
+    const paths = await writeParts(["Pack.z01", "Pack.z02", "Pack.z03", "Pack.zip"], vols, "unordered");
+    const shuffled = [paths[3], paths[1], paths[0], paths[2]];
+
+    const entries = await readZipCentralDirectory(shuffled);
+    expect(entries).toHaveLength(4);
+  });
+
+  it("reads a central directory that begins on an earlier volume and spills forward", async () => {
+    // Files live on volumes 0–2; the directory starts on volume 2 and
+    // continues onto the final .zip, which holds nothing else.
+    const spillFiles = FILES.map((f) => ({ ...f, disk: Math.min(f.disk, 2) }));
+    const vols = buildSpannedStoreZip(spillFiles, 4, { cdStartDisk: 2 });
+    const paths = await writeParts(["Pack.z01", "Pack.z02", "Pack.z03", "Pack.zip"], vols, "spilled");
+
+    const entries = await readZipCentralDirectory(paths);
+    expect(entries.map((e) => e.fileName).sort()).toEqual(["a.bin", "b.bin", "dragon.stl", "readme.txt"]);
+  });
+
+  it("returns [] instead of throwing when a needed volume is missing", async () => {
+    const spillFiles = FILES.map((f) => ({ ...f, disk: Math.min(f.disk, 1) }));
+    const vols = buildSpannedStoreZip(spillFiles, 3, { cdStartDisk: 1 });
+    const paths = await writeParts(["Pack.z01", "Pack.z02", "Pack.zip"], vols, "missing");
+    // Drop Pack.z02 — the volume the central directory starts on.
+    const entries = await readZipCentralDirectory([paths[0], paths[2]]);
+    expect(entries).toEqual([]);
+  });
+
+  it("returns [] instead of throwing when the final volume is garbage", async () => {
+    const vols = buildSpannedStoreZip([{ name: "x.stl", data: Buffer.alloc(64, 1), disk: 0 }], 2);
+    const paths = await writeParts(["Pack.z01", "Pack.zip"], [vols[0], Buffer.alloc(500, 0x5a)], "garbage");
+    expect(await readZipCentralDirectory(paths)).toEqual([]);
+  });
+
+  it("falls back to concatenation semantics when .z01-named parts are really a byte split", async () => {
+    // Some producers name a raw byte split .z01/.zip. The EOCD then reports
+    // disk 0, so concatenation — not volume mapping — is the correct reading.
+    const zip = buildStoreZip([
+      { name: "one.stl", data: Buffer.alloc(3000, 1) },
+      { name: "two.stl", data: Buffer.alloc(3000, 2) },
+    ]);
+    const paths = await writeParts(["Pack.z01", "Pack.zip"], byteSplit(zip, 2), "mislabeled");
+    const entries = await readZipCentralDirectory(paths);
+    expect(entries.map((e) => e.fileName).sort()).toEqual(["one.stl", "two.stl"]);
+  });
+});
+
+describe("readZipCentralDirectory regressions for the shapes that already worked", () => {
+  it("still reads a 7-Zip raw byte split (.zip.001 …)", async () => {
+    const zip = buildStoreZip([
+      { name: "models/knight.stl", data: Buffer.alloc(5000, 9) },
+      { name: "license.txt", data: Buffer.from("MIT") },
+    ]);
+    const paths = await writeParts(
+      ["Pack.zip.001", "Pack.zip.002", "Pack.zip.003"],
+      byteSplit(zip, 3),
+      "bytesplit"
+    );
+    const entries = await readZipCentralDirectory(paths);
+    expect(entries.map((e) => e.fileName).sort()).toEqual(["knight.stl", "license.txt"]);
+    expect(entries.find((e) => e.fileName === "knight.stl")!.uncompressedSize).toBe(5000n);
+  });
+
+  it("still reads a plain single .zip", async () => {
+    const zip = buildStoreZip([{ name: "solo.stl", data: Buffer.from("SOLO") }]);
+    const paths = await writeParts(["Pack.zip"], [zip], "single");
+    const entries = await readZipCentralDirectory(paths);
+    expect(entries.map((e) => e.fileName)).toEqual(["solo.stl"]);
+  });
+});
+
+// ── Cross-checks against real Info-ZIP output ────────────────────────────
+// Skipped automatically where the `zip` CLI is unavailable; the hand-built
+// fixtures above are the authoritative, portable coverage.
+
+const HAS_ZIP_CLI = await execFileAsync("zip", ["-v"]).then(
+  () => true,
+  () => false
+);
+
+/** Collect Pack.z01 … Pack.zip from a directory, in volume order. */
+async function collectVolumes(work: string): Promise<string[]> {
+  const names = await readdir(work);
+  const vols = names.filter((n) => /^Pack\.z\d{2,}$/i.test(n)).sort();
+  const final = names.find((n) => /^Pack\.zip$/i.test(n));
+  expect(final).toBeDefined();
+  expect(vols.length).toBeGreaterThan(0);
+  return [...vols, final!].map((n) => path.join(work, n));
+}
+
+describe.skipIf(!HAS_ZIP_CLI)("readZipCentralDirectory against archives produced by Info-ZIP `zip -s`", () => {
+  it("lists a genuine spanned archive", async () => {
+    const work = path.join(dir, "real");
+    await mkdir(path.join(work, "src/models"), { recursive: true });
+    await writeFile(path.join(work, "src/big.bin"), Buffer.alloc(300_000, 4));
+    await writeFile(path.join(work, "src/models/dragon.stl"), Buffer.alloc(200_000, 5));
+    await writeFile(path.join(work, "src/readme.txt"), "hello world");
+    await execFileAsync("zip", ["-r", "-0", "-s", "100k", "Pack.zip", "src"], { cwd: work });
+
+    const entries = await readZipCentralDirectory(await collectVolumes(work));
+    expect(entries.map((e) => e.path).sort()).toEqual([
+      "src/big.bin",
+      "src/models/dragon.stl",
+      "src/readme.txt",
+    ]);
+    expect(entries.find((e) => e.fileName === "dragon.stl")!.uncompressedSize).toBe(200_000n);
+  });
+
+  it("lists a genuine ZIP64 spanned archive", async () => {
+    const work = path.join(dir, "real64");
+    await mkdir(work, { recursive: true });
+    await writeFile(path.join(work, "big.bin"), Buffer.alloc(300_000, 6));
+    await writeFile(path.join(work, "note.txt"), "zip64 spanned");
+    // -fz forces ZIP64 structures even though the payload is small.
+    await execFileAsync("zip", ["-0", "-fz", "-s", "100k", "Pack.zip", "big.bin", "note.txt"], { cwd: work });
+
+    const entries = await readZipCentralDirectory(await collectVolumes(work));
+    expect(entries.map((e) => e.fileName).sort()).toEqual(["big.bin", "note.txt"]);
+    expect(entries.find((e) => e.fileName === "big.bin")!.uncompressedSize).toBe(300_000n);
+  });
+});

worker/src/archive/zip-spanned.ts

@@ -0,0 +1,170 @@
+import { open as fsOpen, stat as fsStat } from "fs/promises";
+import path from "path";
+import { findEocdOffset, walkCentralDirectory, MIN_ZIP_TAIL_BYTES } from "./central-directory.js";
+import { childLogger } from "../util/logger.js";
+import type { FileEntry } from "./zip-reader.js";
+
+const log = childLogger("zip-spanned");
+
+const ZIP64_LOCATOR_SIG = 0x07064b50;
+const ZIP64_EOCD_SIG = 0x06064b50;
+
+/** Refuse to allocate a buffer for an absurd central-directory size. */
+const MAX_CD_BYTES = 256 * 1024 * 1024;
+
+/**
+ * A `.z01`/`.z02`/…/`.zip` set is a ZIP-spec **spanned (multi-disk)** archive:
+ * a genuinely different on-disk format from a 7-Zip raw byte split
+ * (`.zip.001`, `.zip.002`, …), which is one ZIP file cut into chunks.
+ *
+ * The distinction matters because a byte split is read by concatenating the
+ * chunks, whereas in a spanned archive each volume is its own unit: the EOCD's
+ * central-directory offset is relative to the start of the volume that holds
+ * the directory, and central-directory records carry a volume number. Feeding
+ * a spanned set to a concatenating reader yields nonsense offsets (and yauzl
+ * refuses outright: "multi-disk zip files are not supported").
+ *
+ * Detected from filename shape rather than the detector's multipart `pattern`
+ * so this stays independent of how `detect.ts` labels the two variants.
+ * Order-independent: the caller may hand the volumes over in any order.
+ */
+export function isSpannedZipPartSet(filePaths: string[]): boolean {
+  if (filePaths.length < 2) return false;
+  const names = filePaths.map((p) => path.basename(p));
+  const finals = names.filter((n) => /\.zip$/i.test(n));
+  const volumes = names.filter((n) => /\.z\d{2,}$/i.test(n));
+  return finals.length === 1 && volumes.length === names.length - 1;
+}
+
+export type SpannedZipResult =
+  /** Successfully read; `entries` may legitimately be empty for an empty archive. */
+  | { kind: "entries"; entries: FileEntry[] }
+  /** The EOCD reports a single disk — the parts are really a byte split, so the
+   *  caller should fall back to reading them as one concatenated stream. */
+  | { kind: "not-spanned" }
+  | { kind: "failed"; reason: string };
+
+/**
+ * Read the central directory of a ZIP-spec spanned archive.
+ *
+ * Only the volume holding the central directory (plus any it spills onto) is
+ * read, and only the directory bytes themselves — the file payloads are never
+ * touched, so this is cheap regardless of archive size.
+ *
+ * Coverage: STORE/DEFLATE and ZIP64 spanned archives are handled. Archives
+ * whose central directory is itself encrypted (strong encryption / "hide
+ * filenames") cannot be listed by any header-only reader and return `failed`.
+ */
+export async function readSpannedZipCentralDirectory(filePaths: string[]): Promise<SpannedZipResult> {
+  const volumes = new Map<number, string>();
+  let finalVolume: string | undefined;
+  for (const p of filePaths) {
+    const base = path.basename(p);
+    const m = base.match(/\.z(\d{2,})$/i);
+    if (m) {
+      // .z01 is volume 0, .z02 is volume 1, … (APPNOTE numbers disks from 0).
+      volumes.set(parseInt(m[1], 10) - 1, p);
+    } else if (/\.zip$/i.test(base)) {
+      finalVolume = p;
+    }
+  }
+  if (!finalVolume) return { kind: "failed", reason: "no final .zip volume" };
+
+  const finalSize = (await fsStat(finalVolume)).size;
+  const tailLen = Math.min(finalSize, MIN_ZIP_TAIL_BYTES);
+  const tailStart = finalSize - tailLen;
+  const tail = await readBytes(finalVolume, tailStart, tailLen);
+
+  const eocdPos = findEocdOffset(tail);
+  if (eocdPos < 0) return { kind: "failed", reason: "EOCD not found in final volume" };
+
+  let thisDisk = tail.readUInt16LE(eocdPos + 4);
+  let cdStartDisk = tail.readUInt16LE(eocdPos + 6);
+  let cdSize = tail.readUInt32LE(eocdPos + 12);
+  let cdOffset = tail.readUInt32LE(eocdPos + 16);
+
+  // ZIP64: any saturated field means the real values live in the ZIP64 EOCD
+  // record, which in a spanned archive may sit on a different volume.
+  if (
+    thisDisk === 0xffff ||
+    cdStartDisk === 0xffff ||
+    cdSize === 0xffffffff ||
+    cdOffset === 0xffffffff
+  ) {
+    const locPos = findZip64Locator(tail, eocdPos);
+    if (locPos < 0) return { kind: "failed", reason: "ZIP64 locator not found" };
+    const locDisk = tail.readUInt32LE(locPos + 4);
+    const locOffset = Number(tail.readBigUInt64LE(locPos + 8));
+
+    // The locator's disk number counts the final volume too, so resolve it
+    // through the same map, treating the final .zip as the highest volume.
+    const z64Path =
+      locDisk === thisDisk || locDisk === 0xffff ? finalVolume : volumes.get(locDisk) ?? finalVolume;
+    const z64 = await readBytes(z64Path, locOffset, 56);
+    if (z64.length < 56 || z64.readUInt32LE(0) !== ZIP64_EOCD_SIG) {
+      return { kind: "failed", reason: "ZIP64 EOCD record unreadable" };
+    }
+    thisDisk = z64.readUInt32LE(16);
+    cdStartDisk = z64.readUInt32LE(20);
+    cdSize = Number(z64.readBigUInt64LE(40));
+    cdOffset = Number(z64.readBigUInt64LE(48));
+  }
+
+  // A byte split named .z01/.zip still reports a single disk — concatenation,
+  // not volume mapping, is the correct reading for it.
+  if (thisDisk === 0 && cdStartDisk === 0) return { kind: "not-spanned" };
+
+  if (cdSize < 0 || cdSize > MAX_CD_BYTES) {
+    return { kind: "failed", reason: `implausible central directory size ${cdSize}` };
+  }
+
+  // The final .zip file is the highest-numbered volume.
+  volumes.set(thisDisk, finalVolume);
+
+  // Read cdSize bytes from (cdStartDisk, cdOffset), spilling onto later
+  // volumes if the directory straddles a volume boundary.
+  const chunks: Buffer[] = [];
+  let remaining = cdSize;
+  let disk = cdStartDisk;
+  let offset = cdOffset;
+  while (remaining > 0) {
+    const volPath = volumes.get(disk);
+    if (!volPath) return { kind: "failed", reason: `volume ${disk + 1} of the set is missing` };
+    const chunk = await readBytes(volPath, offset, remaining);
+    if (chunk.length === 0) {
+      return { kind: "failed", reason: `volume ${disk + 1} ended before the central directory did` };
+    }
+    chunks.push(chunk);
+    remaining -= chunk.length;
+    disk++;
+    offset = 0;
+  }
+
+  const cdBuf = Buffer.concat(chunks);
+  const entries = walkCentralDirectory(cdBuf, 0, cdSize);
+  log.debug(
+    { volumes: volumes.size, thisDisk, cdStartDisk, cdSize, entries: entries.length },
+    "Read spanned ZIP central directory"
+  );
+  return { kind: "entries", entries };
+}
+
+/** Find the ZIP64 EOCD locator, which sits just before the EOCD record. */
+function findZip64Locator(tail: Buffer, eocdPos: number): number {
+  for (let i = Math.min(eocdPos - 20, tail.length - 20); i >= 0; i--) {
+    if (tail.readUInt32LE(i) === ZIP64_LOCATOR_SIG) return i;
+  }
+  return -1;
+}
+
+/** Read up to `length` bytes at `offset`; a short read means end of file. */
+async function readBytes(filePath: string, offset: number, length: number): Promise<Buffer> {
+  const fh = await fsOpen(filePath, "r");
+  try {
+    const buf = Buffer.alloc(length);
+    const { bytesRead } = await fh.read(buf, 0, length, offset);
+    return buf.subarray(0, bytesRead);
+  } finally {
+    await fh.close();
+  }
+}

worker/src/provenance-backfill.ts

@@ -100,7 +100,7 @@ export async function resolveCandidateFingerprintEntries(
     if (destParts) {
       const read = tdlibRangeReader(client);
       destEntries =
-        candidate.archiveType === "ZIP" ? await readScannedZipListing(client, destParts)
+        candidate.archiveType === "ZIP" ? await readScannedZipListing(destParts, read)
         : candidate.archiveType === "SEVEN_Z" ? await readSevenZListingRanged(destParts, read)
         : candidate.archiveType === "RAR" ? await readRarListingRanged(destParts, read)
         : null;