diff --git a/worker/src/archive/central-directory.ts b/worker/src/archive/central-directory.ts index 48f8d46..7276a09 100644 --- a/worker/src/archive/central-directory.ts +++ b/worker/src/archive/central-directory.ts @@ -11,13 +11,21 @@ function extOf(name: string): string | null { return e === "" ? null : e; } +/** + * Locate the End Of Central Directory record by scanning backward for its + * signature. Returns -1 when the buffer holds no EOCD. + */ +export function findEocdOffset(tail: Buffer): number { + for (let i = tail.length - 22; i >= 0; i--) { + if (tail.readUInt32LE(i) === EOCD_SIG) return i; + } + return -1; +} + /** Parse a ZIP central directory from the tail of an archive. */ export function parseZipCentralDirectoryFromTail(tail: Buffer, tailStart: number): FileEntry[] { // 1. Find EOCD by scanning backward for its signature. - let eocd = -1; - for (let i = tail.length - 22; i >= 0; i--) { - if (tail.readUInt32LE(i) === EOCD_SIG) { eocd = i; break; } - } + const eocd = findEocdOffset(tail); if (eocd < 0) throw new RangeError("EOCD not found in tail"); let cdSize = tail.readUInt32LE(eocd + 12); @@ -45,10 +53,22 @@ export function parseZipCentralDirectoryFromTail(tail: Buffer, tailStart: number } // 3. Walk central-directory headers. + return walkCentralDirectory(tail, cdLocal, cdSize); +} + +/** + * Walk a run of central-directory file headers and return the file entries + * (directory entries are skipped). + * + * `buf` must contain the whole central directory starting at `start`; for a + * spanned archive that means the caller has already stitched together the + * volumes the directory straddles. + */ +export function walkCentralDirectory(tail: Buffer, start: number, cdSize: number): FileEntry[] { const entries: FileEntry[] = []; - let p = cdLocal; - const end = cdLocal + cdSize; - while (p + 46 <= end && tail.readUInt32LE(p) === CD_SIG) { + let p = start; + const end = start + cdSize; + while (p + 46 <= end && p + 46 <= tail.length && tail.readUInt32LE(p) === CD_SIG) { let crc = tail.readUInt32LE(p + 16) >>> 0; let comp = BigInt(tail.readUInt32LE(p + 20)); let uncomp = BigInt(tail.readUInt32LE(p + 24)); diff --git a/worker/src/archive/ranged/dispatch.test.ts b/worker/src/archive/ranged/dispatch.test.ts index b9d54b1..ed24c52 100644 --- a/worker/src/archive/ranged/dispatch.test.ts +++ b/worker/src/archive/ranged/dispatch.test.ts @@ -1,15 +1,103 @@ import { describe, it, expect } from "vitest"; -import { readScannedListingRanged } from "./dispatch.js"; +import { readScannedListingRanged, readScannedZipListing } from "./dispatch.js"; +import type { RangeReader } from "./range-reader.js"; +import { buildSpannedStoreZip, buildStoreZip, byteSplit } from "../testing/spanned-zip-fixture.js"; + +/** Serve ranged reads out of in-memory part buffers keyed by fileId. */ +function readerFor(parts: { fileId: string; buf: Buffer }[]): RangeReader { + const byId = new Map(parts.map((p) => [p.fileId, p.buf])); + return async (fileId, offset, length) => { + const buf = byId.get(fileId); + if (!buf) throw new Error(`unknown fileId ${fileId}`); + return buf.subarray(offset, offset + length); + }; +} + +const FILES = [ + { name: "src/b.bin", data: Buffer.alloc(2048, 7), disk: 0 }, + { name: "src/models/dragon.stl", data: Buffer.from("DRAGON"), disk: 1 }, + { name: "readme.txt", data: Buffer.from("hello world"), disk: 2 }, +]; describe("readScannedListingRanged", () => { it("returns null for an unknown archive type without calling the reader", async () => { - const read = async () => Buffer.alloc(0); const result = await readScannedListingRanged( "DOCUMENT", { invoke: async () => ({}) } as never, [{ fileId: "1", fileSize: 100n, fileName: "a.pdf" }], ); expect(result).toBeNull(); - void read; // unused placeholder kept out of the dispatch call — DOCUMENT never reaches a reader + }); +}); + +describe("readScannedZipListing", () => { + it("lists a ZIP-spec spanned set (.z01 … .zip) from the final volume's tail", async () => { + const vols = buildSpannedStoreZip(FILES, 3); + const names = ["Pack.z01", "Pack.z02", "Pack.zip"]; + const parts = vols.map((buf, i) => ({ fileId: String(i), fileSize: BigInt(buf.length), fileName: names[i] })); + + const entries = await readScannedZipListing( + parts, + readerFor(parts.map((p, i) => ({ fileId: p.fileId, buf: vols[i] }))), + ); + + expect(entries).not.toBeNull(); + expect(entries!.map((e) => e.path).sort()).toEqual([ + "readme.txt", + "src/b.bin", + "src/models/dragon.stl", + ]); + }); + + it("only reads the final volume — the earlier volumes are never downloaded", async () => { + const vols = buildSpannedStoreZip(FILES, 3); + const names = ["Pack.z01", "Pack.z02", "Pack.zip"]; + const parts = vols.map((buf, i) => ({ fileId: String(i), fileSize: BigInt(buf.length), fileName: names[i] })); + const touched: string[] = []; + const base = readerFor(parts.map((p, i) => ({ fileId: p.fileId, buf: vols[i] }))); + + await readScannedZipListing(parts, async (id, off, len, size) => { + touched.push(id); + return base(id, off, len, size); + }); + + expect([...new Set(touched)]).toEqual(["2"]); + }); + + it("still lists a 7-Zip raw byte split (.zip.001 …) using whole-archive offsets", async () => { + const zip = buildStoreZip([ + { name: "models/knight.stl", data: Buffer.alloc(5000, 9) }, + { name: "license.txt", data: Buffer.from("MIT") }, + ]); + const chunks = byteSplit(zip, 3); + const names = ["Pack.zip.001", "Pack.zip.002", "Pack.zip.003"]; + const parts = chunks.map((buf, i) => ({ fileId: String(i), fileSize: BigInt(buf.length), fileName: names[i] })); + + const entries = await readScannedZipListing( + parts, + readerFor(parts.map((p, i) => ({ fileId: p.fileId, buf: chunks[i] }))), + ); + + expect(entries!.map((e) => e.fileName).sort()).toEqual(["knight.stl", "license.txt"]); + }); + + it("returns null for a spanned set whose central directory starts on an earlier volume", async () => { + const vols = buildSpannedStoreZip( + FILES.map((f) => ({ ...f, disk: Math.min(f.disk, 1) })), + 3, + { cdStartDisk: 1 }, + ); + const names = ["Pack.z01", "Pack.z02", "Pack.zip"]; + const parts = vols.map((buf, i) => ({ fileId: String(i), fileSize: BigInt(buf.length), fileName: names[i] })); + + const entries = await readScannedZipListing( + parts, + readerFor(parts.map((p, i) => ({ fileId: p.fileId, buf: vols[i] }))), + ); + expect(entries).toBeNull(); + }); + + it("returns null when there are no parts", async () => { + expect(await readScannedZipListing([], readerFor([]))).toBeNull(); }); }); diff --git a/worker/src/archive/ranged/dispatch.ts b/worker/src/archive/ranged/dispatch.ts index 2ff4ae1..74a4a73 100644 --- a/worker/src/archive/ranged/dispatch.ts +++ b/worker/src/archive/ranged/dispatch.ts @@ -1,11 +1,11 @@ import type { Client } from "tdl"; -import { downloadFileRange } from "../../tdlib/range-download.js"; -import { parseZipCentralDirectoryFromTail, MIN_ZIP_TAIL_BYTES } from "../central-directory.js"; +import { parseZipCentralDirectoryFromTail, findEocdOffset, MIN_ZIP_TAIL_BYTES } from "../central-directory.js"; +import { isSpannedZipPartSet } from "../zip-spanned.js"; import { childLogger } from "../../util/logger.js"; import type { FileEntry } from "../zip-reader.js"; import { readSevenZListingRanged, type RangedPart } from "./sevenz-ranged.js"; import { readRarListingRanged } from "./rar-ranged.js"; -import { tdlibRangeReader } from "./range-reader.js"; +import { tdlibRangeReader, type RangeReader } from "./range-reader.js"; const log = childLogger("ranged-dispatch"); @@ -13,25 +13,45 @@ const log = childLogger("ranged-dispatch"); * Read a ZIP central directory from the tail of a (possibly multipart) * archive. `parts` is ordered; only the LAST part carries the EOCD record. * `fileSize` on each part is that part's own size (NOT the whole-archive - * total) so the download offset stays within that part's bounds, while - * `tailStart` passed to the parser is the logical whole-archive offset - * (preceding parts' sizes + the offset within the last part). + * total) so the download offset stays within that part's bounds. + * + * Which logical offset the EOCD's central-directory pointer is measured from + * depends on the multipart shape: + * + * - 7-Zip raw byte split (`.zip.001`, …): the parts are one ZIP file cut into + * chunks, so the pointer is a whole-archive offset → `tailStart` is the + * preceding parts' sizes plus the offset within the last part. + * - ZIP-spec spanned archive (`.z01`, …, `.zip`): each volume is its own unit + * and the pointer is relative to the start of the volume holding the + * directory → `tailStart` is just the offset within that final volume. + * + * Getting this wrong makes the computed directory offset wildly negative, and + * the parser then throws RangeError on every tail size — which is exactly how + * spanned sets ended up indexed with no file list at all. */ export async function readScannedZipListing( - client: Client, - parts: { fileId: string; fileSize: bigint }[], + parts: RangedPart[], + read: RangeReader, ): Promise { if (parts.length === 0) return null; const lastPart = parts[parts.length - 1]; - const precedingSize = parts.slice(0, -1).reduce((sum, p) => sum + Number(p.fileSize), 0); + const spanned = isSpannedZipPartSet(parts.map((p) => p.fileName)); + const precedingSize = spanned + ? 0 + : parts.slice(0, -1).reduce((sum, p) => sum + Number(p.fileSize), 0); const lastSize = Number(lastPart.fileSize); for (const tailBytes of [MIN_ZIP_TAIL_BYTES, MIN_ZIP_TAIL_BYTES * 4]) { const partOffset = Math.max(0, lastSize - tailBytes); const downloadLen = Math.min(tailBytes, lastSize); try { - const buf = await downloadFileRange(client, lastPart.fileId, partOffset, downloadLen, lastPart.fileSize); - const tailStart = precedingSize + partOffset; - return parseZipCentralDirectoryFromTail(buf, tailStart); + const buf = await read(lastPart.fileId, partOffset, downloadLen, lastPart.fileSize); + if (spanned && !cdStartsOnFinalVolume(buf)) { + // The directory begins on an earlier volume; reaching it would mean + // ranged-reading that volume too. Leave it to the full-download path. + log.debug({ fileId: lastPart.fileId }, "spanned ZIP central directory is not on the final volume"); + return null; + } + return parseZipCentralDirectoryFromTail(buf, partOffset + precedingSize); } catch (err) { if (err instanceof RangeError) continue; // try a larger tail log.warn({ err, fileId: lastPart.fileId }, "ranged ZIP listing failed"); @@ -41,6 +61,20 @@ export async function readScannedZipListing( return null; } +/** + * For a spanned archive, whether the EOCD says the central directory starts on + * the very volume that EOCD lives on (the usual case). ZIP64's saturated + * 0xFFFF disk fields are treated as "yes" — the ZIP64 record that supersedes + * them is itself in this tail, and the parser resolves it there. + */ +function cdStartsOnFinalVolume(tail: Buffer): boolean { + const eocd = findEocdOffset(tail); + if (eocd < 0) return true; // let the parser report the real problem + const thisDisk = tail.readUInt16LE(eocd + 4); + const cdStartDisk = tail.readUInt16LE(eocd + 6); + return thisDisk === cdStartDisk || thisDisk === 0xffff || cdStartDisk === 0xffff; +} + /** * Dispatch a (no-download) inner-file listing read by archive type. Used both * by the provenance-backfill path (reading an already-uploaded copy) and the @@ -55,7 +89,7 @@ export async function readScannedListingRanged( parts: RangedPart[], ): Promise { const read = tdlibRangeReader(client); - if (archiveType === "ZIP") return readScannedZipListing(client, parts); + if (archiveType === "ZIP") return readScannedZipListing(parts, read); if (archiveType === "SEVEN_Z") return readSevenZListingRanged(parts, read); if (archiveType === "RAR") return readRarListingRanged(parts, read); return null; diff --git a/worker/src/archive/testing/spanned-zip-fixture.ts b/worker/src/archive/testing/spanned-zip-fixture.ts new file mode 100644 index 0000000..4587b7a --- /dev/null +++ b/worker/src/archive/testing/spanned-zip-fixture.ts @@ -0,0 +1,191 @@ +import { crc32 } from "zlib"; // Node 20+ exposes zlib.crc32 + +/** + * Test-only builders that emit real ZIP byte streams. + * + * Two distinct on-disk shapes are produced here, because the worker has to + * tell them apart: + * + * - `buildSpannedStoreZip` → a ZIP-spec **spanned/multi-disk** archive + * (`Pack.z01`, `Pack.z02`, …, `Pack.zip`). Each volume is its own file; + * central-directory records carry a disk number, and the EOCD's + * "offset of start of central directory" is relative to the *start of the + * disk that holds it*, not to a concatenation of the volumes. + * + * - `buildStoreZip` → an ordinary single-file ZIP. Cutting its bytes into + * chunks yields the 7-Zip raw byte split shape (`Pack.zip.001`, …), where + * all disk numbers are 0 and offsets are whole-archive absolute. + * + * Field layouts follow APPNOTE 6.3.x sections 4.3.12 (central directory) and + * 4.3.16 (EOCD). Verified against Info-ZIP `zip -s` output. + */ + +const LOCAL_SIG = 0x04034b50; +const CD_SIG = 0x02014b50; +const EOCD_SIG = 0x06054b50; +/** APPNOTE 8.5.3: the first volume of a spanned archive starts with this. */ +const SPANNING_SIG = 0x08074b50; + +export interface FixtureFile { + name: string; + data: Buffer; + /** 0-based volume this file's local header + data is written to. */ + disk?: number; +} + +function localHeader(name: string, data: Buffer): Buffer { + const nameBuf = Buffer.from(name, "utf8"); + const local = Buffer.alloc(30); + local.writeUInt32LE(LOCAL_SIG, 0); + local.writeUInt16LE(20, 4); // version needed + local.writeUInt16LE(0, 6); // flags + local.writeUInt16LE(0, 8); // method = store + local.writeUInt32LE(crc32(data) >>> 0, 14); + local.writeUInt32LE(data.length, 18); // compressed + local.writeUInt32LE(data.length, 22); // uncompressed + local.writeUInt16LE(nameBuf.length, 26); + local.writeUInt16LE(0, 28); // extra len + return Buffer.concat([local, nameBuf, data]); +} + +function centralHeader(name: string, data: Buffer, diskStart: number, relOffset: number): Buffer { + const nameBuf = Buffer.from(name, "utf8"); + const cd = Buffer.alloc(46); + cd.writeUInt32LE(CD_SIG, 0); + cd.writeUInt16LE(20, 4); // version made by + cd.writeUInt16LE(20, 6); // version needed + cd.writeUInt16LE(0, 8); // flags + cd.writeUInt16LE(0, 10); // method = store + cd.writeUInt32LE(crc32(data) >>> 0, 16); + cd.writeUInt32LE(data.length, 20); // compressed + cd.writeUInt32LE(data.length, 24); // uncompressed + cd.writeUInt16LE(nameBuf.length, 28); + cd.writeUInt16LE(diskStart, 34); // disk number start + cd.writeUInt32LE(relOffset, 42); // offset of local header, relative to its disk + return Buffer.concat([cd, nameBuf]); +} + +function eocd(opts: { + thisDisk: number; + cdStartDisk: number; + entriesThisDisk: number; + entriesTotal: number; + cdSize: number; + cdOffset: number; +}): Buffer { + const buf = Buffer.alloc(22); + buf.writeUInt32LE(EOCD_SIG, 0); + buf.writeUInt16LE(opts.thisDisk, 4); + buf.writeUInt16LE(opts.cdStartDisk, 6); + buf.writeUInt16LE(opts.entriesThisDisk, 8); + buf.writeUInt16LE(opts.entriesTotal, 10); + buf.writeUInt32LE(opts.cdSize, 12); + buf.writeUInt32LE(opts.cdOffset, 16); + return buf; +} + +/** Build an ordinary single-file STORE ZIP (all disk numbers 0). */ +export function buildStoreZip(files: FixtureFile[]): Buffer { + const body: Buffer[] = []; + const central: Buffer[] = []; + let offset = 0; + for (const f of files) { + const local = localHeader(f.name, f.data); + body.push(local); + central.push(centralHeader(f.name, f.data, 0, offset)); + offset += local.length; + } + const cdBuf = Buffer.concat(central); + return Buffer.concat([ + ...body, + cdBuf, + eocd({ + thisDisk: 0, + cdStartDisk: 0, + entriesThisDisk: files.length, + entriesTotal: files.length, + cdSize: cdBuf.length, + cdOffset: offset, + }), + ]); +} + +/** + * Build a ZIP-spec spanned archive as one Buffer per volume. + * Returned array is volume order: [z01, z02, …, zip] (last element is the + * final volume, which carries the central directory and EOCD). + * + * `cdStartDisk` (default: last volume) lets a test place the central + * directory so that it begins on an earlier volume and spills forward, + * exercising the cross-volume read path. As in a real writer, no file data + * may live on a volume after the one the directory starts on — those volumes + * hold directory continuation only. + */ +export function buildSpannedStoreZip( + files: FixtureFile[], + totalDisks: number, + opts: { cdStartDisk?: number } = {} +): Buffer[] { + const cdStart = opts.cdStartDisk ?? totalDisks - 1; + for (const f of files) { + if ((f.disk ?? 0) > cdStart) { + throw new Error(`fixture misuse: ${f.name} is on volume ${f.disk} but the CD starts on ${cdStart}`); + } + if ((f.disk ?? 0) >= totalDisks) { + throw new Error(`fixture misuse: ${f.name} is on volume ${f.disk} of ${totalDisks}`); + } + } + const chunks: Buffer[][] = Array.from({ length: totalDisks }, () => []); + const lengths = new Array(totalDisks).fill(0); + + const marker = Buffer.alloc(4); + marker.writeUInt32LE(SPANNING_SIG, 0); + chunks[0].push(marker); + lengths[0] = 4; + + const central: Buffer[] = []; + for (const f of files) { + const disk = f.disk ?? 0; + const local = localHeader(f.name, f.data); + central.push(centralHeader(f.name, f.data, disk, lengths[disk])); + chunks[disk].push(local); + lengths[disk] += local.length; + } + + const cdBuf = Buffer.concat(central); + const lastDisk = totalDisks - 1; + const cdStartDisk = cdStart; + const cdOffset = lengths[cdStartDisk]; + + // Write the CD starting on cdStartDisk, spilling onto later volumes. + const spillDisks = lastDisk - cdStartDisk + 1; + const firstChunkLen = Math.ceil(cdBuf.length / spillDisks); + let written = 0; + for (let d = cdStartDisk; d <= lastDisk; d++) { + const take = d === lastDisk ? cdBuf.length - written : Math.min(firstChunkLen, cdBuf.length - written); + chunks[d].push(cdBuf.subarray(written, written + take)); + lengths[d] += take; + written += take; + } + + chunks[lastDisk].push( + eocd({ + thisDisk: lastDisk, + cdStartDisk, + entriesThisDisk: files.length, + entriesTotal: files.length, + cdSize: cdBuf.length, + cdOffset, + }) + ); + + return chunks.map((c) => Buffer.concat(c)); +} + +/** Cut a buffer into `count` roughly equal chunks (7-Zip raw byte split). */ +export function byteSplit(buf: Buffer, count: number): Buffer[] { + const size = Math.ceil(buf.length / count); + const out: Buffer[] = []; + for (let i = 0; i < buf.length; i += size) out.push(buf.subarray(i, i + size)); + return out; +} diff --git a/worker/src/archive/zip-reader.ts b/worker/src/archive/zip-reader.ts index 960ea15..c539454 100644 --- a/worker/src/archive/zip-reader.ts +++ b/worker/src/archive/zip-reader.ts @@ -3,6 +3,7 @@ import { open as fsOpen, stat as fsStat } from "fs/promises"; import path from "path"; import { Readable } from "stream"; import { childLogger } from "../util/logger.js"; +import { isSpannedZipPartSet, readSpannedZipCentralDirectory } from "./zip-spanned.js"; const log = childLogger("zip-reader"); @@ -17,9 +18,15 @@ export interface FileEntry { /** * Read the central directory of a ZIP file without extracting any contents. - * For multipart ZIPs (.zip.001, .zip.002 etc.), uses a custom random-access - * reader that spans all parts seamlessly so yauzl can find the central - * directory at the end of the combined data. + * + * Three shapes are handled: + * - a single `.zip` → yauzl directly; + * - a 7-Zip raw byte split (`.zip.001`, `.zip.002`, …), which is one ZIP file + * cut into chunks → a random-access reader that spans the chunks so yauzl + * sees the combined stream; + * - a ZIP-spec spanned/multi-disk archive (`.z01`, `.z02`, …, `.zip`), a + * different on-disk format that yauzl refuses outright → a dedicated + * volume-aware central-directory reader. */ export async function readZipCentralDirectory( filePaths: string[] @@ -28,7 +35,22 @@ export async function readZipCentralDirectory( return readSingleZip(filePaths[0]); } - // Multipart: use a spanning random-access reader + if (isSpannedZipPartSet(filePaths)) { + try { + const result = await readSpannedZipCentralDirectory(filePaths); + if (result.kind === "entries") return result.entries; + if (result.kind === "failed") { + log.warn({ reason: result.reason, parts: filePaths.length }, "Failed to read spanned ZIP"); + return []; + } + // "not-spanned": named like volumes but really a byte split — fall through. + } catch (err) { + log.warn({ err, parts: filePaths.length }, "Failed to read spanned ZIP"); + return []; + } + } + + // Multipart byte split: use a spanning random-access reader return readMultipartZip(filePaths); } diff --git a/worker/src/archive/zip-spanned.test.ts b/worker/src/archive/zip-spanned.test.ts new file mode 100644 index 0000000..402702b --- /dev/null +++ b/worker/src/archive/zip-spanned.test.ts @@ -0,0 +1,214 @@ +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { mkdir, mkdtemp, readdir, rm, writeFile } from "fs/promises"; +import { execFile } from "child_process"; +import { promisify } from "util"; +import { tmpdir } from "os"; +import path from "path"; +import { isSpannedZipPartSet } from "./zip-spanned.js"; +import { readZipCentralDirectory } from "./zip-reader.js"; +import { buildSpannedStoreZip, buildStoreZip, byteSplit } from "./testing/spanned-zip-fixture.js"; + +const execFileAsync = promisify(execFile); + +let dir: string; + +beforeAll(async () => { + dir = await mkdtemp(path.join(tmpdir(), "zip-spanned-")); +}); + +afterAll(async () => { + await rm(dir, { recursive: true, force: true }); +}); + +/** Write volume buffers out under the given names and return their paths. */ +async function writeParts(names: string[], buffers: Buffer[], sub: string): Promise { + const base = path.join(dir, sub); + await rm(base, { recursive: true, force: true }); + await mkdir(base, { recursive: true }); + const paths: string[] = []; + for (let i = 0; i < names.length; i++) { + const p = path.join(base, names[i]); + await writeFile(p, buffers[i]); + paths.push(p); + } + return paths; +} + +const FILES = [ + { name: "src/b.bin", data: Buffer.alloc(2048, 7), disk: 0 }, + { name: "src/models/dragon.stl", data: Buffer.from("DRAGON"), disk: 1 }, + { name: "src/models/", data: Buffer.alloc(0), disk: 1 }, + { name: "src/a.bin", data: Buffer.alloc(4096, 3), disk: 2 }, + { name: "readme.txt", data: Buffer.from("hello world"), disk: 3 }, +]; + +describe("isSpannedZipPartSet", () => { + it("recognizes a .z01 + .zip volume set", () => { + expect(isSpannedZipPartSet(["/t/Pack.z01", "/t/Pack.z02", "/t/Pack.zip"])).toBe(true); + }); + + it("recognizes the set regardless of the order it is handed in", () => { + expect(isSpannedZipPartSet(["/t/Pack.zip", "/t/Pack.z02", "/t/Pack.z01"])).toBe(true); + }); + + it("rejects a 7-Zip raw byte split (.zip.001)", () => { + expect(isSpannedZipPartSet(["/t/Pack.zip.001", "/t/Pack.zip.002"])).toBe(false); + }); + + it("rejects a single .zip", () => { + expect(isSpannedZipPartSet(["/t/Pack.zip"])).toBe(false); + }); + + it("rejects a set with no final .zip volume", () => { + expect(isSpannedZipPartSet(["/t/Pack.z01", "/t/Pack.z02"])).toBe(false); + }); + + it("rejects a set with two .zip volumes", () => { + expect(isSpannedZipPartSet(["/t/A.zip", "/t/B.zip"])).toBe(false); + }); +}); + +describe("readZipCentralDirectory on a spanned (.z01 + .zip) set", () => { + it("lists every entry with correct paths, sizes and crc32", async () => { + const vols = buildSpannedStoreZip(FILES, 4); + const paths = await writeParts(["Pack.z01", "Pack.z02", "Pack.z03", "Pack.zip"], vols, "spanned"); + + const entries = await readZipCentralDirectory(paths); + + expect(entries.map((e) => e.path).sort()).toEqual([ + "readme.txt", + "src/a.bin", + "src/b.bin", + "src/models/dragon.stl", + ]); + const dragon = entries.find((e) => e.fileName === "dragon.stl")!; + expect(dragon.uncompressedSize).toBe(6n); + expect(dragon.extension).toBe("stl"); + expect(dragon.crc32).toMatch(/^[0-9a-f]{8}$/); + const a = entries.find((e) => e.fileName === "a.bin")!; + expect(a.uncompressedSize).toBe(4096n); + expect(a.compressedSize).toBe(4096n); + }); + + it("works when the parts are handed over out of volume order", async () => { + const vols = buildSpannedStoreZip(FILES, 4); + const paths = await writeParts(["Pack.z01", "Pack.z02", "Pack.z03", "Pack.zip"], vols, "unordered"); + const shuffled = [paths[3], paths[1], paths[0], paths[2]]; + + const entries = await readZipCentralDirectory(shuffled); + expect(entries).toHaveLength(4); + }); + + it("reads a central directory that begins on an earlier volume and spills forward", async () => { + // Files live on volumes 0–2; the directory starts on volume 2 and + // continues onto the final .zip, which holds nothing else. + const spillFiles = FILES.map((f) => ({ ...f, disk: Math.min(f.disk, 2) })); + const vols = buildSpannedStoreZip(spillFiles, 4, { cdStartDisk: 2 }); + const paths = await writeParts(["Pack.z01", "Pack.z02", "Pack.z03", "Pack.zip"], vols, "spilled"); + + const entries = await readZipCentralDirectory(paths); + expect(entries.map((e) => e.fileName).sort()).toEqual(["a.bin", "b.bin", "dragon.stl", "readme.txt"]); + }); + + it("returns [] instead of throwing when a needed volume is missing", async () => { + const spillFiles = FILES.map((f) => ({ ...f, disk: Math.min(f.disk, 1) })); + const vols = buildSpannedStoreZip(spillFiles, 3, { cdStartDisk: 1 }); + const paths = await writeParts(["Pack.z01", "Pack.z02", "Pack.zip"], vols, "missing"); + // Drop Pack.z02 — the volume the central directory starts on. + const entries = await readZipCentralDirectory([paths[0], paths[2]]); + expect(entries).toEqual([]); + }); + + it("returns [] instead of throwing when the final volume is garbage", async () => { + const vols = buildSpannedStoreZip([{ name: "x.stl", data: Buffer.alloc(64, 1), disk: 0 }], 2); + const paths = await writeParts(["Pack.z01", "Pack.zip"], [vols[0], Buffer.alloc(500, 0x5a)], "garbage"); + expect(await readZipCentralDirectory(paths)).toEqual([]); + }); + + it("falls back to concatenation semantics when .z01-named parts are really a byte split", async () => { + // Some producers name a raw byte split .z01/.zip. The EOCD then reports + // disk 0, so concatenation — not volume mapping — is the correct reading. + const zip = buildStoreZip([ + { name: "one.stl", data: Buffer.alloc(3000, 1) }, + { name: "two.stl", data: Buffer.alloc(3000, 2) }, + ]); + const paths = await writeParts(["Pack.z01", "Pack.zip"], byteSplit(zip, 2), "mislabeled"); + const entries = await readZipCentralDirectory(paths); + expect(entries.map((e) => e.fileName).sort()).toEqual(["one.stl", "two.stl"]); + }); +}); + +describe("readZipCentralDirectory regressions for the shapes that already worked", () => { + it("still reads a 7-Zip raw byte split (.zip.001 …)", async () => { + const zip = buildStoreZip([ + { name: "models/knight.stl", data: Buffer.alloc(5000, 9) }, + { name: "license.txt", data: Buffer.from("MIT") }, + ]); + const paths = await writeParts( + ["Pack.zip.001", "Pack.zip.002", "Pack.zip.003"], + byteSplit(zip, 3), + "bytesplit" + ); + const entries = await readZipCentralDirectory(paths); + expect(entries.map((e) => e.fileName).sort()).toEqual(["knight.stl", "license.txt"]); + expect(entries.find((e) => e.fileName === "knight.stl")!.uncompressedSize).toBe(5000n); + }); + + it("still reads a plain single .zip", async () => { + const zip = buildStoreZip([{ name: "solo.stl", data: Buffer.from("SOLO") }]); + const paths = await writeParts(["Pack.zip"], [zip], "single"); + const entries = await readZipCentralDirectory(paths); + expect(entries.map((e) => e.fileName)).toEqual(["solo.stl"]); + }); +}); + +// ── Cross-checks against real Info-ZIP output ──────────────────────────── +// Skipped automatically where the `zip` CLI is unavailable; the hand-built +// fixtures above are the authoritative, portable coverage. + +const HAS_ZIP_CLI = await execFileAsync("zip", ["-v"]).then( + () => true, + () => false +); + +/** Collect Pack.z01 … Pack.zip from a directory, in volume order. */ +async function collectVolumes(work: string): Promise { + const names = await readdir(work); + const vols = names.filter((n) => /^Pack\.z\d{2,}$/i.test(n)).sort(); + const final = names.find((n) => /^Pack\.zip$/i.test(n)); + expect(final).toBeDefined(); + expect(vols.length).toBeGreaterThan(0); + return [...vols, final!].map((n) => path.join(work, n)); +} + +describe.skipIf(!HAS_ZIP_CLI)("readZipCentralDirectory against archives produced by Info-ZIP `zip -s`", () => { + it("lists a genuine spanned archive", async () => { + const work = path.join(dir, "real"); + await mkdir(path.join(work, "src/models"), { recursive: true }); + await writeFile(path.join(work, "src/big.bin"), Buffer.alloc(300_000, 4)); + await writeFile(path.join(work, "src/models/dragon.stl"), Buffer.alloc(200_000, 5)); + await writeFile(path.join(work, "src/readme.txt"), "hello world"); + await execFileAsync("zip", ["-r", "-0", "-s", "100k", "Pack.zip", "src"], { cwd: work }); + + const entries = await readZipCentralDirectory(await collectVolumes(work)); + expect(entries.map((e) => e.path).sort()).toEqual([ + "src/big.bin", + "src/models/dragon.stl", + "src/readme.txt", + ]); + expect(entries.find((e) => e.fileName === "dragon.stl")!.uncompressedSize).toBe(200_000n); + }); + + it("lists a genuine ZIP64 spanned archive", async () => { + const work = path.join(dir, "real64"); + await mkdir(work, { recursive: true }); + await writeFile(path.join(work, "big.bin"), Buffer.alloc(300_000, 6)); + await writeFile(path.join(work, "note.txt"), "zip64 spanned"); + // -fz forces ZIP64 structures even though the payload is small. + await execFileAsync("zip", ["-0", "-fz", "-s", "100k", "Pack.zip", "big.bin", "note.txt"], { cwd: work }); + + const entries = await readZipCentralDirectory(await collectVolumes(work)); + expect(entries.map((e) => e.fileName).sort()).toEqual(["big.bin", "note.txt"]); + expect(entries.find((e) => e.fileName === "big.bin")!.uncompressedSize).toBe(300_000n); + }); +}); diff --git a/worker/src/archive/zip-spanned.ts b/worker/src/archive/zip-spanned.ts new file mode 100644 index 0000000..93c8222 --- /dev/null +++ b/worker/src/archive/zip-spanned.ts @@ -0,0 +1,170 @@ +import { open as fsOpen, stat as fsStat } from "fs/promises"; +import path from "path"; +import { findEocdOffset, walkCentralDirectory, MIN_ZIP_TAIL_BYTES } from "./central-directory.js"; +import { childLogger } from "../util/logger.js"; +import type { FileEntry } from "./zip-reader.js"; + +const log = childLogger("zip-spanned"); + +const ZIP64_LOCATOR_SIG = 0x07064b50; +const ZIP64_EOCD_SIG = 0x06064b50; + +/** Refuse to allocate a buffer for an absurd central-directory size. */ +const MAX_CD_BYTES = 256 * 1024 * 1024; + +/** + * A `.z01`/`.z02`/…/`.zip` set is a ZIP-spec **spanned (multi-disk)** archive: + * a genuinely different on-disk format from a 7-Zip raw byte split + * (`.zip.001`, `.zip.002`, …), which is one ZIP file cut into chunks. + * + * The distinction matters because a byte split is read by concatenating the + * chunks, whereas in a spanned archive each volume is its own unit: the EOCD's + * central-directory offset is relative to the start of the volume that holds + * the directory, and central-directory records carry a volume number. Feeding + * a spanned set to a concatenating reader yields nonsense offsets (and yauzl + * refuses outright: "multi-disk zip files are not supported"). + * + * Detected from filename shape rather than the detector's multipart `pattern` + * so this stays independent of how `detect.ts` labels the two variants. + * Order-independent: the caller may hand the volumes over in any order. + */ +export function isSpannedZipPartSet(filePaths: string[]): boolean { + if (filePaths.length < 2) return false; + const names = filePaths.map((p) => path.basename(p)); + const finals = names.filter((n) => /\.zip$/i.test(n)); + const volumes = names.filter((n) => /\.z\d{2,}$/i.test(n)); + return finals.length === 1 && volumes.length === names.length - 1; +} + +export type SpannedZipResult = + /** Successfully read; `entries` may legitimately be empty for an empty archive. */ + | { kind: "entries"; entries: FileEntry[] } + /** The EOCD reports a single disk — the parts are really a byte split, so the + * caller should fall back to reading them as one concatenated stream. */ + | { kind: "not-spanned" } + | { kind: "failed"; reason: string }; + +/** + * Read the central directory of a ZIP-spec spanned archive. + * + * Only the volume holding the central directory (plus any it spills onto) is + * read, and only the directory bytes themselves — the file payloads are never + * touched, so this is cheap regardless of archive size. + * + * Coverage: STORE/DEFLATE and ZIP64 spanned archives are handled. Archives + * whose central directory is itself encrypted (strong encryption / "hide + * filenames") cannot be listed by any header-only reader and return `failed`. + */ +export async function readSpannedZipCentralDirectory(filePaths: string[]): Promise { + const volumes = new Map(); + let finalVolume: string | undefined; + for (const p of filePaths) { + const base = path.basename(p); + const m = base.match(/\.z(\d{2,})$/i); + if (m) { + // .z01 is volume 0, .z02 is volume 1, … (APPNOTE numbers disks from 0). + volumes.set(parseInt(m[1], 10) - 1, p); + } else if (/\.zip$/i.test(base)) { + finalVolume = p; + } + } + if (!finalVolume) return { kind: "failed", reason: "no final .zip volume" }; + + const finalSize = (await fsStat(finalVolume)).size; + const tailLen = Math.min(finalSize, MIN_ZIP_TAIL_BYTES); + const tailStart = finalSize - tailLen; + const tail = await readBytes(finalVolume, tailStart, tailLen); + + const eocdPos = findEocdOffset(tail); + if (eocdPos < 0) return { kind: "failed", reason: "EOCD not found in final volume" }; + + let thisDisk = tail.readUInt16LE(eocdPos + 4); + let cdStartDisk = tail.readUInt16LE(eocdPos + 6); + let cdSize = tail.readUInt32LE(eocdPos + 12); + let cdOffset = tail.readUInt32LE(eocdPos + 16); + + // ZIP64: any saturated field means the real values live in the ZIP64 EOCD + // record, which in a spanned archive may sit on a different volume. + if ( + thisDisk === 0xffff || + cdStartDisk === 0xffff || + cdSize === 0xffffffff || + cdOffset === 0xffffffff + ) { + const locPos = findZip64Locator(tail, eocdPos); + if (locPos < 0) return { kind: "failed", reason: "ZIP64 locator not found" }; + const locDisk = tail.readUInt32LE(locPos + 4); + const locOffset = Number(tail.readBigUInt64LE(locPos + 8)); + + // The locator's disk number counts the final volume too, so resolve it + // through the same map, treating the final .zip as the highest volume. + const z64Path = + locDisk === thisDisk || locDisk === 0xffff ? finalVolume : volumes.get(locDisk) ?? finalVolume; + const z64 = await readBytes(z64Path, locOffset, 56); + if (z64.length < 56 || z64.readUInt32LE(0) !== ZIP64_EOCD_SIG) { + return { kind: "failed", reason: "ZIP64 EOCD record unreadable" }; + } + thisDisk = z64.readUInt32LE(16); + cdStartDisk = z64.readUInt32LE(20); + cdSize = Number(z64.readBigUInt64LE(40)); + cdOffset = Number(z64.readBigUInt64LE(48)); + } + + // A byte split named .z01/.zip still reports a single disk — concatenation, + // not volume mapping, is the correct reading for it. + if (thisDisk === 0 && cdStartDisk === 0) return { kind: "not-spanned" }; + + if (cdSize < 0 || cdSize > MAX_CD_BYTES) { + return { kind: "failed", reason: `implausible central directory size ${cdSize}` }; + } + + // The final .zip file is the highest-numbered volume. + volumes.set(thisDisk, finalVolume); + + // Read cdSize bytes from (cdStartDisk, cdOffset), spilling onto later + // volumes if the directory straddles a volume boundary. + const chunks: Buffer[] = []; + let remaining = cdSize; + let disk = cdStartDisk; + let offset = cdOffset; + while (remaining > 0) { + const volPath = volumes.get(disk); + if (!volPath) return { kind: "failed", reason: `volume ${disk + 1} of the set is missing` }; + const chunk = await readBytes(volPath, offset, remaining); + if (chunk.length === 0) { + return { kind: "failed", reason: `volume ${disk + 1} ended before the central directory did` }; + } + chunks.push(chunk); + remaining -= chunk.length; + disk++; + offset = 0; + } + + const cdBuf = Buffer.concat(chunks); + const entries = walkCentralDirectory(cdBuf, 0, cdSize); + log.debug( + { volumes: volumes.size, thisDisk, cdStartDisk, cdSize, entries: entries.length }, + "Read spanned ZIP central directory" + ); + return { kind: "entries", entries }; +} + +/** Find the ZIP64 EOCD locator, which sits just before the EOCD record. */ +function findZip64Locator(tail: Buffer, eocdPos: number): number { + for (let i = Math.min(eocdPos - 20, tail.length - 20); i >= 0; i--) { + if (tail.readUInt32LE(i) === ZIP64_LOCATOR_SIG) return i; + } + return -1; +} + +/** Read up to `length` bytes at `offset`; a short read means end of file. */ +async function readBytes(filePath: string, offset: number, length: number): Promise { + const fh = await fsOpen(filePath, "r"); + try { + const buf = Buffer.alloc(length); + const { bytesRead } = await fh.read(buf, 0, length, offset); + return buf.subarray(0, bytesRead); + } finally { + await fh.close(); + } +} diff --git a/worker/src/provenance-backfill.ts b/worker/src/provenance-backfill.ts index 0fceb26..06caf7e 100644 --- a/worker/src/provenance-backfill.ts +++ b/worker/src/provenance-backfill.ts @@ -100,7 +100,7 @@ export async function resolveCandidateFingerprintEntries( if (destParts) { const read = tdlibRangeReader(client); destEntries = - candidate.archiveType === "ZIP" ? await readScannedZipListing(client, destParts) + candidate.archiveType === "ZIP" ? await readScannedZipListing(destParts, read) : candidate.archiveType === "SEVEN_Z" ? await readSevenZListingRanged(destParts, read) : candidate.archiveType === "RAR" ? await readRarListingRanged(destParts, read) : null;