From 08422032e5fa4b7281c4f3185720d6c59f012701 Mon Sep 17 00:00:00 2001 From: xCyanGrizzly Date: Sat, 22 Aug 2026 23:47:40 +0200 Subject: [PATCH] fix(worker): read multi-volume 7z listings across the whole volume set MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit readSevenZListingRanged only ever inspected parts[0]. A `.7z.001`/`.7z.002` set is a raw byte split of one logical 7z file, and a 7z file keeps its next header (the archive index) at the *end* of the stream — i.e. in the last volume. So the bounds guard `endStart + nextHeaderSize > size`, with `size` being parts[0].fileSize, rejected every multipart set before a single byte of the index was fetched. Measured on the live DB: forwarding channels + SEVEN_Z + partCount >= 2 listed 1 of 151, while single-volume forwards listed 2301 of 2306 and the full-download path listed 132 of 132. Those 150 packages were forwarded with an empty file list, invisible to content and keyword search. Treat the set as one logical byte stream: mapRangeToVolumes() maps a whole-archive range onto per-volume reads, splitting it when it straddles a volume boundary, and every header region (signature, next header, and an encoded header's packed bytes) is fetched through it. All volumes are reconstructed sparsely, matching what the full-download path already does successfully — it hands `7z l` the first part's path with the rest of the set beside it on disk. Also log every bail-out. The function had six silent `return null` points and an outer catch that only fires on thrown exceptions, so the production failure produced no log line at all — the same gap e123a5c closed for the RAR reader. Byte-split volume sets and single-volume `.7z` are covered by tests; archives with encrypted headers still cannot be listed by any header-only reader and return null (now with a logged reason). Fixtures are built byte-by-byte because no `7z` binary is installed here, so the tests assert the ranged-read offsets and the sparse reconstruction rather than `7z l` output. Co-Authored-By: Claude Sonnet 5 --- .../src/archive/ranged/sevenz-ranged.test.ts | 245 +++++++++++++++++- worker/src/archive/ranged/sevenz-ranged.ts | 190 +++++++++++--- 2 files changed, 398 insertions(+), 37 deletions(-) diff --git a/worker/src/archive/ranged/sevenz-ranged.test.ts b/worker/src/archive/ranged/sevenz-ranged.test.ts index b489f32..6c916ab 100644 --- a/worker/src/archive/ranged/sevenz-ranged.test.ts +++ b/worker/src/archive/ranged/sevenz-ranged.test.ts @@ -64,7 +64,250 @@ describe("readSevenZListingRanged", () => { }); }); -import { read7zNumber, locate7zEncodedHeaderPack } from "./sevenz-ranged.js"; +import { read7zNumber, locate7zEncodedHeaderPack, mapRangeToVolumes, planSevenZSparseParts } from "./sevenz-ranged.js"; +import type { RangedPart } from "./sevenz-ranged.js"; +import type { SparsePart } from "./sparse-list.js"; + +describe("mapRangeToVolumes", () => { + it("maps a range fully inside one volume", () => { + expect(mapRangeToVolumes([100, 100, 100], 120, 30)).toEqual([ + { partIndex: 1, offset: 20, length: 30 }, + ]); + }); + + it("splits a range that straddles a volume boundary", () => { + expect(mapRangeToVolumes([100, 100], 90, 20)).toEqual([ + { partIndex: 0, offset: 90, length: 10 }, + { partIndex: 1, offset: 0, length: 10 }, + ]); + }); + + it("spans three volumes when the range swallows a whole middle volume", () => { + expect(mapRangeToVolumes([100, 50, 100], 90, 80)).toEqual([ + { partIndex: 0, offset: 90, length: 10 }, + { partIndex: 1, offset: 0, length: 50 }, + { partIndex: 2, offset: 0, length: 20 }, + ]); + }); + + it("degenerates to a single-volume identity mapping", () => { + expect(mapRangeToVolumes([5_000_000], 4_900_000, 100)).toEqual([ + { partIndex: 0, offset: 4_900_000, length: 100 }, + ]); + }); + + it("returns null when the range runs past the concatenated end", () => { + expect(mapRangeToVolumes([100, 100], 190, 20)).toBeNull(); + expect(mapRangeToVolumes([100], 100, 1)).toBeNull(); + }); + + it("returns null for negative offsets or lengths", () => { + expect(mapRangeToVolumes([100], -1, 10)).toBeNull(); + expect(mapRangeToVolumes([100], 10, -1)).toBeNull(); + }); + + it("returns no slices for a zero-length range", () => { + expect(mapRangeToVolumes([100, 100], 150, 0)).toEqual([]); + }); +}); + +/** + * A `.7z.001`/`.7z.002` set is a raw byte split of one logical 7z file, so + * fixtures are built as a single logical stream and then cut into volumes. + * No real `7z` binary is needed (and none is installed here) because the + * contract under test is the whole-archive-offset mapping, not `7z l` parsing. + */ +function buildLogical7z(opts: { + total: number; + nextHeaderStart: number; // absolute offset in the logical stream + nextHeader: Buffer; + pack?: { start: number; bytes: Buffer }; // absolute offset of packed header bytes +}): Buffer { + const buf = Buffer.alloc(opts.total, 0x5a); // 0x5a stands in for file payload + MAGIC.copy(buf, 0); + buf.writeUInt8(0, 6); buf.writeUInt8(4, 7); + buf.writeUInt32LE(0, 8); + buf.writeBigUInt64LE(BigInt(opts.nextHeaderStart - 32), 12); // NextHeaderOffset + buf.writeBigUInt64LE(BigInt(opts.nextHeader.length), 20); // NextHeaderSize + buf.writeUInt32LE(0, 28); + opts.nextHeader.copy(buf, opts.nextHeaderStart); + if (opts.pack) opts.pack.bytes.copy(buf, opts.pack.start); + return buf; +} + +function splitIntoVolumes(logical: Buffer, sizes: number[]): Buffer[] { + const out: Buffer[] = []; + let pos = 0; + for (const s of sizes) { out.push(logical.subarray(pos, pos + s)); pos += s; } + return out; +} + +function volumeSet(volumes: Buffer[]): { + parts: RangedPart[]; + read: RangeReader; + reads: { fileId: string; offset: number; length: number }[]; +} { + const parts = volumes.map((v, i) => ({ + fileId: `v${i + 1}`, + fileSize: BigInt(v.length), + fileName: `pack.7z.${String(i + 1).padStart(3, "0")}`, + })); + const reads: { fileId: string; offset: number; length: number }[] = []; + const read: RangeReader = async (fileId, offset, length) => { + reads.push({ fileId, offset, length }); + const vol = volumes[parts.findIndex((p) => p.fileId === fileId)]; + return Buffer.from(vol.subarray(offset, offset + length)); + }; + return { parts, read, reads }; +} + +/** Rebuild the logical stream from the sparse per-volume reconstructions. */ +function reconstruct(sparseParts: SparsePart[]): Buffer { + return Buffer.concat( + sparseParts.map((p) => { + const b = Buffer.alloc(p.size); + for (const r of p.regions) r.bytes.copy(b, r.offset); + return b; + }), + ); +} + +describe("readSevenZListingRanged — multi-volume (.7z.001, .7z.002, ...)", () => { + it("reads the next header from the LAST volume, not the first", async () => { + // Volume 1 is 200 bytes; the index sits at logical 280 — inside volume 2. + const nextHeader = Buffer.concat([Buffer.from([0x01]), Buffer.alloc(19, 0x11)]); + const logical = buildLogical7z({ total: 300, nextHeaderStart: 280, nextHeader }); + const { parts, read, reads } = volumeSet(splitIntoVolumes(logical, [200, 100])); + + const sparse = await planSevenZSparseParts(parts, read); + expect(sparse).not.toBeNull(); + + expect(reads).toEqual([ + { fileId: "v1", offset: 0, length: 32 }, // signature header: volume 1 + { fileId: "v2", offset: 80, length: 20 }, // next header: volume 2 @ 280-200 + ]); + // Both volumes are reconstructed so `7z l pack.7z.001` can concatenate them. + expect(sparse!.map((p) => [p.fileName, p.size])).toEqual([ + ["pack.7z.001", 200], + ["pack.7z.002", 100], + ]); + const rebuilt = reconstruct(sparse!); + expect(rebuilt.length).toBe(300); + expect(rebuilt.subarray(0, 32).equals(logical.subarray(0, 32))).toBe(true); + expect(rebuilt.subarray(280, 300).equals(nextHeader)).toBe(true); + }); + + it("fetches an encoded header's packed bytes from whichever middle volume holds them", async () => { + // 3 volumes of 100. Packed header bytes at logical 150 (volume 2), index at 270 (volume 3). + // kEncodedHeader, kPackInfo, PackPos=118, NumStreams=1, kSize, PackSize=24 + const encHeader = Buffer.from([0x17, 0x06, 0x76, 0x01, 0x09, 0x18]); + const packBytes = Buffer.alloc(24, 0x77); + const logical = buildLogical7z({ + total: 300, + nextHeaderStart: 270, + nextHeader: encHeader, + pack: { start: 150, bytes: packBytes }, + }); + const { parts, read, reads } = volumeSet(splitIntoVolumes(logical, [100, 100, 100])); + + const sparse = await planSevenZSparseParts(parts, read); + expect(sparse).not.toBeNull(); + expect(reads).toEqual([ + { fileId: "v1", offset: 0, length: 32 }, + { fileId: "v3", offset: 70, length: 6 }, // index in the last volume + { fileId: "v2", offset: 50, length: 24 }, // packed header in the middle volume + ]); + expect(sparse!).toHaveLength(3); + const rebuilt = reconstruct(sparse!); + expect(rebuilt.subarray(150, 174).equals(packBytes)).toBe(true); + expect(rebuilt.subarray(270, 276).equals(encHeader)).toBe(true); + }); + + it("splits a header range that straddles a volume boundary into two reads", async () => { + // Index is 40 bytes starting at logical 180: last 20 bytes of volume 2 (100..200) + // and first 20 bytes of volume 3. + const nextHeader = Buffer.concat([Buffer.from([0x01]), Buffer.alloc(39, 0x22)]); + const logical = buildLogical7z({ total: 300, nextHeaderStart: 180, nextHeader }); + const { parts, read, reads } = volumeSet(splitIntoVolumes(logical, [100, 100, 100])); + + const sparse = await planSevenZSparseParts(parts, read); + expect(sparse).not.toBeNull(); + expect(reads).toEqual([ + { fileId: "v1", offset: 0, length: 32 }, + { fileId: "v2", offset: 80, length: 20 }, + { fileId: "v3", offset: 0, length: 20 }, + ]); + // Byte-exact across the seam. + expect(reconstruct(sparse!).subarray(180, 220).equals(nextHeader)).toBe(true); + }); + + it("splits an encoded header's packed bytes across a volume boundary", async () => { + // PackPos=58 -> packStart 90, PackSize=30 -> 90..120 straddles volumes 1|2. + const encHeader = Buffer.from([0x17, 0x06, 0x3a, 0x01, 0x09, 0x1e]); + const packBytes = Buffer.alloc(30, 0x99); + const logical = buildLogical7z({ + total: 300, + nextHeaderStart: 290, + nextHeader: encHeader, + pack: { start: 90, bytes: packBytes }, + }); + const { parts, read, reads } = volumeSet(splitIntoVolumes(logical, [100, 100, 100])); + + const sparse = await planSevenZSparseParts(parts, read); + expect(sparse).not.toBeNull(); + expect(reads).toEqual([ + { fileId: "v1", offset: 0, length: 32 }, + { fileId: "v3", offset: 90, length: 6 }, + { fileId: "v1", offset: 90, length: 10 }, + { fileId: "v2", offset: 0, length: 20 }, + ]); + expect(reconstruct(sparse!).subarray(90, 120).equals(packBytes)).toBe(true); + }); + + it("keeps a single-volume .7z reading exactly as before", async () => { + const nextHeader = Buffer.concat([Buffer.from([0x01]), Buffer.alloc(19, 0x33)]); + const logical = buildLogical7z({ total: 300, nextHeaderStart: 280, nextHeader }); + const { parts, read, reads } = volumeSet([logical]); + + const sparse = await planSevenZSparseParts(parts, read); + expect(reads).toEqual([ + { fileId: "v1", offset: 0, length: 32 }, + { fileId: "v1", offset: 280, length: 20 }, + ]); + expect(sparse!).toHaveLength(1); + expect(sparse![0].size).toBe(300); + expect(reconstruct(sparse!).subarray(280, 300).equals(nextHeader)).toBe(true); + }); + + it("returns null when the next-header offset points past the whole set", async () => { + const nextHeader = Buffer.from([0x01, 0x00]); + // Claim the index lives at 5000 while the set totals only 300 bytes. + const logical = buildLogical7z({ total: 300, nextHeaderStart: 280, nextHeader }); + logical.writeBigUInt64LE(BigInt(5000 - 32), 12); + const { parts, read } = volumeSet(splitIntoVolumes(logical, [200, 100])); + expect(await planSevenZSparseParts(parts, read)).toBeNull(); + }); + + it("returns null on an unrecognized next-header type", async () => { + const logical = buildLogical7z({ + total: 300, + nextHeaderStart: 280, + nextHeader: Buffer.from([0x42, 0x00]), + }); + const { parts, read } = volumeSet(splitIntoVolumes(logical, [200, 100])); + expect(await planSevenZSparseParts(parts, read)).toBeNull(); + }); + + it("returns null when the first volume has no 7z signature", async () => { + const logical = Buffer.alloc(300, 0x00); + const { parts, read } = volumeSet(splitIntoVolumes(logical, [200, 100])); + expect(await planSevenZSparseParts(parts, read)).toBeNull(); + }); + + it("returns null for an empty part list", async () => { + expect(await planSevenZSparseParts([], async () => Buffer.alloc(0))).toBeNull(); + }); +}); describe("read7zNumber", () => { it("reads a single-byte number", () => { diff --git a/worker/src/archive/ranged/sevenz-ranged.ts b/worker/src/archive/ranged/sevenz-ranged.ts index 0ca9d7a..3ca5ddf 100644 --- a/worker/src/archive/ranged/sevenz-ranged.ts +++ b/worker/src/archive/ranged/sevenz-ranged.ts @@ -1,6 +1,6 @@ import type { FileEntry } from "../zip-reader.js"; import { read7zContents } from "../sevenz-reader.js"; -import { listFromSparse } from "./sparse-list.js"; +import { listFromSparse, type SparsePart } from "./sparse-list.js"; import type { RangeReader } from "./range-reader.js"; import { childLogger } from "../../util/logger.js"; @@ -13,6 +13,8 @@ const K_ENCODED_HEADER = 0x17; const K_PACK_INFO = 0x06; const K_SIZE = 0x09; +const SIG_HEADER_BYTES = 32; + /** Read a 7z variable-length number: first byte is a length mask, followed by * little-endian bytes. Math.pow keeps values exact above 2^31. */ export function read7zNumber(buf: Buffer, pos: number): { value: number; next: number } { @@ -61,7 +63,7 @@ export function locate7zEncodedHeaderPack( export function parseSevenZSignatureHeader( buf: Buffer, ): { nextHeaderOffset: number; nextHeaderSize: number } | null { - if (buf.length < 32) return null; + if (buf.length < SIG_HEADER_BYTES) return null; if (!buf.subarray(0, 6).equals(SEVENZ_MAGIC)) return null; return { nextHeaderOffset: Number(buf.readBigUInt64LE(12)), @@ -71,42 +73,158 @@ export function parseSevenZSignatureHeader( export interface RangedPart { fileId: string; fileSize: bigint; fileName: string } +/** One volume's share of a whole-archive byte range. */ +export interface VolumeSlice { partIndex: number; offset: number; length: number } + +/** + * A `.7z.001`/`.7z.002`/… set produced by 7-Zip's `-v` switch is a **raw byte + * split** of one logical `.7z` file, not a ZIP-style spanned archive with + * per-volume structure: `cat pack.7z.00*` reproduces the original archive + * byte-for-byte. So every offset in the 7z headers is an offset into the + * concatenation of the volumes, and the only correct way to read them is to + * treat the set as one logical byte stream. + * + * That matters most for the next header (the archive index), which a 7z file + * keeps at its *end* — i.e. in the **last** volume, never the first. Reading + * only `parts[0]` therefore fails for every multi-volume set. + * + * Map a whole-archive `[start, start + length)` range onto per-volume reads, + * splitting it when it straddles a volume boundary. Returns null when the + * range falls outside the concatenated stream. + */ +export function mapRangeToVolumes( + sizes: number[], + start: number, + length: number, +): VolumeSlice[] | null { + const total = sizes.reduce((sum, s) => sum + s, 0); + if (!Number.isFinite(start) || !Number.isFinite(length)) return null; + if (start < 0 || length < 0 || start + length > total) return null; + + const slices: VolumeSlice[] = []; + let pos = start; + let remaining = length; + let base = 0; + for (let i = 0; i < sizes.length && remaining > 0; i++) { + const end = base + sizes[i]; + if (pos < end) { + const offset = pos - base; + const take = Math.min(remaining, sizes[i] - offset); + if (take > 0) { + slices.push({ partIndex: i, offset, length: take }); + pos += take; + remaining -= take; + } + } + base = end; + } + return remaining === 0 ? slices : null; +} + +/** + * Fetch the 7z header regions of a (possibly multi-volume) archive and return + * them as per-volume sparse reconstructions — the header bytes at their real + * offsets, file payloads left as zero holes. + * + * Exported for testing: it is the whole-archive-offset mapping that is worth + * asserting, and the final `7z l` step needs the real binary. + */ +export async function planSevenZSparseParts( + parts: RangedPart[], + read: RangeReader, +): Promise { + const first = parts[0]; + if (!first) { + log.warn({ partCount: parts.length }, "ranged 7z listing aborted — no parts supplied"); + return null; + } + const sizes = parts.map((p) => Number(p.fileSize)); + const total = sizes.reduce((sum, s) => sum + s, 0); + // Log context shared by every bail-out below; the first part names the set. + const ctx = { fileId: first.fileId, fileName: first.fileName, partCount: parts.length, total }; + const regions: { offset: number; bytes: Buffer }[][] = parts.map(() => []); + + /** + * Read a whole-archive range, recording each volume's slice as a region so + * the sparse reconstruction places the bytes where 7z expects them. + */ + const fetchLogical = async (start: number, length: number, what: string): Promise => { + const slices = mapRangeToVolumes(sizes, start, length); + if (!slices) { + log.warn({ ...ctx, what, start, length, sizes }, `ranged 7z listing aborted — ${what} region out of bounds`); + return null; + } + const chunks: Buffer[] = []; + for (const s of slices) { + const part = parts[s.partIndex]; + const bytes = await read(part.fileId, s.offset, s.length, part.fileSize); + if (bytes.length < s.length) { + log.warn( + { ...ctx, what, volume: s.partIndex + 1, volumeFileId: part.fileId, offset: s.offset, wanted: s.length, got: bytes.length }, + `ranged 7z listing aborted — short read on ${what}`, + ); + return null; + } + regions[s.partIndex].push({ offset: s.offset, bytes }); + chunks.push(bytes); + } + return Buffer.concat(chunks); + }; + + try { + // The signature header is at the very start of volume 1. + const sig = await fetchLogical(0, Math.min(SIG_HEADER_BYTES, total), "signature-header"); + if (!sig) return null; + const parsed = parseSevenZSignatureHeader(sig); + if (!parsed) { + log.warn({ ...ctx, head: sig.subarray(0, 16).toString("hex") }, "ranged 7z listing aborted — signature header did not parse"); + return null; + } + + // NextHeaderOffset is measured from the end of the signature header, into + // the concatenated stream — so this normally lands in the LAST volume. + const endStart = SIG_HEADER_BYTES + parsed.nextHeaderOffset; + if (parsed.nextHeaderSize <= 0) { + log.warn({ ...ctx, endStart, nextHeaderSize: parsed.nextHeaderSize }, "ranged 7z listing aborted — empty next header (no index to read)"); + return null; + } + const endHeader = await fetchLogical(endStart, parsed.nextHeaderSize, "next-header"); + if (!endHeader) return null; + + const headerType = endHeader[0]; + if (headerType === K_ENCODED_HEADER) { + // Compressed header: its packed bytes live mid-stream, not at EOF, so + // they may sit in any volume — or straddle two. + const pack = locate7zEncodedHeaderPack(endHeader); + if (!pack) { + log.warn( + { ...ctx, endStart, nextHeaderSize: parsed.nextHeaderSize, head: endHeader.subarray(0, 16).toString("hex") }, + "ranged 7z listing aborted — encoded-header PackInfo did not parse", + ); + return null; + } + const packStart = SIG_HEADER_BYTES + pack.packPos; + const packBytes = await fetchLogical(packStart, pack.packSize, "packed-header"); + if (!packBytes) return null; + } else if (headerType !== K_HEADER) { + log.warn({ ...ctx, endStart, headerType }, "ranged 7z listing aborted — unknown next-header type"); + return null; + } + + return parts.map((p, i) => ({ fileName: p.fileName, size: sizes[i], regions: regions[i] })); + } catch (err) { + log.warn({ err, ...ctx }, "ranged 7z listing failed"); + return null; + } +} + export async function readSevenZListingRanged( parts: RangedPart[], read: RangeReader, ): Promise { - const part = parts[0]; - if (!part) return null; - const size = Number(part.fileSize); - try { - const sig = await read(part.fileId, 0, 32, part.fileSize); - const parsed = parseSevenZSignatureHeader(sig); - if (!parsed) return null; - const endStart = 32 + parsed.nextHeaderOffset; - if (endStart < 0 || endStart + parsed.nextHeaderSize > size) return null; - const endHeader = await read(part.fileId, endStart, parsed.nextHeaderSize, part.fileSize); - - const regions = [ - { offset: 0, bytes: sig }, - { offset: endStart, bytes: endHeader }, - ]; - - const headerType = endHeader[0]; - if (headerType === K_ENCODED_HEADER) { - // Compressed header: its packed bytes live mid-file, not at EOF. Fetch them. - const pack = locate7zEncodedHeaderPack(endHeader); - if (!pack) return null; - const packStart = 32 + pack.packPos; - if (packStart < 0 || packStart + pack.packSize > size) return null; - const packBytes = await read(part.fileId, packStart, pack.packSize, part.fileSize); - regions.push({ offset: packStart, bytes: packBytes }); - } else if (headerType !== K_HEADER) { - return null; // unknown next-header type - } - - return listFromSparse([{ fileName: part.fileName, size, regions }], read7zContents); - } catch (err) { - log.warn({ err, fileId: part.fileId }, "ranged 7z listing failed"); - return null; - } + const sparseParts = await planSevenZSparseParts(parts, read); + if (!sparseParts) return null; + // 7-Zip opens `pack.7z.001` as a split archive and concatenates the set + // itself, so the whole reconstructed set must be on disk, not just part 1. + return listFromSparse(sparseParts, read7zContents); }