merge: scoped, ranged-first file-list repair for spanned ZIP sets

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

worker/src/archive/central-directory.ts

@@ -11,13 +11,21 @@ function extOf(name: string): string | null {
   return e === "" ? null : e;
 }
 
+/**
+ * Locate the End Of Central Directory record by scanning backward for its
+ * signature. Returns -1 when the buffer holds no EOCD.
+ */
+export function findEocdOffset(tail: Buffer): number {
+  for (let i = tail.length - 22; i >= 0; i--) {
+    if (tail.readUInt32LE(i) === EOCD_SIG) return i;
+  }
+  return -1;
+}
+
 /** Parse a ZIP central directory from the tail of an archive. */
 export function parseZipCentralDirectoryFromTail(tail: Buffer, tailStart: number): FileEntry[] {
   // 1. Find EOCD by scanning backward for its signature.
-  let eocd = -1;
+  const eocd = findEocdOffset(tail);
-  for (let i = tail.length - 22; i >= 0; i--) {
-    if (tail.readUInt32LE(i) === EOCD_SIG) { eocd = i; break; }
-  }
   if (eocd < 0) throw new RangeError("EOCD not found in tail");
 
   let cdSize = tail.readUInt32LE(eocd + 12);
@@ -45,10 +53,22 @@ export function parseZipCentralDirectoryFromTail(tail: Buffer, tailStart: number
   }
 
   // 3. Walk central-directory headers.
+  return walkCentralDirectory(tail, cdLocal, cdSize);
+}
+
+/**
+ * Walk a run of central-directory file headers and return the file entries
+ * (directory entries are skipped).
+ *
+ * `buf` must contain the whole central directory starting at `start`; for a
+ * spanned archive that means the caller has already stitched together the
+ * volumes the directory straddles.
+ */
+export function walkCentralDirectory(tail: Buffer, start: number, cdSize: number): FileEntry[] {
   const entries: FileEntry[] = [];
-  let p = cdLocal;
+  let p = start;
-  const end = cdLocal + cdSize;
+  const end = start + cdSize;
-  while (p + 46 <= end && tail.readUInt32LE(p) === CD_SIG) {
+  while (p + 46 <= end && p + 46 <= tail.length && tail.readUInt32LE(p) === CD_SIG) {
     let crc = tail.readUInt32LE(p + 16) >>> 0;
     let comp = BigInt(tail.readUInt32LE(p + 20));
     let uncomp = BigInt(tail.readUInt32LE(p + 24));

worker/src/archive/detect.test.ts

@@ -1,7 +1,7 @@
 import { describe, it, expect } from "vitest";
 import { detectArchive, isArchiveAttachment } from "./detect.js";
 
-describe("detectArchive — 7z numbered multipart (pack.7z.001, pack.7z.002, ...)", () => {
+describe("detectArchive — numbered volumes (pack.EXT.001, pack.EXT.002, ...)", () => {
   it("recognizes a 7z multipart part as an archive attachment", () => {
     expect(isArchiveAttachment("Lost Adventures Vol2.7z.001")).toBe(true);
   });
@@ -12,7 +12,7 @@ describe("detectArchive — 7z numbered multipart (pack.7z.001, pack.7z.002, ...
       baseName: "Lost Adventures Vol2.7z",
       partNumber: 1,
       format: "7Z",
-      pattern: "SEVENZ_NUMBERED",
+      pattern: "ARCHIVE_NUMBERED",
     });
   });
 
@@ -36,4 +36,249 @@ describe("detectArchive — 7z numbered multipart (pack.7z.001, pack.7z.002, ...
       pattern: "SINGLE",
     });
   });
+
+  it("recognizes numbered ZIP volumes", () => {
+    expect(detectArchive("Big Pack.zip.001")).toEqual({
+      baseName: "Big Pack.zip",
+      partNumber: 1,
+      format: "ZIP",
+      pattern: "ARCHIVE_NUMBERED",
+    });
+  });
+
+  it("recognizes numbered RAR volumes (previously dropped silently)", () => {
+    expect(detectArchive("Big Pack.rar.001")).toEqual({
+      baseName: "Big Pack.rar",
+      partNumber: 1,
+      format: "RAR",
+      pattern: "ARCHIVE_NUMBERED",
+    });
+  });
+
+  it("derives the format from the archive extension, not a hardcoded value", () => {
+    expect(detectArchive("A.zip.004")?.format).toBe("ZIP");
+    expect(detectArchive("A.RAR.004")?.format).toBe("RAR");
+    expect(detectArchive("A.7z.004")?.format).toBe("7Z");
+  });
+
+  it("accepts hand-renamed two-digit volumes for every format", () => {
+    expect(detectArchive("Pack.zip.01")).toEqual({
+      baseName: "Pack.zip",
+      partNumber: 1,
+      format: "ZIP",
+      pattern: "ARCHIVE_NUMBERED",
+    });
+    expect(detectArchive("Pack.rar.02")?.partNumber).toBe(2);
+    expect(detectArchive("Pack.7z.03")?.partNumber).toBe(3);
+  });
+
+  it("still accepts four-or-more-digit volumes", () => {
+    expect(detectArchive("Pack.7z.0001")).toEqual({
+      baseName: "Pack.7z",
+      partNumber: 1,
+      format: "7Z",
+      pattern: "ARCHIVE_NUMBERED",
+    });
+    expect(detectArchive("Pack.zip.10001")?.partNumber).toBe(10001);
+  });
+
+  it("does not match a single-digit suffix (too ambiguous with real extensions)", () => {
+    expect(detectArchive("Pack.zip.1")).toBeNull();
+  });
+});
+
+describe("detectArchive — pattern ordering safety", () => {
+  it("does not let ZIP_LEGACY swallow a .7z.NNN name", () => {
+    expect(detectArchive("Pack.7z.001")?.pattern).toBe("ARCHIVE_NUMBERED");
+    expect(detectArchive("Pack.7z.001")?.format).toBe("7Z");
+  });
+
+  it("does not let ARCHIVE_NUMBERED swallow legacy .zNN names", () => {
+    expect(detectArchive("Pack.z01")).toEqual({
+      baseName: "Pack",
+      partNumber: 1,
+      format: "ZIP",
+      pattern: "ZIP_LEGACY",
+    });
+  });
+
+  it("does not let ARCHIVE_NUMBERED swallow legacy .rNN names", () => {
+    expect(detectArchive("Pack.r00")).toEqual({
+      baseName: "Pack",
+      partNumber: 0,
+      format: "RAR",
+      pattern: "RAR_LEGACY",
+    });
+    expect(detectArchive("Pack.r01")?.pattern).toBe("RAR_LEGACY");
+  });
+
+  it("keeps .partN.rar on the RAR_PART pattern", () => {
+    expect(detectArchive("Pack.part2.rar")).toEqual({
+      baseName: "Pack",
+      partNumber: 2,
+      format: "RAR",
+      pattern: "RAR_PART",
+    });
+  });
+});
+
+describe("detectArchive — filename normalization", () => {
+  it("recognizes a name with a trailing space and reports the trimmed baseName", () => {
+    expect(detectArchive("Pack.zip ")).toEqual({
+      baseName: "Pack",
+      partNumber: -1,
+      format: "ZIP",
+      pattern: "SINGLE",
+    });
+  });
+
+  it("recognizes a name with a leading space", () => {
+    expect(detectArchive("  Pack.rar")).toEqual({
+      baseName: "Pack",
+      partNumber: -1,
+      format: "RAR",
+      pattern: "SINGLE",
+    });
+  });
+
+  it("recognizes a multipart name with surrounding whitespace", () => {
+    expect(detectArchive("\tPack.rar.002 \n")).toEqual({
+      baseName: "Pack.rar",
+      partNumber: 2,
+      format: "RAR",
+      pattern: "ARCHIVE_NUMBERED",
+    });
+  });
+
+  it("recognizes a name with a trailing dot", () => {
+    expect(detectArchive("Pack.zip.")).toEqual({
+      baseName: "Pack",
+      partNumber: -1,
+      format: "ZIP",
+      pattern: "SINGLE",
+    });
+  });
+
+  it("recognizes a name with a trailing dot followed by a space", () => {
+    expect(detectArchive("Pack.part3.rar. ")?.pattern).toBe("RAR_PART");
+  });
+
+  it("still returns null for a whitespace-only or empty name", () => {
+    expect(detectArchive("   ")).toBeNull();
+    expect(detectArchive("")).toBeNull();
+  });
+});
+
+describe("detectArchive — self-extracting RAR first volume (.partN.exe)", () => {
+  it("recognizes Pack.part1.exe as the first volume of a RAR_PART set", () => {
+    expect(detectArchive("Pack.part1.exe")).toEqual({
+      baseName: "Pack",
+      partNumber: 1,
+      format: "RAR",
+      pattern: "RAR_PART",
+    });
+  });
+
+  it("groups the SFX first volume with its .rar continuation volumes", () => {
+    const sfx = detectArchive("Pack.part1.exe");
+    const cont = detectArchive("Pack.part2.rar");
+    expect(sfx?.baseName).toBe(cont?.baseName);
+    expect(sfx?.format).toBe(cont?.format);
+  });
+
+  it("does not open the door to arbitrary .exe attachments", () => {
+    expect(detectArchive("Installer.exe")).toBeNull();
+    expect(detectArchive("Pack.exe")).toBeNull();
+  });
+});
+
+describe("detectArchive — standalone documents", () => {
+  it("keeps recognizing the pre-existing document extensions", () => {
+    expect(detectArchive("Model.stl")).toEqual({
+      baseName: "Model",
+      partNumber: -1,
+      format: "DOCUMENT",
+      pattern: "SINGLE",
+    });
+    expect(detectArchive("Sheet.pdf")?.format).toBe("DOCUMENT");
+  });
+
+  it("recognizes slicer-project formats", () => {
+    for (const name of [
+      "Bust.lys",
+      "Bust.lyt",
+      "Bust.lymesh",
+      "Bust.chitubox",
+      "Bust.ctp",
+      "Bust.ctb",
+      "Bust.cbddlp",
+      "Bust.photon",
+      "Bust.pwmx",
+      "Bust.pwmo",
+      "Bust.pws",
+      "Bust.sl1",
+      "Bust.goo",
+      "Bust.phz",
+      "Bust.pm3",
+      "Bust.form",
+    ]) {
+      expect(detectArchive(name), name).toEqual({
+        baseName: "Bust",
+        partNumber: -1,
+        format: "DOCUMENT",
+        pattern: "SINGLE",
+      });
+    }
+  });
+
+  it("recognizes 3D-model and CAD formats", () => {
+    for (const name of [
+      "Bust.fbx",
+      "Bust.ply",
+      "Bust.glb",
+      "Bust.gltf",
+      "Bust.3ds",
+      "Bust.max",
+      "Bust.c4d",
+      "Bust.ztl",
+      "Bust.zpr",
+      "Bust.mtl",
+      "Bust.f3d",
+      "Bust.scad",
+      "Bust.igs",
+      "Bust.iges",
+      "Bust.sldprt",
+      "Bust.skp",
+      "Bust.wrl",
+    ]) {
+      expect(detectArchive(name), name).toEqual({
+        baseName: "Bust",
+        partNumber: -1,
+        format: "DOCUMENT",
+        pattern: "SINGLE",
+      });
+    }
+  });
+
+  it("recognizes the .blend1 autosave sibling of .blend", () => {
+    expect(detectArchive("Scene.blend")?.format).toBe("DOCUMENT");
+    expect(detectArchive("Scene.blend1")).toEqual({
+      baseName: "Scene",
+      partNumber: -1,
+      format: "DOCUMENT",
+      pattern: "SINGLE",
+    });
+  });
+
+  it("does NOT recognize image attachments (they must not become packages)", () => {
+    for (const name of ["Preview.jpg", "Preview.jpeg", "Preview.png", "Preview.webp", "Preview.gif"]) {
+      expect(detectArchive(name), name).toBeNull();
+    }
+  });
+
+  it("returns null for unrelated files", () => {
+    expect(detectArchive("notes.txt")).toBeNull();
+    expect(detectArchive("song.mp3")).toBeNull();
+    expect(detectArchive("noextension")).toBeNull();
+  });
 });

worker/src/archive/detect.ts

@@ -4,25 +4,29 @@ export interface MultipartInfo {
   baseName: string;
   partNumber: number;
   format: ArchiveFormat;
-  pattern: "ZIP_NUMBERED" | "ZIP_LEGACY" | "RAR_PART" | "RAR_LEGACY" | "SEVENZ_NUMBERED" | "SINGLE";
+  pattern: "ARCHIVE_NUMBERED" | "ZIP_LEGACY" | "RAR_PART" | "RAR_LEGACY" | "SINGLE";
 }
 
 const patterns: {
   regex: RegExp;
-  format: ArchiveFormat;
+  /** A fixed format, or one derived from the match for patterns spanning several formats. */
+  format: ArchiveFormat | ((match: RegExpMatchArray) => ArchiveFormat);
   pattern: MultipartInfo["pattern"];
   getBaseName: (match: RegExpMatchArray) => string;
   getPartNumber: (match: RegExpMatchArray) => number;
 }[] = [
-  // pack.zip.001, pack.zip.002
+  // pack.zip.001, pack.rar.001, pack.7z.001 (numbered volume split — one pattern for
+  // every format, so a new format can never be silently dropped for lack of its own entry).
+  // {2,} digits also picks up hand-renamed sets like pack.rar.01.
   {
-    regex: /^(.+\.zip)\.(\d{3,})$/i,
+    regex: /^(.+\.(zip|7z|rar))\.(\d{2,})$/i,
-    format: "ZIP",
+    // The regex only ever captures zip/7z/rar, so uppercasing yields a valid ArchiveFormat.
-    pattern: "ZIP_NUMBERED",
+    format: (m) => m[2].toUpperCase() as ArchiveFormat,
-    getBaseName: (m) => m[1],
+    pattern: "ARCHIVE_NUMBERED",
-    getPartNumber: (m) => parseInt(m[2], 10),
+    getBaseName: (m) => m[1], // includes the archive extension
+    getPartNumber: (m) => parseInt(m[3], 10),
   },
-  // pack.z01, pack.z02 (legacy split — final part is pack.zip)
+  // pack.z01, pack.z02 (legacy split — pack.zip is the FINAL disk of the set)
   {
     regex: /^(.+)\.z(\d{2,})$/i,
     format: "ZIP",
@@ -30,15 +34,16 @@ const patterns: {
     getBaseName: (m) => m[1],
     getPartNumber: (m) => parseInt(m[2], 10),
   },
-  // pack.part1.rar, pack.part2.rar
+  // pack.part1.rar, pack.part2.rar — .exe covers a self-extracting first volume
+  // (pack.part1.exe + pack.part2.rar + ...), which is still a RAR volume set.
   {
-    regex: /^(.+)\.part(\d+)\.rar$/i,
+    regex: /^(.+)\.part(\d+)\.(rar|exe)$/i,
     format: "RAR",
     pattern: "RAR_PART",
     getBaseName: (m) => m[1],
     getPartNumber: (m) => parseInt(m[2], 10),
   },
-  // pack.r00, pack.r01 (legacy split — final part is pack.rar)
+  // pack.r00, pack.r01 (legacy split — pack.rar is the FIRST volume, .r00 onwards follow it)
   {
     regex: /^(.+)\.r(\d{2,})$/i,
     format: "RAR",
@@ -46,50 +51,53 @@ const patterns: {
     getBaseName: (m) => m[1],
     getPartNumber: (m) => parseInt(m[2], 10),
   },
-  // pack.7z.001, pack.7z.002 (native 7z volume split)
-  {
-    regex: /^(.+\.7z)\.(\d{3,})$/i,
-    format: "7Z",
-    pattern: "SEVENZ_NUMBERED",
-    getBaseName: (m) => m[1],
-    getPartNumber: (m) => parseInt(m[2], 10),
-  },
 ];
 
-/** Extensions we recognize as fetchable documents (archives + standalone files) */
+/** Extensions we recognize as fetchable documents (archives + standalone files).
-const DOCUMENT_EXTENSIONS = /\.(pdf|stl|obj|3mf|step|stp|blend|gcode|svg|dxf|ai|eps|psd)$/i;
+ *  Deliberately excludes image formats — previews posted as uncompressed documents
+ *  must go through the photo-matching path, not become packages of their own. */
+const DOCUMENT_EXTENSIONS =
+  /\.(pdf|stl|obj|3mf|step|stp|blend1|blend|gcode|svg|dxf|ai|eps|psd|lys|lyt|lymesh|chitubox|ctp|ctb|cbddlp|photon|pwmx|pwmo|pws|sl1|goo|phz|pm3|fbx|ply|glb|gltf|3ds|max|c4d|ztl|zpr|mtl|f3d|scad|igs|iges|sldprt|form|skp|wrl)$/i;
 
 /**
  * Detect if a filename is an archive and extract multipart info.
  */
 export function detectArchive(fileName: string): MultipartInfo | null {
+  // TDLib hands us `document.file_name` verbatim and every pattern below is `$`-anchored,
+  // so "Pack.zip " or "Pack.zip." would otherwise be dropped without a trace. Trailing dots
+  // are stripped too: no filesystem or archiver can produce a meaningful one (Windows
+  // silently drops them), so a trailing dot is always cosmetic damage from a re-upload,
+  // never part of the real name. The normalized value is used for matching AND baseName.
+  const name = fileName.trim().replace(/[.\s]+$/, "");
+  if (!name) return null;
+
   // Check multipart patterns first
   for (const p of patterns) {
-    const match = fileName.match(p.regex);
+    const match = name.match(p.regex);
     if (match) {
       return {
         baseName: p.getBaseName(match),
         partNumber: p.getPartNumber(match),
-        format: p.format,
+        format: typeof p.format === "function" ? p.format(match) : p.format,
         pattern: p.pattern,
       };
     }
   }
 
   // Single .zip file — could be a standalone or the final part of a ZIP_LEGACY set
-  if (/\.zip$/i.test(fileName)) {
+  if (/\.zip$/i.test(name)) {
     return {
-      baseName: fileName.replace(/\.zip$/i, ""),
+      baseName: name.replace(/\.zip$/i, ""),
       partNumber: -1, // -1 signals "could be single or final legacy part"
       format: "ZIP",
       pattern: "SINGLE",
     };
   }
 
-  // Single .rar file — could be standalone or final part of RAR_LEGACY set
+  // Single .rar file — could be standalone or the FIRST part of a RAR_LEGACY set
-  if (/\.rar$/i.test(fileName)) {
+  if (/\.rar$/i.test(name)) {
     return {
-      baseName: fileName.replace(/\.rar$/i, ""),
+      baseName: name.replace(/\.rar$/i, ""),
       partNumber: -1,
       format: "RAR",
       pattern: "SINGLE",
@@ -97,20 +105,19 @@ export function detectArchive(fileName: string): MultipartInfo | null {
   }
 
   // Single .7z file
-  if (/\.7z$/i.test(fileName)) {
+  if (/\.7z$/i.test(name)) {
     return {
-      baseName: fileName.replace(/\.7z$/i, ""),
+      baseName: name.replace(/\.7z$/i, ""),
       partNumber: -1,
       format: "7Z",
       pattern: "SINGLE",
     };
   }
 
-  // Standalone documents (PDFs, STLs, 3D files, etc.)
+  // Standalone documents (PDFs, STLs, 3D files, slicer projects, etc.)
-  if (DOCUMENT_EXTENSIONS.test(fileName)) {
+  if (DOCUMENT_EXTENSIONS.test(name)) {
-    const ext = fileName.match(DOCUMENT_EXTENSIONS)![0];
     return {
-      baseName: fileName.replace(DOCUMENT_EXTENSIONS, ""),
+      baseName: name.replace(DOCUMENT_EXTENSIONS, ""),
       partNumber: -1,
       format: "DOCUMENT",
       pattern: "SINGLE",

worker/src/archive/listing-plan.test.ts

@@ -0,0 +1,130 @@
+import { describe, it, expect } from "vitest";
+import {
+  planListingRead,
+  planRangedFallback,
+  classifySourceShape,
+  isConcatRepackName,
+  concatRepackBase,
+  concatChunkIndex,
+  isVolumeSet,
+} from "./listing-plan.js";
+
+const GB = 1024n * 1024n * 1024n;
+
+/** Defaults for a plan input; individual tests override what they care about. */
+function input(over: Partial<Parameters<typeof planListingRead>[0]>) {
+  return {
+    archiveType: "ZIP",
+    sourceFileName: "Pack.z01",
+    destFileNames: ["Pack.z01", "Pack.zip"],
+    totalSize: 10n * GB,
+    maxDownloadBytes: 200n * GB,
+    rangedOnly: false,
+    ...over,
+  };
+}
+
+describe("classifySourceShape", () => {
+  it("separates spanned ZIP volumes from a raw byte split", () => {
+    expect(classifySourceShape("Pack.z01")).toBe("spanned-zip");
+    expect(classifySourceShape("Pack.z12")).toBe("spanned-zip");
+    expect(classifySourceShape("Pack.zip.001")).toBe("byte-split");
+    expect(classifySourceShape("Pack.7z.001")).toBe("byte-split");
+  });
+
+  it("recognizes RAR volume sets and lone archives", () => {
+    expect(classifySourceShape("Pack.part1.rar")).toBe("rar-volume-set");
+    expect(classifySourceShape("Pack.r00")).toBe("rar-volume-set");
+    expect(classifySourceShape("Pack.zip")).toBe("single");
+    expect(classifySourceShape("notes.txt")).toBe("unknown");
+  });
+
+  it("marks exactly the layouts whose volumes are independent containers", () => {
+    expect(isVolumeSet("spanned-zip")).toBe(true);
+    expect(isVolumeSet("rar-volume-set")).toBe(true);
+    expect(isVolumeSet("byte-split")).toBe(false);
+    expect(isVolumeSet("single")).toBe(false);
+  });
+});
+
+describe("concat repack naming", () => {
+  it("recognizes the repack chunk names the uploader produces", () => {
+    expect(isConcatRepackName("Pack.concat.001")).toBe(true);
+    expect(isConcatRepackName("Pack.concat.017")).toBe(true);
+    expect(isConcatRepackName("Pack.concat")).toBe(true);
+    expect(isConcatRepackName("Pack.z01")).toBe(false);
+    expect(isConcatRepackName("Pack.zip.001")).toBe(false);
+    // "concat" appearing mid-name must not count
+    expect(isConcatRepackName("concat-models.zip")).toBe(false);
+  });
+
+  it("groups and orders chunks of one repack", () => {
+    expect(concatRepackBase("Pack.concat.002")).toBe("pack.concat");
+    expect(concatRepackBase("Pack.concat")).toBe("pack.concat");
+    expect(concatChunkIndex("Pack.concat.017")).toBe(17);
+    expect(concatChunkIndex("Pack.concat")).toBe(0);
+  });
+});
+
+describe("planListingRead", () => {
+  it("routes a spanned ZIP set to the ranged read", () => {
+    const plan = planListingRead(input({}));
+    expect(plan.route).toBe("ranged");
+    expect(plan.reason).toContain("spanned-zip");
+  });
+
+  it("skips a concatenated spanned ZIP set — no reader can ever list it", () => {
+    const plan = planListingRead(
+      input({ sourceFileName: "Pack.z01", destFileNames: ["Pack.concat.001", "Pack.concat.002"] })
+    );
+    expect(plan.route).toBe("skip");
+    expect(plan.reason).toContain("not a valid archive");
+  });
+
+  it("skips a concatenated RAR volume set for the same reason", () => {
+    const plan = planListingRead(
+      input({
+        archiveType: "RAR",
+        sourceFileName: "Pack.part1.rar",
+        destFileNames: ["Pack.concat.001", "Pack.concat.002"],
+      })
+    );
+    expect(plan.route).toBe("skip");
+    expect(plan.reason).toContain("rar-volume-set");
+  });
+
+  it("still reads a concatenated BYTE SPLIT — re-cutting one stream is lossless", () => {
+    const plan = planListingRead(
+      input({ sourceFileName: "Pack.zip.001", destFileNames: ["Pack.concat.001", "Pack.concat.002"] })
+    );
+    expect(plan.route).toBe("ranged");
+    expect(plan.reason).toContain("byte split");
+  });
+
+  it("skips archive types with no file-list reader without touching the API", () => {
+    expect(planListingRead(input({ archiveType: "DOCUMENT" }))).toMatchObject({ route: "skip" });
+  });
+
+  it("skips when no destination part could be resolved", () => {
+    expect(planListingRead(input({ destFileNames: [] }))).toMatchObject({ route: "skip" });
+  });
+});
+
+describe("planRangedFallback", () => {
+  it("refuses to download when rangedOnly is set, however large the archive", () => {
+    const plan = planRangedFallback({ totalSize: 35n * GB, maxDownloadBytes: 200n * GB, rangedOnly: true });
+    expect(plan.route).toBe("skip");
+    expect(plan.reason).toContain("rangedOnly");
+  });
+
+  it("refuses to download past the size cap", () => {
+    const plan = planRangedFallback({ totalSize: 300n * GB, maxDownloadBytes: 200n * GB, rangedOnly: false });
+    expect(plan.route).toBe("skip");
+    expect(plan.reason).toContain("exceeds the download cap");
+  });
+
+  it("falls back to a download when it is allowed and affordable", () => {
+    const plan = planRangedFallback({ totalSize: 2n * GB, maxDownloadBytes: 200n * GB, rangedOnly: false });
+    expect(plan.route).toBe("download");
+  });
+});

worker/src/archive/listing-plan.ts

@@ -0,0 +1,164 @@
+import { detectArchive } from "./detect.js";
+
+/**
+ * Decide *how* to read an already-uploaded archive's inner-file listing before
+ * spending a single byte of Telegram traffic on it.
+ *
+ * Three outcomes matter:
+ *
+ *  - `ranged`   — read only the header/tail bytes that carry the listing.
+ *                 Tens of kilobytes regardless of archive size.
+ *  - `download` — the ranged read can't work (or already failed); pull the whole
+ *                 archive down and let the on-disk reader handle it.
+ *  - `skip`     — no reader can ever list this destination copy. Saying so
+ *                 up-front is the whole point: the alternative is burning API
+ *                 calls and bandwidth on something structurally unreadable.
+ *
+ * The `skip` case that motivated this module: when any source volume exceeded
+ * the upload cap, the ingestion worker concatenated every volume into one file
+ * and re-split it into uniform `<base>.concat.NNN` chunks. For a *byte split*
+ * (`pack.zip.001`, …) that round-trips fine — the bytes are the same stream.
+ * For a ZIP-spec **spanned** set (`pack.z01`, …, `pack.zip`) or a RAR volume
+ * set it does not: those volumes are separate containers, and their
+ * concatenation is not a valid archive in any format. The destination copy of
+ * such a package is permanently unlistable, and no amount of downloading will
+ * change that.
+ */
+
+/** `<base>.concat`, `<base>.concat.001`, … — the re-split repack naming. */
+const CONCAT_REPACK_RE = /\.concat(?:\.\d{2,})?$/i;
+
+export function isConcatRepackName(fileName: string): boolean {
+  return CONCAT_REPACK_RE.test(fileName.trim().replace(/[.\s]+$/, ""));
+}
+
+/** Strip the `.NNN` chunk suffix, so every chunk of one repack shares a key. */
+export function concatRepackBase(fileName: string): string {
+  return fileName
+    .trim()
+    .replace(/[.\s]+$/, "")
+    .replace(/\.(\d{2,})$/, "")
+    .toLowerCase();
+}
+
+/** Numeric chunk index of a `<base>.concat.NNN` name; 0 for a bare `.concat`. */
+export function concatChunkIndex(fileName: string): number {
+  const m = fileName.trim().replace(/[.\s]+$/, "").match(/\.(\d{2,})$/);
+  return m ? parseInt(m[1], 10) : 0;
+}
+
+/**
+ * How the *source* archive was laid out, derived from the Package's own
+ * fileName. This is what decides whether a `.concat.NNN` repack is survivable:
+ * only a stream that was contiguous to begin with can be re-cut.
+ */
+export type SourceShape =
+  /** `.z01`, `.z02`, … + `.zip` — ZIP-spec spanned, one container per volume. */
+  | "spanned-zip"
+  /** `.partN.rar` or `.rNN` — RAR volume set, one container per volume. */
+  | "rar-volume-set"
+  /** `.zip.001`, `.7z.001`, … — one file cut into chunks. */
+  | "byte-split"
+  /** A lone `.zip` / `.rar` / `.7z` / document. */
+  | "single"
+  | "unknown";
+
+export function classifySourceShape(fileName: string): SourceShape {
+  const info = detectArchive(fileName);
+  if (!info) return "unknown";
+  switch (info.pattern) {
+    case "ZIP_LEGACY":
+      return "spanned-zip";
+    case "RAR_PART":
+    case "RAR_LEGACY":
+      return "rar-volume-set";
+    case "ARCHIVE_NUMBERED":
+      return "byte-split";
+    case "SINGLE":
+      return "single";
+  }
+}
+
+/** True for layouts whose volumes are independent containers, not one stream. */
+export function isVolumeSet(shape: SourceShape): boolean {
+  return shape === "spanned-zip" || shape === "rar-volume-set";
+}
+
+export type ListingRoute =
+  | { route: "ranged"; reason: string }
+  | { route: "download"; reason: string }
+  | { route: "skip"; reason: string };
+
+export interface ListingPlanInput {
+  /** Package.archiveType. */
+  archiveType: string;
+  /** Package.fileName — the *source* name, which encodes the original layout. */
+  sourceFileName: string;
+  /** File names of the resolved destination parts, in upload order. */
+  destFileNames: string[];
+  /** Total size of the destination parts. */
+  totalSize: bigint;
+  /** Cap on a full download (WORKER_MAX_ZIP_SIZE_MB, in bytes). */
+  maxDownloadBytes: bigint;
+  /** When true, the full-download fallback is off the table entirely. */
+  rangedOnly: boolean;
+}
+
+const RANGED_TYPES = new Set(["ZIP", "RAR", "SEVEN_Z"]);
+
+/**
+ * Pick the first route to try for a package. `download` is only returned when
+ * ranged reading is structurally impossible for the type; a ranged read that
+ * fails at runtime is handled by {@link planRangedFallback}.
+ */
+export function planListingRead(input: ListingPlanInput): ListingRoute {
+  if (!RANGED_TYPES.has(input.archiveType)) {
+    return { route: "skip", reason: `archiveType ${input.archiveType} has no file-list reader` };
+  }
+  if (input.destFileNames.length === 0) {
+    return { route: "skip", reason: "no destination parts could be resolved" };
+  }
+
+  const shape = classifySourceShape(input.sourceFileName);
+  const repacked = input.destFileNames.some(isConcatRepackName);
+
+  if (repacked && isVolumeSet(shape)) {
+    return {
+      route: "skip",
+      reason:
+        `destination copy is a .concat.NNN repack of a ${shape} — concatenated volumes ` +
+        "are not a valid archive, so no reader can ever list it",
+    };
+  }
+
+  if (repacked) {
+    // A re-cut byte split is still the original contiguous stream, so the
+    // ranged reader's whole-archive offset arithmetic applies unchanged.
+    return { route: "ranged", reason: `.concat.NNN repack of a ${shape} — readable as a byte split` };
+  }
+
+  return { route: "ranged", reason: `${shape} destination copy, ${input.destFileNames.length} part(s)` };
+}
+
+/**
+ * What to do once a ranged read has come back empty. Kept separate from
+ * {@link planListingRead} so the "we tried cheap and it didn't work" decision
+ * is explicit and testable rather than buried in a conditional.
+ */
+export function planRangedFallback(
+  input: Pick<ListingPlanInput, "totalSize" | "maxDownloadBytes" | "rangedOnly">
+): { route: "download"; reason: string } | { route: "skip"; reason: string } {
+  if (input.rangedOnly) {
+    return {
+      route: "skip",
+      reason: `ranged read failed and rangedOnly is set — not downloading ${input.totalSize} bytes`,
+    };
+  }
+  if (input.totalSize > input.maxDownloadBytes) {
+    return {
+      route: "skip",
+      reason: `ranged read failed and ${input.totalSize} bytes exceeds the download cap of ${input.maxDownloadBytes}`,
+    };
+  }
+  return { route: "download", reason: `ranged read failed — falling back to a ${input.totalSize} byte download` };
+}

worker/src/archive/multipart.test.ts

@@ -0,0 +1,72 @@
+import { describe, it, expect } from "vitest";
+import { groupArchiveSets, type TelegramMessage } from "./multipart.js";
+
+let nextId = 1000n;
+
+function msg(fileName: string): TelegramMessage {
+  const id = nextId++;
+  return {
+    id,
+    fileName,
+    fileId: `file-${id}`,
+    fileSize: 1024n,
+    date: new Date("2026-01-01T00:00:00Z"),
+  };
+}
+
+function names(files: string[]): string[] {
+  const sets = groupArchiveSets(files.map(msg));
+  expect(sets).toHaveLength(1);
+  return sets[0].parts.map((p) => p.fileName);
+}
+
+describe("groupArchiveSets — legacy split part ordering", () => {
+  it("puts the bare .rar FIRST in a RAR_LEGACY set (it is volume 1)", () => {
+    expect(names(["Pack.r01", "Pack.rar", "Pack.r00"])).toEqual([
+      "Pack.rar",
+      "Pack.r00",
+      "Pack.r01",
+    ]);
+  });
+
+  it("puts the bare .zip LAST in a ZIP_LEGACY set (it is the final disk)", () => {
+    expect(names(["Pack.z02", "Pack.zip", "Pack.z01"])).toEqual([
+      "Pack.z01",
+      "Pack.z02",
+      "Pack.zip",
+    ]);
+  });
+
+  it("marks both legacy sets as multipart with the right format", () => {
+    const rar = groupArchiveSets([msg("Pack.rar"), msg("Pack.r00")])[0];
+    expect(rar.isMultipart).toBe(true);
+    expect(rar.type).toBe("RAR");
+
+    const zip = groupArchiveSets([msg("Pack.zip"), msg("Pack.z01")])[0];
+    expect(zip.isMultipart).toBe(true);
+    expect(zip.type).toBe("ZIP");
+  });
+
+  it("orders numbered volume sets by part number", () => {
+    expect(names(["Pack.rar.003", "Pack.rar.001", "Pack.rar.002"])).toEqual([
+      "Pack.rar.001",
+      "Pack.rar.002",
+      "Pack.rar.003",
+    ]);
+  });
+
+  it("orders .partN sets by part number with an SFX first volume", () => {
+    expect(names(["Pack.part3.rar", "Pack.part1.exe", "Pack.part2.rar"])).toEqual([
+      "Pack.part1.exe",
+      "Pack.part2.rar",
+      "Pack.part3.rar",
+    ]);
+  });
+
+  it("treats unrelated singles as their own non-multipart sets", () => {
+    const sets = groupArchiveSets([msg("A.zip"), msg("B.rar")]);
+    expect(sets).toHaveLength(2);
+    expect(sets.every((s) => !s.isMultipart)).toBe(true);
+    expect(sets.every((s) => s.parts.length === 1)).toBe(true);
+  });
+});

worker/src/archive/multipart.ts

@@ -78,10 +78,17 @@ export function groupArchiveSets(messages: TelegramMessage[]): ArchiveSet[] {
         }
       }
 
-      // Sort by part number (singles get a very high number so they come last — they're the final part)
+      // Sort by part number. A bare single (partNumber -1) sits at a different end of the
+      // set depending on the legacy scheme: in a .zip/.z01/.z02 set the bare pack.zip is the
+      // FINAL disk, but in a .rar/.r00/.r01 set the bare pack.rar is volume 1 and .r00
+      // onwards follow it. Getting this backwards makes parts[0] a headerless continuation
+      // volume, which breaks listing and mislabels the package.
+      const singleRank = multipartEntries.some((e) => e.info.pattern === "RAR_LEGACY")
+        ? -1 // before .r00
+        : 999999;
       allEntries.sort((a, b) => {
-        const aNum = a.info.partNumber === -1 ? 999999 : a.info.partNumber;
+        const aNum = a.info.partNumber === -1 ? singleRank : a.info.partNumber;
-        const bNum = b.info.partNumber === -1 ? 999999 : b.info.partNumber;
+        const bNum = b.info.partNumber === -1 ? singleRank : b.info.partNumber;
         return aNum - bNum;
       });
 

worker/src/archive/ranged/dispatch.test.ts

@@ -1,15 +1,103 @@
 import { describe, it, expect } from "vitest";
-import { readScannedListingRanged } from "./dispatch.js";
+import { readScannedListingRanged, readScannedZipListing } from "./dispatch.js";
+import type { RangeReader } from "./range-reader.js";
+import { buildSpannedStoreZip, buildStoreZip, byteSplit } from "../testing/spanned-zip-fixture.js";
+
+/** Serve ranged reads out of in-memory part buffers keyed by fileId. */
+function readerFor(parts: { fileId: string; buf: Buffer }[]): RangeReader {
+  const byId = new Map(parts.map((p) => [p.fileId, p.buf]));
+  return async (fileId, offset, length) => {
+    const buf = byId.get(fileId);
+    if (!buf) throw new Error(`unknown fileId ${fileId}`);
+    return buf.subarray(offset, offset + length);
+  };
+}
+
+const FILES = [
+  { name: "src/b.bin", data: Buffer.alloc(2048, 7), disk: 0 },
+  { name: "src/models/dragon.stl", data: Buffer.from("DRAGON"), disk: 1 },
+  { name: "readme.txt", data: Buffer.from("hello world"), disk: 2 },
+];
 
 describe("readScannedListingRanged", () => {
   it("returns null for an unknown archive type without calling the reader", async () => {
-    const read = async () => Buffer.alloc(0);
     const result = await readScannedListingRanged(
       "DOCUMENT",
       { invoke: async () => ({}) } as never,
       [{ fileId: "1", fileSize: 100n, fileName: "a.pdf" }],
     );
     expect(result).toBeNull();
-    void read; // unused placeholder kept out of the dispatch call — DOCUMENT never reaches a reader
+  });
+});
+
+describe("readScannedZipListing", () => {
+  it("lists a ZIP-spec spanned set (.z01 … .zip) from the final volume's tail", async () => {
+    const vols = buildSpannedStoreZip(FILES, 3);
+    const names = ["Pack.z01", "Pack.z02", "Pack.zip"];
+    const parts = vols.map((buf, i) => ({ fileId: String(i), fileSize: BigInt(buf.length), fileName: names[i] }));
+
+    const entries = await readScannedZipListing(
+      parts,
+      readerFor(parts.map((p, i) => ({ fileId: p.fileId, buf: vols[i] }))),
+    );
+
+    expect(entries).not.toBeNull();
+    expect(entries!.map((e) => e.path).sort()).toEqual([
+      "readme.txt",
+      "src/b.bin",
+      "src/models/dragon.stl",
+    ]);
+  });
+
+  it("only reads the final volume — the earlier volumes are never downloaded", async () => {
+    const vols = buildSpannedStoreZip(FILES, 3);
+    const names = ["Pack.z01", "Pack.z02", "Pack.zip"];
+    const parts = vols.map((buf, i) => ({ fileId: String(i), fileSize: BigInt(buf.length), fileName: names[i] }));
+    const touched: string[] = [];
+    const base = readerFor(parts.map((p, i) => ({ fileId: p.fileId, buf: vols[i] })));
+
+    await readScannedZipListing(parts, async (id, off, len, size) => {
+      touched.push(id);
+      return base(id, off, len, size);
+    });
+
+    expect([...new Set(touched)]).toEqual(["2"]);
+  });
+
+  it("still lists a 7-Zip raw byte split (.zip.001 …) using whole-archive offsets", async () => {
+    const zip = buildStoreZip([
+      { name: "models/knight.stl", data: Buffer.alloc(5000, 9) },
+      { name: "license.txt", data: Buffer.from("MIT") },
+    ]);
+    const chunks = byteSplit(zip, 3);
+    const names = ["Pack.zip.001", "Pack.zip.002", "Pack.zip.003"];
+    const parts = chunks.map((buf, i) => ({ fileId: String(i), fileSize: BigInt(buf.length), fileName: names[i] }));
+
+    const entries = await readScannedZipListing(
+      parts,
+      readerFor(parts.map((p, i) => ({ fileId: p.fileId, buf: chunks[i] }))),
+    );
+
+    expect(entries!.map((e) => e.fileName).sort()).toEqual(["knight.stl", "license.txt"]);
+  });
+
+  it("returns null for a spanned set whose central directory starts on an earlier volume", async () => {
+    const vols = buildSpannedStoreZip(
+      FILES.map((f) => ({ ...f, disk: Math.min(f.disk, 1) })),
+      3,
+      { cdStartDisk: 1 },
+    );
+    const names = ["Pack.z01", "Pack.z02", "Pack.zip"];
+    const parts = vols.map((buf, i) => ({ fileId: String(i), fileSize: BigInt(buf.length), fileName: names[i] }));
+
+    const entries = await readScannedZipListing(
+      parts,
+      readerFor(parts.map((p, i) => ({ fileId: p.fileId, buf: vols[i] }))),
+    );
+    expect(entries).toBeNull();
+  });
+
+  it("returns null when there are no parts", async () => {
+    expect(await readScannedZipListing([], readerFor([]))).toBeNull();
   });
 });

worker/src/archive/ranged/dispatch.ts

@@ -1,11 +1,11 @@
 import type { Client } from "tdl";
-import { downloadFileRange } from "../../tdlib/range-download.js";
+import { parseZipCentralDirectoryFromTail, findEocdOffset, MIN_ZIP_TAIL_BYTES } from "../central-directory.js";
-import { parseZipCentralDirectoryFromTail, MIN_ZIP_TAIL_BYTES } from "../central-directory.js";
+import { isSpannedZipPartSet } from "../zip-spanned.js";
 import { childLogger } from "../../util/logger.js";
 import type { FileEntry } from "../zip-reader.js";
 import { readSevenZListingRanged, type RangedPart } from "./sevenz-ranged.js";
 import { readRarListingRanged } from "./rar-ranged.js";
-import { tdlibRangeReader } from "./range-reader.js";
+import { tdlibRangeReader, type RangeReader } from "./range-reader.js";
 
 const log = childLogger("ranged-dispatch");
 
@@ -13,25 +13,45 @@ const log = childLogger("ranged-dispatch");
  * Read a ZIP central directory from the tail of a (possibly multipart)
  * archive. `parts` is ordered; only the LAST part carries the EOCD record.
  * `fileSize` on each part is that part's own size (NOT the whole-archive
- * total) so the download offset stays within that part's bounds, while
+ * total) so the download offset stays within that part's bounds.
- * `tailStart` passed to the parser is the logical whole-archive offset
+ *
- * (preceding parts' sizes + the offset within the last part).
+ * Which logical offset the EOCD's central-directory pointer is measured from
+ * depends on the multipart shape:
+ *
+ *  - 7-Zip raw byte split (`.zip.001`, …): the parts are one ZIP file cut into
+ *    chunks, so the pointer is a whole-archive offset → `tailStart` is the
+ *    preceding parts' sizes plus the offset within the last part.
+ *  - ZIP-spec spanned archive (`.z01`, …, `.zip`): each volume is its own unit
+ *    and the pointer is relative to the start of the volume holding the
+ *    directory → `tailStart` is just the offset within that final volume.
+ *
+ * Getting this wrong makes the computed directory offset wildly negative, and
+ * the parser then throws RangeError on every tail size — which is exactly how
+ * spanned sets ended up indexed with no file list at all.
  */
 export async function readScannedZipListing(
-  client: Client,
+  parts: RangedPart[],
-  parts: { fileId: string; fileSize: bigint }[],
+  read: RangeReader,
 ): Promise<FileEntry[] | null> {
   if (parts.length === 0) return null;
   const lastPart = parts[parts.length - 1];
-  const precedingSize = parts.slice(0, -1).reduce((sum, p) => sum + Number(p.fileSize), 0);
+  const spanned = isSpannedZipPartSet(parts.map((p) => p.fileName));
+  const precedingSize = spanned
+    ? 0
+    : parts.slice(0, -1).reduce((sum, p) => sum + Number(p.fileSize), 0);
   const lastSize = Number(lastPart.fileSize);
   for (const tailBytes of [MIN_ZIP_TAIL_BYTES, MIN_ZIP_TAIL_BYTES * 4]) {
     const partOffset = Math.max(0, lastSize - tailBytes);
     const downloadLen = Math.min(tailBytes, lastSize);
     try {
-      const buf = await downloadFileRange(client, lastPart.fileId, partOffset, downloadLen, lastPart.fileSize);
+      const buf = await read(lastPart.fileId, partOffset, downloadLen, lastPart.fileSize);
-      const tailStart = precedingSize + partOffset;
+      if (spanned && !cdStartsOnFinalVolume(buf)) {
-      return parseZipCentralDirectoryFromTail(buf, tailStart);
+        // The directory begins on an earlier volume; reaching it would mean
+        // ranged-reading that volume too. Leave it to the full-download path.
+        log.debug({ fileId: lastPart.fileId }, "spanned ZIP central directory is not on the final volume");
+        return null;
+      }
+      return parseZipCentralDirectoryFromTail(buf, partOffset + precedingSize);
     } catch (err) {
       if (err instanceof RangeError) continue; // try a larger tail
       log.warn({ err, fileId: lastPart.fileId }, "ranged ZIP listing failed");
@@ -41,6 +61,20 @@ export async function readScannedZipListing(
   return null;
 }
 
+/**
+ * For a spanned archive, whether the EOCD says the central directory starts on
+ * the very volume that EOCD lives on (the usual case). ZIP64's saturated
+ * 0xFFFF disk fields are treated as "yes" — the ZIP64 record that supersedes
+ * them is itself in this tail, and the parser resolves it there.
+ */
+function cdStartsOnFinalVolume(tail: Buffer): boolean {
+  const eocd = findEocdOffset(tail);
+  if (eocd < 0) return true; // let the parser report the real problem
+  const thisDisk = tail.readUInt16LE(eocd + 4);
+  const cdStartDisk = tail.readUInt16LE(eocd + 6);
+  return thisDisk === cdStartDisk || thisDisk === 0xffff || cdStartDisk === 0xffff;
+}
+
 /**
  * Dispatch a (no-download) inner-file listing read by archive type. Used both
  * by the provenance-backfill path (reading an already-uploaded copy) and the
@@ -55,7 +89,7 @@ export async function readScannedListingRanged(
   parts: RangedPart[],
 ): Promise<FileEntry[] | null> {
   const read = tdlibRangeReader(client);
-  if (archiveType === "ZIP") return readScannedZipListing(client, parts);
+  if (archiveType === "ZIP") return readScannedZipListing(parts, read);
   if (archiveType === "SEVEN_Z") return readSevenZListingRanged(parts, read);
   if (archiveType === "RAR") return readRarListingRanged(parts, read);
   return null;

worker/src/archive/ranged/sevenz-ranged.test.ts

@@ -64,7 +64,250 @@ describe("readSevenZListingRanged", () => {
   });
 });
 
-import { read7zNumber, locate7zEncodedHeaderPack } from "./sevenz-ranged.js";
+import { read7zNumber, locate7zEncodedHeaderPack, mapRangeToVolumes, planSevenZSparseParts } from "./sevenz-ranged.js";
+import type { RangedPart } from "./sevenz-ranged.js";
+import type { SparsePart } from "./sparse-list.js";
+
+describe("mapRangeToVolumes", () => {
+  it("maps a range fully inside one volume", () => {
+    expect(mapRangeToVolumes([100, 100, 100], 120, 30)).toEqual([
+      { partIndex: 1, offset: 20, length: 30 },
+    ]);
+  });
+
+  it("splits a range that straddles a volume boundary", () => {
+    expect(mapRangeToVolumes([100, 100], 90, 20)).toEqual([
+      { partIndex: 0, offset: 90, length: 10 },
+      { partIndex: 1, offset: 0, length: 10 },
+    ]);
+  });
+
+  it("spans three volumes when the range swallows a whole middle volume", () => {
+    expect(mapRangeToVolumes([100, 50, 100], 90, 80)).toEqual([
+      { partIndex: 0, offset: 90, length: 10 },
+      { partIndex: 1, offset: 0, length: 50 },
+      { partIndex: 2, offset: 0, length: 20 },
+    ]);
+  });
+
+  it("degenerates to a single-volume identity mapping", () => {
+    expect(mapRangeToVolumes([5_000_000], 4_900_000, 100)).toEqual([
+      { partIndex: 0, offset: 4_900_000, length: 100 },
+    ]);
+  });
+
+  it("returns null when the range runs past the concatenated end", () => {
+    expect(mapRangeToVolumes([100, 100], 190, 20)).toBeNull();
+    expect(mapRangeToVolumes([100], 100, 1)).toBeNull();
+  });
+
+  it("returns null for negative offsets or lengths", () => {
+    expect(mapRangeToVolumes([100], -1, 10)).toBeNull();
+    expect(mapRangeToVolumes([100], 10, -1)).toBeNull();
+  });
+
+  it("returns no slices for a zero-length range", () => {
+    expect(mapRangeToVolumes([100, 100], 150, 0)).toEqual([]);
+  });
+});
+
+/**
+ * A `.7z.001`/`.7z.002` set is a raw byte split of one logical 7z file, so
+ * fixtures are built as a single logical stream and then cut into volumes.
+ * No real `7z` binary is needed (and none is installed here) because the
+ * contract under test is the whole-archive-offset mapping, not `7z l` parsing.
+ */
+function buildLogical7z(opts: {
+  total: number;
+  nextHeaderStart: number; // absolute offset in the logical stream
+  nextHeader: Buffer;
+  pack?: { start: number; bytes: Buffer }; // absolute offset of packed header bytes
+}): Buffer {
+  const buf = Buffer.alloc(opts.total, 0x5a); // 0x5a stands in for file payload
+  MAGIC.copy(buf, 0);
+  buf.writeUInt8(0, 6); buf.writeUInt8(4, 7);
+  buf.writeUInt32LE(0, 8);
+  buf.writeBigUInt64LE(BigInt(opts.nextHeaderStart - 32), 12); // NextHeaderOffset
+  buf.writeBigUInt64LE(BigInt(opts.nextHeader.length), 20);    // NextHeaderSize
+  buf.writeUInt32LE(0, 28);
+  opts.nextHeader.copy(buf, opts.nextHeaderStart);
+  if (opts.pack) opts.pack.bytes.copy(buf, opts.pack.start);
+  return buf;
+}
+
+function splitIntoVolumes(logical: Buffer, sizes: number[]): Buffer[] {
+  const out: Buffer[] = [];
+  let pos = 0;
+  for (const s of sizes) { out.push(logical.subarray(pos, pos + s)); pos += s; }
+  return out;
+}
+
+function volumeSet(volumes: Buffer[]): {
+  parts: RangedPart[];
+  read: RangeReader;
+  reads: { fileId: string; offset: number; length: number }[];
+} {
+  const parts = volumes.map((v, i) => ({
+    fileId: `v${i + 1}`,
+    fileSize: BigInt(v.length),
+    fileName: `pack.7z.${String(i + 1).padStart(3, "0")}`,
+  }));
+  const reads: { fileId: string; offset: number; length: number }[] = [];
+  const read: RangeReader = async (fileId, offset, length) => {
+    reads.push({ fileId, offset, length });
+    const vol = volumes[parts.findIndex((p) => p.fileId === fileId)];
+    return Buffer.from(vol.subarray(offset, offset + length));
+  };
+  return { parts, read, reads };
+}
+
+/** Rebuild the logical stream from the sparse per-volume reconstructions. */
+function reconstruct(sparseParts: SparsePart[]): Buffer {
+  return Buffer.concat(
+    sparseParts.map((p) => {
+      const b = Buffer.alloc(p.size);
+      for (const r of p.regions) r.bytes.copy(b, r.offset);
+      return b;
+    }),
+  );
+}
+
+describe("readSevenZListingRanged — multi-volume (.7z.001, .7z.002, ...)", () => {
+  it("reads the next header from the LAST volume, not the first", async () => {
+    // Volume 1 is 200 bytes; the index sits at logical 280 — inside volume 2.
+    const nextHeader = Buffer.concat([Buffer.from([0x01]), Buffer.alloc(19, 0x11)]);
+    const logical = buildLogical7z({ total: 300, nextHeaderStart: 280, nextHeader });
+    const { parts, read, reads } = volumeSet(splitIntoVolumes(logical, [200, 100]));
+
+    const sparse = await planSevenZSparseParts(parts, read);
+    expect(sparse).not.toBeNull();
+
+    expect(reads).toEqual([
+      { fileId: "v1", offset: 0, length: 32 },   // signature header: volume 1
+      { fileId: "v2", offset: 80, length: 20 },  // next header: volume 2 @ 280-200
+    ]);
+    // Both volumes are reconstructed so `7z l pack.7z.001` can concatenate them.
+    expect(sparse!.map((p) => [p.fileName, p.size])).toEqual([
+      ["pack.7z.001", 200],
+      ["pack.7z.002", 100],
+    ]);
+    const rebuilt = reconstruct(sparse!);
+    expect(rebuilt.length).toBe(300);
+    expect(rebuilt.subarray(0, 32).equals(logical.subarray(0, 32))).toBe(true);
+    expect(rebuilt.subarray(280, 300).equals(nextHeader)).toBe(true);
+  });
+
+  it("fetches an encoded header's packed bytes from whichever middle volume holds them", async () => {
+    // 3 volumes of 100. Packed header bytes at logical 150 (volume 2), index at 270 (volume 3).
+    // kEncodedHeader, kPackInfo, PackPos=118, NumStreams=1, kSize, PackSize=24
+    const encHeader = Buffer.from([0x17, 0x06, 0x76, 0x01, 0x09, 0x18]);
+    const packBytes = Buffer.alloc(24, 0x77);
+    const logical = buildLogical7z({
+      total: 300,
+      nextHeaderStart: 270,
+      nextHeader: encHeader,
+      pack: { start: 150, bytes: packBytes },
+    });
+    const { parts, read, reads } = volumeSet(splitIntoVolumes(logical, [100, 100, 100]));
+
+    const sparse = await planSevenZSparseParts(parts, read);
+    expect(sparse).not.toBeNull();
+    expect(reads).toEqual([
+      { fileId: "v1", offset: 0, length: 32 },
+      { fileId: "v3", offset: 70, length: 6 },   // index in the last volume
+      { fileId: "v2", offset: 50, length: 24 },  // packed header in the middle volume
+    ]);
+    expect(sparse!).toHaveLength(3);
+    const rebuilt = reconstruct(sparse!);
+    expect(rebuilt.subarray(150, 174).equals(packBytes)).toBe(true);
+    expect(rebuilt.subarray(270, 276).equals(encHeader)).toBe(true);
+  });
+
+  it("splits a header range that straddles a volume boundary into two reads", async () => {
+    // Index is 40 bytes starting at logical 180: last 20 bytes of volume 2 (100..200)
+    // and first 20 bytes of volume 3.
+    const nextHeader = Buffer.concat([Buffer.from([0x01]), Buffer.alloc(39, 0x22)]);
+    const logical = buildLogical7z({ total: 300, nextHeaderStart: 180, nextHeader });
+    const { parts, read, reads } = volumeSet(splitIntoVolumes(logical, [100, 100, 100]));
+
+    const sparse = await planSevenZSparseParts(parts, read);
+    expect(sparse).not.toBeNull();
+    expect(reads).toEqual([
+      { fileId: "v1", offset: 0, length: 32 },
+      { fileId: "v2", offset: 80, length: 20 },
+      { fileId: "v3", offset: 0, length: 20 },
+    ]);
+    // Byte-exact across the seam.
+    expect(reconstruct(sparse!).subarray(180, 220).equals(nextHeader)).toBe(true);
+  });
+
+  it("splits an encoded header's packed bytes across a volume boundary", async () => {
+    // PackPos=58 -> packStart 90, PackSize=30 -> 90..120 straddles volumes 1|2.
+    const encHeader = Buffer.from([0x17, 0x06, 0x3a, 0x01, 0x09, 0x1e]);
+    const packBytes = Buffer.alloc(30, 0x99);
+    const logical = buildLogical7z({
+      total: 300,
+      nextHeaderStart: 290,
+      nextHeader: encHeader,
+      pack: { start: 90, bytes: packBytes },
+    });
+    const { parts, read, reads } = volumeSet(splitIntoVolumes(logical, [100, 100, 100]));
+
+    const sparse = await planSevenZSparseParts(parts, read);
+    expect(sparse).not.toBeNull();
+    expect(reads).toEqual([
+      { fileId: "v1", offset: 0, length: 32 },
+      { fileId: "v3", offset: 90, length: 6 },
+      { fileId: "v1", offset: 90, length: 10 },
+      { fileId: "v2", offset: 0, length: 20 },
+    ]);
+    expect(reconstruct(sparse!).subarray(90, 120).equals(packBytes)).toBe(true);
+  });
+
+  it("keeps a single-volume .7z reading exactly as before", async () => {
+    const nextHeader = Buffer.concat([Buffer.from([0x01]), Buffer.alloc(19, 0x33)]);
+    const logical = buildLogical7z({ total: 300, nextHeaderStart: 280, nextHeader });
+    const { parts, read, reads } = volumeSet([logical]);
+
+    const sparse = await planSevenZSparseParts(parts, read);
+    expect(reads).toEqual([
+      { fileId: "v1", offset: 0, length: 32 },
+      { fileId: "v1", offset: 280, length: 20 },
+    ]);
+    expect(sparse!).toHaveLength(1);
+    expect(sparse![0].size).toBe(300);
+    expect(reconstruct(sparse!).subarray(280, 300).equals(nextHeader)).toBe(true);
+  });
+
+  it("returns null when the next-header offset points past the whole set", async () => {
+    const nextHeader = Buffer.from([0x01, 0x00]);
+    // Claim the index lives at 5000 while the set totals only 300 bytes.
+    const logical = buildLogical7z({ total: 300, nextHeaderStart: 280, nextHeader });
+    logical.writeBigUInt64LE(BigInt(5000 - 32), 12);
+    const { parts, read } = volumeSet(splitIntoVolumes(logical, [200, 100]));
+    expect(await planSevenZSparseParts(parts, read)).toBeNull();
+  });
+
+  it("returns null on an unrecognized next-header type", async () => {
+    const logical = buildLogical7z({
+      total: 300,
+      nextHeaderStart: 280,
+      nextHeader: Buffer.from([0x42, 0x00]),
+    });
+    const { parts, read } = volumeSet(splitIntoVolumes(logical, [200, 100]));
+    expect(await planSevenZSparseParts(parts, read)).toBeNull();
+  });
+
+  it("returns null when the first volume has no 7z signature", async () => {
+    const logical = Buffer.alloc(300, 0x00);
+    const { parts, read } = volumeSet(splitIntoVolumes(logical, [200, 100]));
+    expect(await planSevenZSparseParts(parts, read)).toBeNull();
+  });
+
+  it("returns null for an empty part list", async () => {
+    expect(await planSevenZSparseParts([], async () => Buffer.alloc(0))).toBeNull();
+  });
+});
 
 describe("read7zNumber", () => {
   it("reads a single-byte number", () => {

worker/src/archive/ranged/sevenz-ranged.ts

@@ -1,6 +1,6 @@
 import type { FileEntry } from "../zip-reader.js";
 import { read7zContents } from "../sevenz-reader.js";
-import { listFromSparse } from "./sparse-list.js";
+import { listFromSparse, type SparsePart } from "./sparse-list.js";
 import type { RangeReader } from "./range-reader.js";
 import { childLogger } from "../../util/logger.js";
 
@@ -13,6 +13,8 @@ const K_ENCODED_HEADER = 0x17;
 const K_PACK_INFO = 0x06;
 const K_SIZE = 0x09;
 
+const SIG_HEADER_BYTES = 32;
+
 /** Read a 7z variable-length number: first byte is a length mask, followed by
  *  little-endian bytes. Math.pow keeps values exact above 2^31. */
 export function read7zNumber(buf: Buffer, pos: number): { value: number; next: number } {
@@ -61,7 +63,7 @@ export function locate7zEncodedHeaderPack(
 export function parseSevenZSignatureHeader(
   buf: Buffer,
 ): { nextHeaderOffset: number; nextHeaderSize: number } | null {
-  if (buf.length < 32) return null;
+  if (buf.length < SIG_HEADER_BYTES) return null;
   if (!buf.subarray(0, 6).equals(SEVENZ_MAGIC)) return null;
   return {
     nextHeaderOffset: Number(buf.readBigUInt64LE(12)),
@@ -71,42 +73,158 @@ export function parseSevenZSignatureHeader(
 
 export interface RangedPart { fileId: string; fileSize: bigint; fileName: string }
 
+/** One volume's share of a whole-archive byte range. */
+export interface VolumeSlice { partIndex: number; offset: number; length: number }
+
+/**
+ * A `.7z.001`/`.7z.002`/… set produced by 7-Zip's `-v` switch is a **raw byte
+ * split** of one logical `.7z` file, not a ZIP-style spanned archive with
+ * per-volume structure: `cat pack.7z.00*` reproduces the original archive
+ * byte-for-byte. So every offset in the 7z headers is an offset into the
+ * concatenation of the volumes, and the only correct way to read them is to
+ * treat the set as one logical byte stream.
+ *
+ * That matters most for the next header (the archive index), which a 7z file
+ * keeps at its *end* — i.e. in the **last** volume, never the first. Reading
+ * only `parts[0]` therefore fails for every multi-volume set.
+ *
+ * Map a whole-archive `[start, start + length)` range onto per-volume reads,
+ * splitting it when it straddles a volume boundary. Returns null when the
+ * range falls outside the concatenated stream.
+ */
+export function mapRangeToVolumes(
+  sizes: number[],
+  start: number,
+  length: number,
+): VolumeSlice[] | null {
+  const total = sizes.reduce((sum, s) => sum + s, 0);
+  if (!Number.isFinite(start) || !Number.isFinite(length)) return null;
+  if (start < 0 || length < 0 || start + length > total) return null;
+
+  const slices: VolumeSlice[] = [];
+  let pos = start;
+  let remaining = length;
+  let base = 0;
+  for (let i = 0; i < sizes.length && remaining > 0; i++) {
+    const end = base + sizes[i];
+    if (pos < end) {
+      const offset = pos - base;
+      const take = Math.min(remaining, sizes[i] - offset);
+      if (take > 0) {
+        slices.push({ partIndex: i, offset, length: take });
+        pos += take;
+        remaining -= take;
+      }
+    }
+    base = end;
+  }
+  return remaining === 0 ? slices : null;
+}
+
+/**
+ * Fetch the 7z header regions of a (possibly multi-volume) archive and return
+ * them as per-volume sparse reconstructions — the header bytes at their real
+ * offsets, file payloads left as zero holes.
+ *
+ * Exported for testing: it is the whole-archive-offset mapping that is worth
+ * asserting, and the final `7z l` step needs the real binary.
+ */
+export async function planSevenZSparseParts(
+  parts: RangedPart[],
+  read: RangeReader,
+): Promise<SparsePart[] | null> {
+  const first = parts[0];
+  if (!first) {
+    log.warn({ partCount: parts.length }, "ranged 7z listing aborted — no parts supplied");
+    return null;
+  }
+  const sizes = parts.map((p) => Number(p.fileSize));
+  const total = sizes.reduce((sum, s) => sum + s, 0);
+  // Log context shared by every bail-out below; the first part names the set.
+  const ctx = { fileId: first.fileId, fileName: first.fileName, partCount: parts.length, total };
+  const regions: { offset: number; bytes: Buffer }[][] = parts.map(() => []);
+
+  /**
+   * Read a whole-archive range, recording each volume's slice as a region so
+   * the sparse reconstruction places the bytes where 7z expects them.
+   */
+  const fetchLogical = async (start: number, length: number, what: string): Promise<Buffer | null> => {
+    const slices = mapRangeToVolumes(sizes, start, length);
+    if (!slices) {
+      log.warn({ ...ctx, what, start, length, sizes }, `ranged 7z listing aborted — ${what} region out of bounds`);
+      return null;
+    }
+    const chunks: Buffer[] = [];
+    for (const s of slices) {
+      const part = parts[s.partIndex];
+      const bytes = await read(part.fileId, s.offset, s.length, part.fileSize);
+      if (bytes.length < s.length) {
+        log.warn(
+          { ...ctx, what, volume: s.partIndex + 1, volumeFileId: part.fileId, offset: s.offset, wanted: s.length, got: bytes.length },
+          `ranged 7z listing aborted — short read on ${what}`,
+        );
+        return null;
+      }
+      regions[s.partIndex].push({ offset: s.offset, bytes });
+      chunks.push(bytes);
+    }
+    return Buffer.concat(chunks);
+  };
+
+  try {
+    // The signature header is at the very start of volume 1.
+    const sig = await fetchLogical(0, Math.min(SIG_HEADER_BYTES, total), "signature-header");
+    if (!sig) return null;
+    const parsed = parseSevenZSignatureHeader(sig);
+    if (!parsed) {
+      log.warn({ ...ctx, head: sig.subarray(0, 16).toString("hex") }, "ranged 7z listing aborted — signature header did not parse");
+      return null;
+    }
+
+    // NextHeaderOffset is measured from the end of the signature header, into
+    // the concatenated stream — so this normally lands in the LAST volume.
+    const endStart = SIG_HEADER_BYTES + parsed.nextHeaderOffset;
+    if (parsed.nextHeaderSize <= 0) {
+      log.warn({ ...ctx, endStart, nextHeaderSize: parsed.nextHeaderSize }, "ranged 7z listing aborted — empty next header (no index to read)");
+      return null;
+    }
+    const endHeader = await fetchLogical(endStart, parsed.nextHeaderSize, "next-header");
+    if (!endHeader) return null;
+
+    const headerType = endHeader[0];
+    if (headerType === K_ENCODED_HEADER) {
+      // Compressed header: its packed bytes live mid-stream, not at EOF, so
+      // they may sit in any volume — or straddle two.
+      const pack = locate7zEncodedHeaderPack(endHeader);
+      if (!pack) {
+        log.warn(
+          { ...ctx, endStart, nextHeaderSize: parsed.nextHeaderSize, head: endHeader.subarray(0, 16).toString("hex") },
+          "ranged 7z listing aborted — encoded-header PackInfo did not parse",
+        );
+        return null;
+      }
+      const packStart = SIG_HEADER_BYTES + pack.packPos;
+      const packBytes = await fetchLogical(packStart, pack.packSize, "packed-header");
+      if (!packBytes) return null;
+    } else if (headerType !== K_HEADER) {
+      log.warn({ ...ctx, endStart, headerType }, "ranged 7z listing aborted — unknown next-header type");
+      return null;
+    }
+
+    return parts.map((p, i) => ({ fileName: p.fileName, size: sizes[i], regions: regions[i] }));
+  } catch (err) {
+    log.warn({ err, ...ctx }, "ranged 7z listing failed");
+    return null;
+  }
+}
+
 export async function readSevenZListingRanged(
   parts: RangedPart[],
   read: RangeReader,
 ): Promise<FileEntry[] | null> {
-  const part = parts[0];
+  const sparseParts = await planSevenZSparseParts(parts, read);
-  if (!part) return null;
+  if (!sparseParts) return null;
-  const size = Number(part.fileSize);
+  // 7-Zip opens `pack.7z.001` as a split archive and concatenates the set
-  try {
+  // itself, so the whole reconstructed set must be on disk, not just part 1.
-    const sig = await read(part.fileId, 0, 32, part.fileSize);
+  return listFromSparse(sparseParts, read7zContents);
-    const parsed = parseSevenZSignatureHeader(sig);
-    if (!parsed) return null;
-    const endStart = 32 + parsed.nextHeaderOffset;
-    if (endStart < 0 || endStart + parsed.nextHeaderSize > size) return null;
-    const endHeader = await read(part.fileId, endStart, parsed.nextHeaderSize, part.fileSize);
-
-    const regions = [
-      { offset: 0, bytes: sig },
-      { offset: endStart, bytes: endHeader },
-    ];
-
-    const headerType = endHeader[0];
-    if (headerType === K_ENCODED_HEADER) {
-      // Compressed header: its packed bytes live mid-file, not at EOF. Fetch them.
-      const pack = locate7zEncodedHeaderPack(endHeader);
-      if (!pack) return null;
-      const packStart = 32 + pack.packPos;
-      if (packStart < 0 || packStart + pack.packSize > size) return null;
-      const packBytes = await read(part.fileId, packStart, pack.packSize, part.fileSize);
-      regions.push({ offset: packStart, bytes: packBytes });
-    } else if (headerType !== K_HEADER) {
-      return null; // unknown next-header type
-    }
-
-    return listFromSparse([{ fileName: part.fileName, size, regions }], read7zContents);
-  } catch (err) {
-    log.warn({ err, fileId: part.fileId }, "ranged 7z listing failed");
-    return null;
-  }
 }

worker/src/archive/testing/spanned-zip-fixture.ts

@@ -0,0 +1,191 @@
+import { crc32 } from "zlib"; // Node 20+ exposes zlib.crc32
+
+/**
+ * Test-only builders that emit real ZIP byte streams.
+ *
+ * Two distinct on-disk shapes are produced here, because the worker has to
+ * tell them apart:
+ *
+ *  - `buildSpannedStoreZip` → a ZIP-spec **spanned/multi-disk** archive
+ *    (`Pack.z01`, `Pack.z02`, …, `Pack.zip`). Each volume is its own file;
+ *    central-directory records carry a disk number, and the EOCD's
+ *    "offset of start of central directory" is relative to the *start of the
+ *    disk that holds it*, not to a concatenation of the volumes.
+ *
+ *  - `buildStoreZip` → an ordinary single-file ZIP. Cutting its bytes into
+ *    chunks yields the 7-Zip raw byte split shape (`Pack.zip.001`, …), where
+ *    all disk numbers are 0 and offsets are whole-archive absolute.
+ *
+ * Field layouts follow APPNOTE 6.3.x sections 4.3.12 (central directory) and
+ * 4.3.16 (EOCD). Verified against Info-ZIP `zip -s` output.
+ */
+
+const LOCAL_SIG = 0x04034b50;
+const CD_SIG = 0x02014b50;
+const EOCD_SIG = 0x06054b50;
+/** APPNOTE 8.5.3: the first volume of a spanned archive starts with this. */
+const SPANNING_SIG = 0x08074b50;
+
+export interface FixtureFile {
+  name: string;
+  data: Buffer;
+  /** 0-based volume this file's local header + data is written to. */
+  disk?: number;
+}
+
+function localHeader(name: string, data: Buffer): Buffer {
+  const nameBuf = Buffer.from(name, "utf8");
+  const local = Buffer.alloc(30);
+  local.writeUInt32LE(LOCAL_SIG, 0);
+  local.writeUInt16LE(20, 4); // version needed
+  local.writeUInt16LE(0, 6); // flags
+  local.writeUInt16LE(0, 8); // method = store
+  local.writeUInt32LE(crc32(data) >>> 0, 14);
+  local.writeUInt32LE(data.length, 18); // compressed
+  local.writeUInt32LE(data.length, 22); // uncompressed
+  local.writeUInt16LE(nameBuf.length, 26);
+  local.writeUInt16LE(0, 28); // extra len
+  return Buffer.concat([local, nameBuf, data]);
+}
+
+function centralHeader(name: string, data: Buffer, diskStart: number, relOffset: number): Buffer {
+  const nameBuf = Buffer.from(name, "utf8");
+  const cd = Buffer.alloc(46);
+  cd.writeUInt32LE(CD_SIG, 0);
+  cd.writeUInt16LE(20, 4); // version made by
+  cd.writeUInt16LE(20, 6); // version needed
+  cd.writeUInt16LE(0, 8); // flags
+  cd.writeUInt16LE(0, 10); // method = store
+  cd.writeUInt32LE(crc32(data) >>> 0, 16);
+  cd.writeUInt32LE(data.length, 20); // compressed
+  cd.writeUInt32LE(data.length, 24); // uncompressed
+  cd.writeUInt16LE(nameBuf.length, 28);
+  cd.writeUInt16LE(diskStart, 34); // disk number start
+  cd.writeUInt32LE(relOffset, 42); // offset of local header, relative to its disk
+  return Buffer.concat([cd, nameBuf]);
+}
+
+function eocd(opts: {
+  thisDisk: number;
+  cdStartDisk: number;
+  entriesThisDisk: number;
+  entriesTotal: number;
+  cdSize: number;
+  cdOffset: number;
+}): Buffer {
+  const buf = Buffer.alloc(22);
+  buf.writeUInt32LE(EOCD_SIG, 0);
+  buf.writeUInt16LE(opts.thisDisk, 4);
+  buf.writeUInt16LE(opts.cdStartDisk, 6);
+  buf.writeUInt16LE(opts.entriesThisDisk, 8);
+  buf.writeUInt16LE(opts.entriesTotal, 10);
+  buf.writeUInt32LE(opts.cdSize, 12);
+  buf.writeUInt32LE(opts.cdOffset, 16);
+  return buf;
+}
+
+/** Build an ordinary single-file STORE ZIP (all disk numbers 0). */
+export function buildStoreZip(files: FixtureFile[]): Buffer {
+  const body: Buffer[] = [];
+  const central: Buffer[] = [];
+  let offset = 0;
+  for (const f of files) {
+    const local = localHeader(f.name, f.data);
+    body.push(local);
+    central.push(centralHeader(f.name, f.data, 0, offset));
+    offset += local.length;
+  }
+  const cdBuf = Buffer.concat(central);
+  return Buffer.concat([
+    ...body,
+    cdBuf,
+    eocd({
+      thisDisk: 0,
+      cdStartDisk: 0,
+      entriesThisDisk: files.length,
+      entriesTotal: files.length,
+      cdSize: cdBuf.length,
+      cdOffset: offset,
+    }),
+  ]);
+}
+
+/**
+ * Build a ZIP-spec spanned archive as one Buffer per volume.
+ * Returned array is volume order: [z01, z02, …, zip] (last element is the
+ * final volume, which carries the central directory and EOCD).
+ *
+ * `cdStartDisk` (default: last volume) lets a test place the central
+ * directory so that it begins on an earlier volume and spills forward,
+ * exercising the cross-volume read path. As in a real writer, no file data
+ * may live on a volume after the one the directory starts on — those volumes
+ * hold directory continuation only.
+ */
+export function buildSpannedStoreZip(
+  files: FixtureFile[],
+  totalDisks: number,
+  opts: { cdStartDisk?: number } = {}
+): Buffer[] {
+  const cdStart = opts.cdStartDisk ?? totalDisks - 1;
+  for (const f of files) {
+    if ((f.disk ?? 0) > cdStart) {
+      throw new Error(`fixture misuse: ${f.name} is on volume ${f.disk} but the CD starts on ${cdStart}`);
+    }
+    if ((f.disk ?? 0) >= totalDisks) {
+      throw new Error(`fixture misuse: ${f.name} is on volume ${f.disk} of ${totalDisks}`);
+    }
+  }
+  const chunks: Buffer[][] = Array.from({ length: totalDisks }, () => []);
+  const lengths = new Array<number>(totalDisks).fill(0);
+
+  const marker = Buffer.alloc(4);
+  marker.writeUInt32LE(SPANNING_SIG, 0);
+  chunks[0].push(marker);
+  lengths[0] = 4;
+
+  const central: Buffer[] = [];
+  for (const f of files) {
+    const disk = f.disk ?? 0;
+    const local = localHeader(f.name, f.data);
+    central.push(centralHeader(f.name, f.data, disk, lengths[disk]));
+    chunks[disk].push(local);
+    lengths[disk] += local.length;
+  }
+
+  const cdBuf = Buffer.concat(central);
+  const lastDisk = totalDisks - 1;
+  const cdStartDisk = cdStart;
+  const cdOffset = lengths[cdStartDisk];
+
+  // Write the CD starting on cdStartDisk, spilling onto later volumes.
+  const spillDisks = lastDisk - cdStartDisk + 1;
+  const firstChunkLen = Math.ceil(cdBuf.length / spillDisks);
+  let written = 0;
+  for (let d = cdStartDisk; d <= lastDisk; d++) {
+    const take = d === lastDisk ? cdBuf.length - written : Math.min(firstChunkLen, cdBuf.length - written);
+    chunks[d].push(cdBuf.subarray(written, written + take));
+    lengths[d] += take;
+    written += take;
+  }
+
+  chunks[lastDisk].push(
+    eocd({
+      thisDisk: lastDisk,
+      cdStartDisk,
+      entriesThisDisk: files.length,
+      entriesTotal: files.length,
+      cdSize: cdBuf.length,
+      cdOffset,
+    })
+  );
+
+  return chunks.map((c) => Buffer.concat(c));
+}
+
+/** Cut a buffer into `count` roughly equal chunks (7-Zip raw byte split). */
+export function byteSplit(buf: Buffer, count: number): Buffer[] {
+  const size = Math.ceil(buf.length / count);
+  const out: Buffer[] = [];
+  for (let i = 0; i < buf.length; i += size) out.push(buf.subarray(i, i + size));
+  return out;
+}

worker/src/archive/zip-reader.ts

@@ -3,6 +3,7 @@ import { open as fsOpen, stat as fsStat } from "fs/promises";
 import path from "path";
 import { Readable } from "stream";
 import { childLogger } from "../util/logger.js";
+import { isSpannedZipPartSet, readSpannedZipCentralDirectory } from "./zip-spanned.js";
 
 const log = childLogger("zip-reader");
 
@@ -17,9 +18,15 @@ export interface FileEntry {
 
 /**
  * Read the central directory of a ZIP file without extracting any contents.
- * For multipart ZIPs (.zip.001, .zip.002 etc.), uses a custom random-access
+ *
- * reader that spans all parts seamlessly so yauzl can find the central
+ * Three shapes are handled:
- * directory at the end of the combined data.
+ *  - a single `.zip` → yauzl directly;
+ *  - a 7-Zip raw byte split (`.zip.001`, `.zip.002`, …), which is one ZIP file
+ *    cut into chunks → a random-access reader that spans the chunks so yauzl
+ *    sees the combined stream;
+ *  - a ZIP-spec spanned/multi-disk archive (`.z01`, `.z02`, …, `.zip`), a
+ *    different on-disk format that yauzl refuses outright → a dedicated
+ *    volume-aware central-directory reader.
  */
 export async function readZipCentralDirectory(
   filePaths: string[]
@@ -28,7 +35,22 @@ export async function readZipCentralDirectory(
     return readSingleZip(filePaths[0]);
   }
 
-  // Multipart: use a spanning random-access reader
+  if (isSpannedZipPartSet(filePaths)) {
+    try {
+      const result = await readSpannedZipCentralDirectory(filePaths);
+      if (result.kind === "entries") return result.entries;
+      if (result.kind === "failed") {
+        log.warn({ reason: result.reason, parts: filePaths.length }, "Failed to read spanned ZIP");
+        return [];
+      }
+      // "not-spanned": named like volumes but really a byte split — fall through.
+    } catch (err) {
+      log.warn({ err, parts: filePaths.length }, "Failed to read spanned ZIP");
+      return [];
+    }
+  }
+
+  // Multipart byte split: use a spanning random-access reader
   return readMultipartZip(filePaths);
 }
 

worker/src/archive/zip-spanned.test.ts

@@ -0,0 +1,214 @@
+import { describe, it, expect, beforeAll, afterAll } from "vitest";
+import { mkdir, mkdtemp, readdir, rm, writeFile } from "fs/promises";
+import { execFile } from "child_process";
+import { promisify } from "util";
+import { tmpdir } from "os";
+import path from "path";
+import { isSpannedZipPartSet } from "./zip-spanned.js";
+import { readZipCentralDirectory } from "./zip-reader.js";
+import { buildSpannedStoreZip, buildStoreZip, byteSplit } from "./testing/spanned-zip-fixture.js";
+
+const execFileAsync = promisify(execFile);
+
+let dir: string;
+
+beforeAll(async () => {
+  dir = await mkdtemp(path.join(tmpdir(), "zip-spanned-"));
+});
+
+afterAll(async () => {
+  await rm(dir, { recursive: true, force: true });
+});
+
+/** Write volume buffers out under the given names and return their paths. */
+async function writeParts(names: string[], buffers: Buffer[], sub: string): Promise<string[]> {
+  const base = path.join(dir, sub);
+  await rm(base, { recursive: true, force: true });
+  await mkdir(base, { recursive: true });
+  const paths: string[] = [];
+  for (let i = 0; i < names.length; i++) {
+    const p = path.join(base, names[i]);
+    await writeFile(p, buffers[i]);
+    paths.push(p);
+  }
+  return paths;
+}
+
+const FILES = [
+  { name: "src/b.bin", data: Buffer.alloc(2048, 7), disk: 0 },
+  { name: "src/models/dragon.stl", data: Buffer.from("DRAGON"), disk: 1 },
+  { name: "src/models/", data: Buffer.alloc(0), disk: 1 },
+  { name: "src/a.bin", data: Buffer.alloc(4096, 3), disk: 2 },
+  { name: "readme.txt", data: Buffer.from("hello world"), disk: 3 },
+];
+
+describe("isSpannedZipPartSet", () => {
+  it("recognizes a .z01 + .zip volume set", () => {
+    expect(isSpannedZipPartSet(["/t/Pack.z01", "/t/Pack.z02", "/t/Pack.zip"])).toBe(true);
+  });
+
+  it("recognizes the set regardless of the order it is handed in", () => {
+    expect(isSpannedZipPartSet(["/t/Pack.zip", "/t/Pack.z02", "/t/Pack.z01"])).toBe(true);
+  });
+
+  it("rejects a 7-Zip raw byte split (.zip.001)", () => {
+    expect(isSpannedZipPartSet(["/t/Pack.zip.001", "/t/Pack.zip.002"])).toBe(false);
+  });
+
+  it("rejects a single .zip", () => {
+    expect(isSpannedZipPartSet(["/t/Pack.zip"])).toBe(false);
+  });
+
+  it("rejects a set with no final .zip volume", () => {
+    expect(isSpannedZipPartSet(["/t/Pack.z01", "/t/Pack.z02"])).toBe(false);
+  });
+
+  it("rejects a set with two .zip volumes", () => {
+    expect(isSpannedZipPartSet(["/t/A.zip", "/t/B.zip"])).toBe(false);
+  });
+});
+
+describe("readZipCentralDirectory on a spanned (.z01 + .zip) set", () => {
+  it("lists every entry with correct paths, sizes and crc32", async () => {
+    const vols = buildSpannedStoreZip(FILES, 4);
+    const paths = await writeParts(["Pack.z01", "Pack.z02", "Pack.z03", "Pack.zip"], vols, "spanned");
+
+    const entries = await readZipCentralDirectory(paths);
+
+    expect(entries.map((e) => e.path).sort()).toEqual([
+      "readme.txt",
+      "src/a.bin",
+      "src/b.bin",
+      "src/models/dragon.stl",
+    ]);
+    const dragon = entries.find((e) => e.fileName === "dragon.stl")!;
+    expect(dragon.uncompressedSize).toBe(6n);
+    expect(dragon.extension).toBe("stl");
+    expect(dragon.crc32).toMatch(/^[0-9a-f]{8}$/);
+    const a = entries.find((e) => e.fileName === "a.bin")!;
+    expect(a.uncompressedSize).toBe(4096n);
+    expect(a.compressedSize).toBe(4096n);
+  });
+
+  it("works when the parts are handed over out of volume order", async () => {
+    const vols = buildSpannedStoreZip(FILES, 4);
+    const paths = await writeParts(["Pack.z01", "Pack.z02", "Pack.z03", "Pack.zip"], vols, "unordered");
+    const shuffled = [paths[3], paths[1], paths[0], paths[2]];
+
+    const entries = await readZipCentralDirectory(shuffled);
+    expect(entries).toHaveLength(4);
+  });
+
+  it("reads a central directory that begins on an earlier volume and spills forward", async () => {
+    // Files live on volumes 0–2; the directory starts on volume 2 and
+    // continues onto the final .zip, which holds nothing else.
+    const spillFiles = FILES.map((f) => ({ ...f, disk: Math.min(f.disk, 2) }));
+    const vols = buildSpannedStoreZip(spillFiles, 4, { cdStartDisk: 2 });
+    const paths = await writeParts(["Pack.z01", "Pack.z02", "Pack.z03", "Pack.zip"], vols, "spilled");
+
+    const entries = await readZipCentralDirectory(paths);
+    expect(entries.map((e) => e.fileName).sort()).toEqual(["a.bin", "b.bin", "dragon.stl", "readme.txt"]);
+  });
+
+  it("returns [] instead of throwing when a needed volume is missing", async () => {
+    const spillFiles = FILES.map((f) => ({ ...f, disk: Math.min(f.disk, 1) }));
+    const vols = buildSpannedStoreZip(spillFiles, 3, { cdStartDisk: 1 });
+    const paths = await writeParts(["Pack.z01", "Pack.z02", "Pack.zip"], vols, "missing");
+    // Drop Pack.z02 — the volume the central directory starts on.
+    const entries = await readZipCentralDirectory([paths[0], paths[2]]);
+    expect(entries).toEqual([]);
+  });
+
+  it("returns [] instead of throwing when the final volume is garbage", async () => {
+    const vols = buildSpannedStoreZip([{ name: "x.stl", data: Buffer.alloc(64, 1), disk: 0 }], 2);
+    const paths = await writeParts(["Pack.z01", "Pack.zip"], [vols[0], Buffer.alloc(500, 0x5a)], "garbage");
+    expect(await readZipCentralDirectory(paths)).toEqual([]);
+  });
+
+  it("falls back to concatenation semantics when .z01-named parts are really a byte split", async () => {
+    // Some producers name a raw byte split .z01/.zip. The EOCD then reports
+    // disk 0, so concatenation — not volume mapping — is the correct reading.
+    const zip = buildStoreZip([
+      { name: "one.stl", data: Buffer.alloc(3000, 1) },
+      { name: "two.stl", data: Buffer.alloc(3000, 2) },
+    ]);
+    const paths = await writeParts(["Pack.z01", "Pack.zip"], byteSplit(zip, 2), "mislabeled");
+    const entries = await readZipCentralDirectory(paths);
+    expect(entries.map((e) => e.fileName).sort()).toEqual(["one.stl", "two.stl"]);
+  });
+});
+
+describe("readZipCentralDirectory regressions for the shapes that already worked", () => {
+  it("still reads a 7-Zip raw byte split (.zip.001 …)", async () => {
+    const zip = buildStoreZip([
+      { name: "models/knight.stl", data: Buffer.alloc(5000, 9) },
+      { name: "license.txt", data: Buffer.from("MIT") },
+    ]);
+    const paths = await writeParts(
+      ["Pack.zip.001", "Pack.zip.002", "Pack.zip.003"],
+      byteSplit(zip, 3),
+      "bytesplit"
+    );
+    const entries = await readZipCentralDirectory(paths);
+    expect(entries.map((e) => e.fileName).sort()).toEqual(["knight.stl", "license.txt"]);
+    expect(entries.find((e) => e.fileName === "knight.stl")!.uncompressedSize).toBe(5000n);
+  });
+
+  it("still reads a plain single .zip", async () => {
+    const zip = buildStoreZip([{ name: "solo.stl", data: Buffer.from("SOLO") }]);
+    const paths = await writeParts(["Pack.zip"], [zip], "single");
+    const entries = await readZipCentralDirectory(paths);
+    expect(entries.map((e) => e.fileName)).toEqual(["solo.stl"]);
+  });
+});
+
+// ── Cross-checks against real Info-ZIP output ────────────────────────────
+// Skipped automatically where the `zip` CLI is unavailable; the hand-built
+// fixtures above are the authoritative, portable coverage.
+
+const HAS_ZIP_CLI = await execFileAsync("zip", ["-v"]).then(
+  () => true,
+  () => false
+);
+
+/** Collect Pack.z01 … Pack.zip from a directory, in volume order. */
+async function collectVolumes(work: string): Promise<string[]> {
+  const names = await readdir(work);
+  const vols = names.filter((n) => /^Pack\.z\d{2,}$/i.test(n)).sort();
+  const final = names.find((n) => /^Pack\.zip$/i.test(n));
+  expect(final).toBeDefined();
+  expect(vols.length).toBeGreaterThan(0);
+  return [...vols, final!].map((n) => path.join(work, n));
+}
+
+describe.skipIf(!HAS_ZIP_CLI)("readZipCentralDirectory against archives produced by Info-ZIP `zip -s`", () => {
+  it("lists a genuine spanned archive", async () => {
+    const work = path.join(dir, "real");
+    await mkdir(path.join(work, "src/models"), { recursive: true });
+    await writeFile(path.join(work, "src/big.bin"), Buffer.alloc(300_000, 4));
+    await writeFile(path.join(work, "src/models/dragon.stl"), Buffer.alloc(200_000, 5));
+    await writeFile(path.join(work, "src/readme.txt"), "hello world");
+    await execFileAsync("zip", ["-r", "-0", "-s", "100k", "Pack.zip", "src"], { cwd: work });
+
+    const entries = await readZipCentralDirectory(await collectVolumes(work));
+    expect(entries.map((e) => e.path).sort()).toEqual([
+      "src/big.bin",
+      "src/models/dragon.stl",
+      "src/readme.txt",
+    ]);
+    expect(entries.find((e) => e.fileName === "dragon.stl")!.uncompressedSize).toBe(200_000n);
+  });
+
+  it("lists a genuine ZIP64 spanned archive", async () => {
+    const work = path.join(dir, "real64");
+    await mkdir(work, { recursive: true });
+    await writeFile(path.join(work, "big.bin"), Buffer.alloc(300_000, 6));
+    await writeFile(path.join(work, "note.txt"), "zip64 spanned");
+    // -fz forces ZIP64 structures even though the payload is small.
+    await execFileAsync("zip", ["-0", "-fz", "-s", "100k", "Pack.zip", "big.bin", "note.txt"], { cwd: work });
+
+    const entries = await readZipCentralDirectory(await collectVolumes(work));
+    expect(entries.map((e) => e.fileName).sort()).toEqual(["big.bin", "note.txt"]);
+    expect(entries.find((e) => e.fileName === "big.bin")!.uncompressedSize).toBe(300_000n);
+  });
+});

worker/src/archive/zip-spanned.ts

@@ -0,0 +1,170 @@
+import { open as fsOpen, stat as fsStat } from "fs/promises";
+import path from "path";
+import { findEocdOffset, walkCentralDirectory, MIN_ZIP_TAIL_BYTES } from "./central-directory.js";
+import { childLogger } from "../util/logger.js";
+import type { FileEntry } from "./zip-reader.js";
+
+const log = childLogger("zip-spanned");
+
+const ZIP64_LOCATOR_SIG = 0x07064b50;
+const ZIP64_EOCD_SIG = 0x06064b50;
+
+/** Refuse to allocate a buffer for an absurd central-directory size. */
+const MAX_CD_BYTES = 256 * 1024 * 1024;
+
+/**
+ * A `.z01`/`.z02`/…/`.zip` set is a ZIP-spec **spanned (multi-disk)** archive:
+ * a genuinely different on-disk format from a 7-Zip raw byte split
+ * (`.zip.001`, `.zip.002`, …), which is one ZIP file cut into chunks.
+ *
+ * The distinction matters because a byte split is read by concatenating the
+ * chunks, whereas in a spanned archive each volume is its own unit: the EOCD's
+ * central-directory offset is relative to the start of the volume that holds
+ * the directory, and central-directory records carry a volume number. Feeding
+ * a spanned set to a concatenating reader yields nonsense offsets (and yauzl
+ * refuses outright: "multi-disk zip files are not supported").
+ *
+ * Detected from filename shape rather than the detector's multipart `pattern`
+ * so this stays independent of how `detect.ts` labels the two variants.
+ * Order-independent: the caller may hand the volumes over in any order.
+ */
+export function isSpannedZipPartSet(filePaths: string[]): boolean {
+  if (filePaths.length < 2) return false;
+  const names = filePaths.map((p) => path.basename(p));
+  const finals = names.filter((n) => /\.zip$/i.test(n));
+  const volumes = names.filter((n) => /\.z\d{2,}$/i.test(n));
+  return finals.length === 1 && volumes.length === names.length - 1;
+}
+
+export type SpannedZipResult =
+  /** Successfully read; `entries` may legitimately be empty for an empty archive. */
+  | { kind: "entries"; entries: FileEntry[] }
+  /** The EOCD reports a single disk — the parts are really a byte split, so the
+   *  caller should fall back to reading them as one concatenated stream. */
+  | { kind: "not-spanned" }
+  | { kind: "failed"; reason: string };
+
+/**
+ * Read the central directory of a ZIP-spec spanned archive.
+ *
+ * Only the volume holding the central directory (plus any it spills onto) is
+ * read, and only the directory bytes themselves — the file payloads are never
+ * touched, so this is cheap regardless of archive size.
+ *
+ * Coverage: STORE/DEFLATE and ZIP64 spanned archives are handled. Archives
+ * whose central directory is itself encrypted (strong encryption / "hide
+ * filenames") cannot be listed by any header-only reader and return `failed`.
+ */
+export async function readSpannedZipCentralDirectory(filePaths: string[]): Promise<SpannedZipResult> {
+  const volumes = new Map<number, string>();
+  let finalVolume: string | undefined;
+  for (const p of filePaths) {
+    const base = path.basename(p);
+    const m = base.match(/\.z(\d{2,})$/i);
+    if (m) {
+      // .z01 is volume 0, .z02 is volume 1, … (APPNOTE numbers disks from 0).
+      volumes.set(parseInt(m[1], 10) - 1, p);
+    } else if (/\.zip$/i.test(base)) {
+      finalVolume = p;
+    }
+  }
+  if (!finalVolume) return { kind: "failed", reason: "no final .zip volume" };
+
+  const finalSize = (await fsStat(finalVolume)).size;
+  const tailLen = Math.min(finalSize, MIN_ZIP_TAIL_BYTES);
+  const tailStart = finalSize - tailLen;
+  const tail = await readBytes(finalVolume, tailStart, tailLen);
+
+  const eocdPos = findEocdOffset(tail);
+  if (eocdPos < 0) return { kind: "failed", reason: "EOCD not found in final volume" };
+
+  let thisDisk = tail.readUInt16LE(eocdPos + 4);
+  let cdStartDisk = tail.readUInt16LE(eocdPos + 6);
+  let cdSize = tail.readUInt32LE(eocdPos + 12);
+  let cdOffset = tail.readUInt32LE(eocdPos + 16);
+
+  // ZIP64: any saturated field means the real values live in the ZIP64 EOCD
+  // record, which in a spanned archive may sit on a different volume.
+  if (
+    thisDisk === 0xffff ||
+    cdStartDisk === 0xffff ||
+    cdSize === 0xffffffff ||
+    cdOffset === 0xffffffff
+  ) {
+    const locPos = findZip64Locator(tail, eocdPos);
+    if (locPos < 0) return { kind: "failed", reason: "ZIP64 locator not found" };
+    const locDisk = tail.readUInt32LE(locPos + 4);
+    const locOffset = Number(tail.readBigUInt64LE(locPos + 8));
+
+    // The locator's disk number counts the final volume too, so resolve it
+    // through the same map, treating the final .zip as the highest volume.
+    const z64Path =
+      locDisk === thisDisk || locDisk === 0xffff ? finalVolume : volumes.get(locDisk) ?? finalVolume;
+    const z64 = await readBytes(z64Path, locOffset, 56);
+    if (z64.length < 56 || z64.readUInt32LE(0) !== ZIP64_EOCD_SIG) {
+      return { kind: "failed", reason: "ZIP64 EOCD record unreadable" };
+    }
+    thisDisk = z64.readUInt32LE(16);
+    cdStartDisk = z64.readUInt32LE(20);
+    cdSize = Number(z64.readBigUInt64LE(40));
+    cdOffset = Number(z64.readBigUInt64LE(48));
+  }
+
+  // A byte split named .z01/.zip still reports a single disk — concatenation,
+  // not volume mapping, is the correct reading for it.
+  if (thisDisk === 0 && cdStartDisk === 0) return { kind: "not-spanned" };
+
+  if (cdSize < 0 || cdSize > MAX_CD_BYTES) {
+    return { kind: "failed", reason: `implausible central directory size ${cdSize}` };
+  }
+
+  // The final .zip file is the highest-numbered volume.
+  volumes.set(thisDisk, finalVolume);
+
+  // Read cdSize bytes from (cdStartDisk, cdOffset), spilling onto later
+  // volumes if the directory straddles a volume boundary.
+  const chunks: Buffer[] = [];
+  let remaining = cdSize;
+  let disk = cdStartDisk;
+  let offset = cdOffset;
+  while (remaining > 0) {
+    const volPath = volumes.get(disk);
+    if (!volPath) return { kind: "failed", reason: `volume ${disk + 1} of the set is missing` };
+    const chunk = await readBytes(volPath, offset, remaining);
+    if (chunk.length === 0) {
+      return { kind: "failed", reason: `volume ${disk + 1} ended before the central directory did` };
+    }
+    chunks.push(chunk);
+    remaining -= chunk.length;
+    disk++;
+    offset = 0;
+  }
+
+  const cdBuf = Buffer.concat(chunks);
+  const entries = walkCentralDirectory(cdBuf, 0, cdSize);
+  log.debug(
+    { volumes: volumes.size, thisDisk, cdStartDisk, cdSize, entries: entries.length },
+    "Read spanned ZIP central directory"
+  );
+  return { kind: "entries", entries };
+}
+
+/** Find the ZIP64 EOCD locator, which sits just before the EOCD record. */
+function findZip64Locator(tail: Buffer, eocdPos: number): number {
+  for (let i = Math.min(eocdPos - 20, tail.length - 20); i >= 0; i--) {
+    if (tail.readUInt32LE(i) === ZIP64_LOCATOR_SIG) return i;
+  }
+  return -1;
+}
+
+/** Read up to `length` bytes at `offset`; a short read means end of file. */
+async function readBytes(filePath: string, offset: number, length: number): Promise<Buffer> {
+  const fh = await fsOpen(filePath, "r");
+  try {
+    const buf = Buffer.alloc(length);
+    const { bytesRead } = await fh.read(buf, 0, length, offset);
+    return buf.subarray(0, bytesRead);
+  } finally {
+    await fh.close();
+  }
+}

worker/src/backfill-scope.test.ts

@@ -0,0 +1,121 @@
+import { describe, it, expect } from "vitest";
+import {
+  parseBackfillPayload,
+  parseFileNameLike,
+  MAX_BACKFILL_LIMIT,
+  DEFAULT_BACKFILL_LIMIT,
+} from "./backfill-scope.js";
+
+/** The plan of a payload that must parse; fails the test if it doesn't. */
+function planOf(payload: string) {
+  const parsed = parseBackfillPayload(payload);
+  if (!parsed.ok) throw new Error(`expected payload to parse, got: ${parsed.error}`);
+  return parsed.plan;
+}
+
+function errorOf(payload: string): string {
+  const parsed = parseBackfillPayload(payload);
+  if (parsed.ok) throw new Error(`expected payload to be rejected, got plan: ${parsed.plan.describe}`);
+  return parsed.error;
+}
+
+describe("parseBackfillPayload — refusing the unbounded sweep", () => {
+  it("rejects an empty payload rather than selecting every empty package", () => {
+    expect(errorOf("{}")).toMatch(/refusing an unbounded backfill/);
+    expect(errorOf("")).toMatch(/refusing an unbounded backfill/);
+  });
+
+  it("rejects a payload that only names an archiveType", () => {
+    // This is the shape that used to select 4,330 packages / 5.4TB.
+    expect(errorOf('{"archiveType":"ZIP"}')).toMatch(/refusing an unbounded backfill/);
+  });
+
+  it("rejects invalid JSON and non-object payloads", () => {
+    expect(errorOf("not json")).toMatch(/not valid JSON/);
+    expect(errorOf("[]")).toMatch(/must be a JSON object/);
+    expect(errorOf("null")).toMatch(/must be a JSON object/);
+  });
+
+  it("rejects an unknown field instead of ignoring it", () => {
+    // A typo'd selector must not silently degrade into "no selector".
+    expect(errorOf('{"packageIDs":["abc"]}')).toMatch(/unknown field\(s\): packageIDs/);
+  });
+
+  it("allows the broad sweep only when it asks for itself", () => {
+    const plan = planOf('{"archiveType":"RAR","limit":50,"allowBroadSweep":true}');
+    expect(plan.selector).toEqual({ archiveType: "RAR" });
+    expect(plan.limit).toBe(50);
+  });
+
+  it("rejects allowBroadSweep combined with a selector as a contradiction", () => {
+    expect(errorOf('{"fileNameLike":"%.z01","allowBroadSweep":true}')).toMatch(/unscoped sweeps only/);
+  });
+});
+
+describe("parseBackfillPayload — packageIds", () => {
+  it("accepts, trims and de-duplicates an explicit id list", () => {
+    const plan = planOf('{"packageIds":[" abc123 ","abc123","def456"]}');
+    expect(plan.selector.packageIds).toEqual(["abc123", "def456"]);
+  });
+
+  it("rejects an empty list, non-strings and non-identifier ids", () => {
+    expect(errorOf('{"packageIds":[]}')).toMatch(/must not be empty/);
+    expect(errorOf('{"packageIds":[1,2]}')).toMatch(/only strings/);
+    expect(errorOf('{"packageIds":["a\'; DROP TABLE packages--"]}')).toMatch(/not a valid identifier/);
+    expect(errorOf('{"packageIds":"abc"}')).toMatch(/must be an array/);
+  });
+});
+
+describe("parseFileNameLike", () => {
+  it("maps the four accepted wildcard positions to Prisma filters", () => {
+    expect(parseFileNameLike("%.z01")).toEqual({ endsWith: ".z01" });
+    expect(parseFileNameLike("Dragon%")).toEqual({ startsWith: "Dragon" });
+    expect(parseFileNameLike("%dragon%")).toEqual({ contains: "dragon" });
+    expect(parseFileNameLike("Pack.z01")).toEqual({ equals: "Pack.z01" });
+  });
+
+  it("refuses patterns that would match far more than intended", () => {
+    expect(parseFileNameLike("%")).toHaveProperty("error");
+    expect(parseFileNameLike("%%")).toHaveProperty("error");
+    expect(parseFileNameLike("  ")).toHaveProperty("error");
+    // Too little literal text to be a meaningful scope
+    expect(parseFileNameLike("%a%")).toHaveProperty("error");
+  });
+
+  it("refuses interior and underscore wildcards rather than mis-translating them", () => {
+    expect(parseFileNameLike("%.z%1")).toMatchObject({ error: expect.stringMatching(/interior wildcard/) });
+    expect(parseFileNameLike("%.z0_")).toMatchObject({ error: expect.stringMatching(/_ wildcard/) });
+  });
+});
+
+describe("parseBackfillPayload — limits and flags", () => {
+  it("defaults the limit and caps it", () => {
+    expect(planOf('{"fileNameLike":"%.z01"}').limit).toBe(DEFAULT_BACKFILL_LIMIT);
+    expect(errorOf(`{"fileNameLike":"%.z01","limit":${MAX_BACKFILL_LIMIT + 1}}`)).toMatch(/exceeds the maximum/);
+    expect(errorOf('{"fileNameLike":"%.z01","limit":0}')).toMatch(/positive integer/);
+    expect(errorOf('{"fileNameLike":"%.z01","limit":1.5}')).toMatch(/positive integer/);
+  });
+
+  it("defaults rangedOnly and recoverDestIds off, and requires booleans", () => {
+    const plan = planOf('{"fileNameLike":"%.z01"}');
+    expect(plan.rangedOnly).toBe(false);
+    expect(plan.recoverDestIds).toBe(false);
+    expect(errorOf('{"fileNameLike":"%.z01","rangedOnly":"yes"}')).toMatch(/rangedOnly must be a boolean/);
+  });
+
+  it("parses the full spanned-ZIP repair payload", () => {
+    const plan = planOf(
+      '{"fileNameLike":"%.z01","archiveType":"ZIP","limit":250,"rangedOnly":true,"recoverDestIds":true}'
+    );
+    expect(plan.selector).toEqual({ fileName: { endsWith: ".z01" }, archiveType: "ZIP" });
+    expect(plan.limit).toBe(250);
+    expect(plan.rangedOnly).toBe(true);
+    expect(plan.recoverDestIds).toBe(true);
+    expect(plan.describe).toContain('fileName.endsWith=".z01"');
+    expect(plan.describe).toContain("rangedOnly");
+  });
+
+  it("rejects an unsupported archiveType", () => {
+    expect(errorOf('{"fileNameLike":"%.z01","archiveType":"TAR"}')).toMatch(/archiveType must be one of/);
+  });
+});

worker/src/backfill-scope.ts

@@ -0,0 +1,244 @@
+/**
+ * Payload parsing for the `backfill_filelists` pg_notify request.
+ *
+ * The original payload shape was `{limit, archiveType}` — both optional, both
+ * defaulted. That made the *unbounded* sweep the easiest thing to trigger:
+ * `SELECT pg_notify('backfill_filelists', '{}')` selected every Package with
+ * `fileCount = 0` of every archive type, oldest first, and started downloading.
+ * On a real catalogue that is multiple terabytes of traffic aimed at packages
+ * nobody asked to repair.
+ *
+ * So the rule here is: **a request must name what it wants**. A payload with no
+ * selector is rejected outright, and the broad "every empty package of type X"
+ * sweep has to opt in explicitly via `allowBroadSweep`. Omitting a field can
+ * only ever narrow the job or fail it — never widen it.
+ */
+
+export type BackfillArchiveType = "ZIP" | "RAR" | "SEVEN_Z";
+
+const ARCHIVE_TYPES: BackfillArchiveType[] = ["ZIP", "RAR", "SEVEN_Z"];
+
+/** Hard ceiling on `limit`, so even a deliberate broad sweep stays bounded. */
+export const MAX_BACKFILL_LIMIT = 2000;
+/** Hard ceiling on an explicit id list — keeps the SQL `IN (...)` sane. */
+export const MAX_BACKFILL_PACKAGE_IDS = 2000;
+export const DEFAULT_BACKFILL_LIMIT = 200;
+
+/** Minimum literal characters in a `fileNameLike` pattern, so `%` can't stand alone. */
+const MIN_FILENAME_LITERAL = 3;
+
+/** Prisma-shaped filename filter — a closed set of forms, never raw SQL. */
+export type FileNameFilter =
+  | { equals: string }
+  | { startsWith: string }
+  | { endsWith: string }
+  | { contains: string };
+
+export interface BackfillSelector {
+  /** Explicit package ids: the most tightly bounded selector there is. */
+  packageIds?: string[];
+  fileName?: FileNameFilter;
+  archiveType?: BackfillArchiveType;
+}
+
+export interface BackfillPlan {
+  selector: BackfillSelector;
+  limit: number;
+  /**
+   * Never fall back to a full `downloadFile` of the archive, even when the
+   * cheap ranged read fails. Set this for repairs where the full-download cost
+   * would be absurd (a spanned set's listing lives in ~64KB of its final
+   * volume; downloading the set to reach it can be hundreds of gigabytes).
+   */
+  rangedOnly: boolean;
+  /**
+   * Allow one scan of the destination channel to recover `destMessageIds` for
+   * candidates that have none. Opt-in because the scan itself costs a few
+   * hundred paginated `searchChatMessages` calls.
+   */
+  recoverDestIds: boolean;
+  /** Compact description of the scope, for the batch log line. */
+  describe: string;
+}
+
+export type ParsedBackfillPayload =
+  | { ok: true; plan: BackfillPlan }
+  | { ok: false; error: string };
+
+/**
+ * Translate a restricted LIKE pattern into a Prisma filter.
+ *
+ * Only leading and/or trailing `%` are accepted — an interior `%`, a `_`
+ * wildcard, or a pattern with too little literal text is rejected rather than
+ * quietly matching far more than the caller meant.
+ */
+export function parseFileNameLike(pattern: string): FileNameFilter | { error: string } {
+  if (typeof pattern !== "string") return { error: "fileNameLike must be a string" };
+  const raw = pattern.trim();
+  if (raw.length === 0) return { error: "fileNameLike must not be empty" };
+
+  const leading = raw.startsWith("%");
+  const trailing = raw.endsWith("%");
+  const literal = raw.slice(leading ? 1 : 0, trailing && raw.length > 1 ? -1 : undefined);
+
+  if (literal.includes("%")) {
+    return { error: "fileNameLike supports a leading and/or trailing % only (no interior wildcard)" };
+  }
+  if (literal.includes("_")) {
+    return { error: "fileNameLike does not support the _ wildcard — use % or a literal name" };
+  }
+  if (literal.length < MIN_FILENAME_LITERAL) {
+    return {
+      error: `fileNameLike needs at least ${MIN_FILENAME_LITERAL} literal characters (got "${literal}")`,
+    };
+  }
+
+  if (leading && trailing) return { contains: literal };
+  if (leading) return { endsWith: literal };
+  if (trailing) return { startsWith: literal };
+  return { equals: literal };
+}
+
+function parsePackageIds(value: unknown): string[] | { error: string } {
+  if (!Array.isArray(value)) return { error: "packageIds must be an array of package ids" };
+  if (value.length === 0) return { error: "packageIds must not be empty" };
+  if (value.length > MAX_BACKFILL_PACKAGE_IDS) {
+    return { error: `packageIds holds ${value.length} ids — the maximum is ${MAX_BACKFILL_PACKAGE_IDS}` };
+  }
+  const ids: string[] = [];
+  const seen = new Set<string>();
+  for (const entry of value) {
+    if (typeof entry !== "string") return { error: "packageIds must contain only strings" };
+    const id = entry.trim();
+    // cuid()s are alphanumeric; the character class also rules out anything
+    // that could confuse a log line or a hand-written SQL check.
+    if (!/^[A-Za-z0-9_-]{1,64}$/.test(id)) {
+      return { error: `packageIds contains an id that is not a valid identifier: "${entry}"` };
+    }
+    if (seen.has(id)) continue;
+    seen.add(id);
+    ids.push(id);
+  }
+  return ids;
+}
+
+function describeSelector(selector: BackfillSelector, plan: Pick<BackfillPlan, "limit" | "rangedOnly" | "recoverDestIds">): string {
+  const bits: string[] = [];
+  if (selector.packageIds) bits.push(`packageIds=${selector.packageIds.length}`);
+  if (selector.fileName) {
+    const [key, value] = Object.entries(selector.fileName)[0];
+    bits.push(`fileName.${key}=${JSON.stringify(value)}`);
+  }
+  bits.push(`archiveType=${selector.archiveType ?? "ZIP|RAR|SEVEN_Z"}`);
+  bits.push(`limit=${plan.limit}`);
+  if (plan.rangedOnly) bits.push("rangedOnly");
+  if (plan.recoverDestIds) bits.push("recoverDestIds");
+  return bits.join(" ");
+}
+
+/**
+ * Parse and validate a `backfill_filelists` payload.
+ *
+ * Accepted fields (all optional except that *some* selector is required):
+ *   packageIds      string[]  — repair exactly these packages
+ *   fileNameLike    string    — restricted LIKE: "%.z01", "Pack%", "%dragon%"
+ *   archiveType     ZIP|RAR|SEVEN_Z
+ *   limit           number    — 1..MAX_BACKFILL_LIMIT, default DEFAULT_BACKFILL_LIMIT
+ *   rangedOnly      boolean   — refuse the full-download fallback
+ *   recoverDestIds  boolean   — allow one destination-channel scan to recover ids
+ *   allowBroadSweep boolean   — required to run with no narrowing selector
+ */
+export function parseBackfillPayload(payloadJson: string): ParsedBackfillPayload {
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(payloadJson === "" ? "{}" : payloadJson);
+  } catch {
+    return { ok: false, error: "payload is not valid JSON" };
+  }
+  if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) {
+    return { ok: false, error: "payload must be a JSON object" };
+  }
+  const raw = parsed as Record<string, unknown>;
+
+  const known = new Set([
+    "packageIds",
+    "fileNameLike",
+    "archiveType",
+    "limit",
+    "rangedOnly",
+    "recoverDestIds",
+    "allowBroadSweep",
+  ]);
+  const unknownKeys = Object.keys(raw).filter((k) => !known.has(k));
+  if (unknownKeys.length > 0) {
+    // Fail rather than ignore: a typo'd `packageIDs` would otherwise silently
+    // become "no selector" and — worse, if allowBroadSweep were also set —
+    // a full sweep.
+    return { ok: false, error: `unknown field(s): ${unknownKeys.join(", ")}` };
+  }
+
+  let limit = DEFAULT_BACKFILL_LIMIT;
+  if (raw.limit !== undefined) {
+    if (typeof raw.limit !== "number" || !Number.isInteger(raw.limit) || raw.limit < 1) {
+      return { ok: false, error: "limit must be a positive integer" };
+    }
+    if (raw.limit > MAX_BACKFILL_LIMIT) {
+      return { ok: false, error: `limit ${raw.limit} exceeds the maximum of ${MAX_BACKFILL_LIMIT}` };
+    }
+    limit = raw.limit;
+  }
+
+  for (const flag of ["rangedOnly", "recoverDestIds", "allowBroadSweep"] as const) {
+    if (raw[flag] !== undefined && typeof raw[flag] !== "boolean") {
+      return { ok: false, error: `${flag} must be a boolean` };
+    }
+  }
+  const rangedOnly = raw.rangedOnly === true;
+  const recoverDestIds = raw.recoverDestIds === true;
+  const allowBroadSweep = raw.allowBroadSweep === true;
+
+  const selector: BackfillSelector = {};
+
+  if (raw.packageIds !== undefined) {
+    const ids = parsePackageIds(raw.packageIds);
+    if (!Array.isArray(ids)) return { ok: false, error: ids.error };
+    selector.packageIds = ids;
+  }
+
+  if (raw.fileNameLike !== undefined) {
+    const filter = parseFileNameLike(raw.fileNameLike as string);
+    if ("error" in filter) return { ok: false, error: filter.error };
+    selector.fileName = filter;
+  }
+
+  if (raw.archiveType !== undefined) {
+    if (typeof raw.archiveType !== "string" || !ARCHIVE_TYPES.includes(raw.archiveType as BackfillArchiveType)) {
+      return { ok: false, error: `archiveType must be one of ${ARCHIVE_TYPES.join(", ")}` };
+    }
+    selector.archiveType = raw.archiveType as BackfillArchiveType;
+  }
+
+  const isNarrowed = selector.packageIds !== undefined || selector.fileName !== undefined;
+  if (!isNarrowed && !allowBroadSweep) {
+    return {
+      ok: false,
+      error:
+        "refusing an unbounded backfill: pass packageIds or fileNameLike to scope it, " +
+        'or set {"allowBroadSweep":true} to deliberately sweep every empty package',
+    };
+  }
+  if (allowBroadSweep && isNarrowed) {
+    return { ok: false, error: "allowBroadSweep is for unscoped sweeps only — drop it or drop the selector" };
+  }
+
+  return {
+    ok: true,
+    plan: {
+      selector,
+      limit,
+      rangedOnly,
+      recoverDestIds,
+      describe: describeSelector(selector, { limit, rangedOnly, recoverDestIds }),
+    },
+  };
+}

worker/src/backfill.ts

@@ -1,79 +1,71 @@
 import path from "path";
+import type { Client } from "tdl";
 import { mkdir, rm } from "fs/promises";
+import type { Prisma } from "@prisma/client";
 import { db } from "./db/client.js";
 import { config } from "./util/config.js";
 import { childLogger } from "./util/logger.js";
 import { withTdlibMutex } from "./util/mutex.js";
 import { createTdlibClient, closeTdlibClient } from "./tdlib/client.js";
-import { downloadFile } from "./tdlib/download.js";
+import { downloadFile, invokeWithTimeout } from "./tdlib/download.js";
+import { scanChatDocuments, type ChatDocument } from "./tdlib/chat-documents.js";
 import { getActiveAccounts } from "./db/queries.js";
 import { readZipCentralDirectory } from "./archive/zip-reader.js";
 import { readRarContents } from "./archive/rar-reader.js";
 import { read7zContents } from "./archive/sevenz-reader.js";
 import { extractSlicerTags } from "./archive/slicer-tags.js";
+import { readScannedListingRanged } from "./archive/ranged/dispatch.js";
+import type { RangedPart } from "./archive/ranged/sevenz-ranged.js";
+import { planListingRead, planRangedFallback } from "./archive/listing-plan.js";
+import { buildDestIndex, resolveDestPartSet, type DestIndex } from "./dest-index.js";
+import { parseBackfillPayload, type BackfillPlan } from "./backfill-scope.js";
 import type { FileEntry } from "./archive/zip-reader.js";
 
 const log = childLogger("backfill");
 
 /**
- * Re-extract file listings for Packages whose fileCount is 0 — usually
+ * Re-extract file listings for Packages whose fileCount is 0 — historically a
- * caused by historical bugs in the archive readers (e.g. the RAR parser
+ * reader bug (the RAR parser that silently returned [] before 0bdd4ba; the
- * that silently returned [] for every archive before 0bdd4ba).
+ * spanned-ZIP offset bug before 402c317) rather than a genuinely empty archive.
  *
- * For each candidate Package:
+ * Two things dominate the cost of doing this, and both are handled here:
- *   1. Download all destMessageIds from the destination channel
- *   2. Run the appropriate reader (ZIP / RAR / 7Z) on the assembled files
- *   3. Insert PackageFile rows + update Package.fileCount
- *   4. Clean up the temp files
  *
- * Triggered via pg_notify "backfill_filelists" with optional payload
+ *  1. **Reading the listing.** A file list lives in a few tens of kilobytes of
- * `{"limit": N, "archiveType": "RAR"}` — both fields optional, defaults
+ *     an archive's header or tail. `readScannedListingRanged` fetches exactly
- * are limit=100, archiveType=any.
+ *     those bytes, so a 35GB spanned set costs ~64KB instead of 35GB. The
+ *     full-download path remains as a fallback for archives ranged reading
+ *     genuinely cannot handle — and `rangedOnly` turns it off when the
+ *     difference would be terabytes.
+ *
+ *  2. **Knowing which messages the archive's parts are.** A Package whose
+ *     `destMessageIds` array is empty falls back to `[destMessageId]`, which is
+ *     the *first* uploaded part. A lone `.z01` has no central directory at all,
+ *     so that package can never be listed. With `recoverDestIds` the batch pays
+ *     for one destination-channel scan and recovers the complete, ordered part
+ *     set for every candidate at once — then persists it, so the package stays
+ *     repairable.
+ *
+ * Triggered via pg_notify "backfill_filelists"; see `backfill-scope.ts` for the
+ * payload contract. A payload with no narrowing selector is rejected — the
+ * unscoped sweep has to ask for itself.
  */
 export async function processBackfillRequest(payloadJson: string): Promise<void> {
-  let limit = 100;
+  const parsed = parseBackfillPayload(payloadJson);
-  let archiveTypeFilter: "ZIP" | "RAR" | "SEVEN_Z" | undefined;
+  if (!parsed.ok) {
-  try {
+    log.warn({ payload: payloadJson, error: parsed.error }, "Backfill request rejected — nothing was read or written");
-    const parsed = JSON.parse(payloadJson) as { limit?: number; archiveType?: string };
+    return;
-    if (typeof parsed.limit === "number" && parsed.limit > 0) limit = parsed.limit;
-    if (parsed.archiveType === "ZIP" || parsed.archiveType === "RAR" || parsed.archiveType === "SEVEN_Z") {
-      archiveTypeFilter = parsed.archiveType;
-    }
-  } catch {
-    // Empty / invalid payload — use defaults
   }
+  const plan = parsed.plan;
 
-  const candidates = await db.package.findMany({
+  const candidates = await findBackfillCandidates(plan);
-    where: {
-      fileCount: 0,
-      destChannelId: { not: null },
-      destMessageId: { not: null },
-      archiveType: archiveTypeFilter
-        ? archiveTypeFilter
-        : { in: ["ZIP", "RAR", "SEVEN_Z"] },
-    },
-    select: {
-      id: true,
-      fileName: true,
-      fileSize: true,
-      archiveType: true,
-      destChannelId: true,
-      destMessageId: true,
-      destMessageIds: true,
-      isMultipart: true,
-      partCount: true,
-    },
-    orderBy: { createdAt: "asc" },
-    take: limit,
-  });
-
   if (candidates.length === 0) {
-    log.info({ archiveTypeFilter }, "Backfill: no candidates with fileCount=0");
+    log.info({ scope: plan.describe }, "Backfill: no candidates with fileCount=0");
     return;
   }
 
+  const totalBytes = candidates.reduce((sum, c) => sum + c.fileSize, 0n);
   log.info(
-    { count: candidates.length, archiveTypeFilter },
+    { count: candidates.length, scope: plan.describe, totalBytes: totalBytes.toString() },
     "Backfill: starting batch"
   );
 
@@ -101,33 +93,47 @@ export async function processBackfillRequest(payloadJson: string): Promise<void>
         // May already be loaded
       }
 
-      let processed = 0;
+      const destIndexes = await buildDestIndexes(client, plan, candidates);
-      let succeeded = 0;
+
-      let failed = 0;
+      const counters: BackfillCounters = {
+        processed: 0,
+        listedRanged: 0,
+        listedDownload: 0,
+        skipped: 0,
+        failed: 0,
+        idsRecovered: 0,
+        concatUnlistable: 0,
+      };
 
       for (const pkg of candidates) {
-        processed++;
+        counters.processed++;
         const ctx = { packageId: pkg.id, fileName: pkg.fileName };
 
         try {
-          await processOnePackage(client, pkg, ctx);
+          await repairOnePackage(client, pkg, ctx, plan, destIndexes, counters);
-          succeeded++;
         } catch (err) {
-          failed++;
+          counters.failed++;
           log.warn({ err, ...ctx }, "Backfill failed for package");
         }
       }
 
-      log.info(
+      log.info({ ...counters, scope: plan.describe }, "Backfill batch complete");
-        { processed, succeeded, failed, archiveTypeFilter },
-        "Backfill batch complete"
-      );
     } finally {
       await closeTdlibClient(client).catch(() => {});
     }
   });
 }
 
+interface BackfillCounters {
+  processed: number;
+  listedRanged: number;
+  listedDownload: number;
+  skipped: number;
+  failed: number;
+  idsRecovered: number;
+  concatUnlistable: number;
+}
+
 interface BackfillPackage {
   id: string;
   fileName: string;
@@ -140,95 +146,325 @@ interface BackfillPackage {
   partCount: number;
 }
 
-async function processOnePackage(
+/** Build the `where` clause from a validated plan. */
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+export function backfillCandidateWhere(plan: BackfillPlan): Prisma.PackageWhereInput {
-  client: any,
+  const { selector } = plan;
+  return {
+    fileCount: 0,
+    destChannelId: { not: null },
+    destMessageId: { not: null },
+    archiveType: selector.archiveType ?? { in: ["ZIP", "RAR", "SEVEN_Z"] },
+    ...(selector.packageIds ? { id: { in: selector.packageIds } } : {}),
+    ...(selector.fileName ? { fileName: selector.fileName } : {}),
+  };
+}
+
+async function findBackfillCandidates(plan: BackfillPlan): Promise<BackfillPackage[]> {
+  return db.package.findMany({
+    where: backfillCandidateWhere(plan),
+    select: {
+      id: true,
+      fileName: true,
+      fileSize: true,
+      archiveType: true,
+      destChannelId: true,
+      destMessageId: true,
+      destMessageIds: true,
+      isMultipart: true,
+      partCount: true,
+    },
+    orderBy: { createdAt: "asc" },
+    take: plan.limit,
+  });
+}
+
+/**
+ * Scan each destination channel that has candidates needing id recovery, once.
+ *
+ * The scan is the expensive part of a repair run (a few hundred paginated
+ * `searchChatMessages` calls on a large channel), so it is opt-in via
+ * `recoverDestIds` and it is amortised across the whole batch. Its by-product —
+ * a fileId and size for every destination document — also removes the need for
+ * a per-part `getMessage` on every candidate, recovered or not.
+ */
+async function buildDestIndexes(
+  client: Client,
+  plan: BackfillPlan,
+  candidates: BackfillPackage[]
+): Promise<Map<string, DestIndex>> {
+  const indexes = new Map<string, DestIndex>();
+  if (!plan.recoverDestIds) return indexes;
+
+  const channelIds = new Set<string>();
+  for (const pkg of candidates) {
+    if (pkg.destChannelId && pkg.destMessageIds.length === 0) channelIds.add(pkg.destChannelId);
+  }
+  if (channelIds.size === 0) {
+    log.info("Backfill: every candidate already has destMessageIds — skipping the destination scan");
+    return indexes;
+  }
+
+  for (const channelId of channelIds) {
+    const channel = await db.telegramChannel.findUnique({
+      where: { id: channelId },
+      select: { telegramId: true, title: true },
+    });
+    if (!channel) {
+      log.warn({ channelId }, "Backfill: destination channel not found in DB — cannot recover ids for it");
+      continue;
+    }
+    log.info({ channelId, title: channel.title }, "Backfill: scanning destination channel to recover destMessageIds");
+    const scan = await scanChatDocuments(client, channel.telegramId);
+    if (scan.truncated) {
+      log.warn(
+        { channelId, pages: scan.pages },
+        "Backfill: destination scan hit the page limit — recovery may be incomplete for older packages"
+      );
+    }
+    indexes.set(channelId, buildDestIndex(scan.documents));
+  }
+  return indexes;
+}
+
+/** Resolved destination parts plus how they were obtained. */
+interface ResolvedDestParts {
+  parts: RangedPart[];
+  /** Message ids in upload order, when a complete set was recovered from a scan. */
+  recoveredIds: bigint[] | null;
+}
+
+/**
+ * Turn a Package's destination messages into ranged parts (fileId + size + name),
+ * in upload order. Prefers the scan index (free) over `getMessage` (one API call
+ * per part), and recovers the full set from the anchor message when
+ * `destMessageIds` is empty.
+ */
+async function resolveDestParts(
+  client: Client,
   pkg: BackfillPackage,
-  ctx: { packageId: string; fileName: string }
+  chatTelegramId: bigint,
+  index: DestIndex | undefined
+): Promise<ResolvedDestParts | { error: string }> {
+  const toRangedPart = (doc: ChatDocument): RangedPart => ({
+    fileId: doc.fileId,
+    fileSize: doc.fileSize,
+    fileName: doc.fileName,
+  });
+
+  if (pkg.destMessageIds.length > 0) {
+    const parts: RangedPart[] = [];
+    for (const msgId of pkg.destMessageIds) {
+      const cached = index?.byMessageId.get(msgId.toString());
+      if (cached) {
+        parts.push(toRangedPart(cached));
+        continue;
+      }
+      const resolved = await fetchDocumentPart(client, chatTelegramId, msgId, pkg);
+      if ("error" in resolved) return resolved;
+      parts.push(resolved.part);
+    }
+    return { parts, recoveredIds: null };
+  }
+
+  if (!pkg.destMessageId) return { error: "package has no destination message id" };
+
+  // A single-part package needs no recovery: its one message id is complete.
+  if (pkg.partCount <= 1 && !pkg.isMultipart) {
+    const cached = index?.byMessageId.get(pkg.destMessageId.toString());
+    if (cached) return { parts: [toRangedPart(cached)], recoveredIds: null };
+    const resolved = await fetchDocumentPart(client, chatTelegramId, pkg.destMessageId, pkg);
+    if ("error" in resolved) return resolved;
+    return { parts: [resolved.part], recoveredIds: null };
+  }
+
+  if (!index) {
+    return {
+      error:
+        `destMessageIds is empty and the package has ${pkg.partCount} parts; ` +
+        "destMessageId alone is only the first part. Re-run with recoverDestIds to scan the destination channel",
+    };
+  }
+
+  const resolution = resolveDestPartSet(index, pkg.destMessageId, pkg.partCount);
+  if (!resolution.ok) return { error: resolution.reason };
+
+  return {
+    parts: resolution.parts.map(toRangedPart),
+    recoveredIds: resolution.parts.map((p) => p.id),
+  };
+}
+
+async function fetchDocumentPart(
+  client: Client,
+  chatTelegramId: bigint,
+  messageId: bigint,
+  pkg: BackfillPackage
+): Promise<{ part: RangedPart } | { error: string }> {
+  const message = await invokeWithTimeout<{
+    content?: { document?: { file_name?: string; document?: { id: number; size: number } } };
+  }>(client, {
+    _: "getMessage",
+    chat_id: Number(chatTelegramId),
+    message_id: Number(messageId),
+  });
+  const doc = message?.content?.document;
+  if (!doc?.document?.id) {
+    return { error: `destination message ${messageId} has no document` };
+  }
+  return {
+    part: {
+      fileId: String(doc.document.id),
+      fileSize: BigInt(doc.document.size),
+      fileName: doc.file_name ?? pkg.fileName,
+    },
+  };
+}
+
+async function repairOnePackage(
+  client: Client,
+  pkg: BackfillPackage,
+  ctx: { packageId: string; fileName: string },
+  plan: BackfillPlan,
+  destIndexes: Map<string, DestIndex>,
+  counters: BackfillCounters
 ): Promise<void> {
   if (!pkg.destChannelId || !pkg.destMessageId) {
-    log.debug(ctx, "Skipping: no destination channel/message");
+    counters.skipped++;
+    log.info({ ...ctx, reason: "no destination channel/message" }, "Backfill skipped");
     return;
   }
 
-  // Look up the destination channel's Telegram ID
   const destChannel = await db.telegramChannel.findUnique({
     where: { id: pkg.destChannelId },
     select: { telegramId: true },
   });
-  if (!destChannel) {
+  if (!destChannel) throw new Error("Destination channel not found in DB");
-    throw new Error("Destination channel not found in DB");
+
+  const index = destIndexes.get(pkg.destChannelId);
+  const resolved = await resolveDestParts(client, pkg, destChannel.telegramId, index);
+  if ("error" in resolved) {
+    counters.skipped++;
+    log.info({ ...ctx, reason: resolved.error }, "Backfill skipped");
+    return;
   }
-  const chatId = Number(destChannel.telegramId);
+  const { parts, recoveredIds } = resolved;
 
-  // Resolve which message IDs to download. The Package may carry a
+  // Persist a recovered part set before attempting the read: the ids are correct
-  // single destMessageId or multiple destMessageIds (for multipart).
+  // regardless of whether the listing turns out to be readable, and recording
-  const messageIds: bigint[] =
+  // them is what makes the package repairable on any later attempt (and lets the
-    pkg.destMessageIds.length > 0
+  // bot deliver every part rather than just the first).
-      ? pkg.destMessageIds
+  if (recoveredIds) {
-      : pkg.destMessageId
+    const written = await persistRecoveredDestIds(pkg.id, recoveredIds);
-      ? [pkg.destMessageId]
+    if (written) {
-      : [];
+      counters.idsRecovered++;
-
+      log.info({ ...ctx, destMessageIds: recoveredIds.map(Number) }, "Recovered destination message ids");
-  if (messageIds.length === 0) {
+    }
-    throw new Error("Package has no destination message IDs");
   }
 
+  const route = planListingRead({
+    archiveType: pkg.archiveType,
+    sourceFileName: pkg.fileName,
+    destFileNames: parts.map((p) => p.fileName),
+    totalSize: parts.reduce((sum, p) => sum + p.fileSize, 0n),
+    maxDownloadBytes: BigInt(config.maxZipSizeMB) * 1024n * 1024n,
+    rangedOnly: plan.rangedOnly,
+  });
+
+  if (route.route === "skip") {
+    counters.skipped++;
+    if (/concat/.test(route.reason)) counters.concatUnlistable++;
+    log.info(
+      { ...ctx, reason: route.reason, destFileNames: parts.map((p) => p.fileName) },
+      "Backfill skipped — destination copy cannot be listed"
+    );
+    return;
+  }
+
+  // ── Cheap path: read only the bytes that hold the listing ──
+  log.info(
+    { ...ctx, parts: parts.length, reason: route.reason },
+    "Backfill reading listing via RANGED read (no full download)"
+  );
+  let entries = await readScannedListingRanged(pkg.archiveType, client, parts);
+  let pathTaken: "ranged" | "download" = "ranged";
+
+  if (!entries || entries.length === 0) {
+    const totalSize = parts.reduce((sum, p) => sum + p.fileSize, 0n);
+    const fallback = planRangedFallback({
+      totalSize,
+      maxDownloadBytes: BigInt(config.maxZipSizeMB) * 1024n * 1024n,
+      rangedOnly: plan.rangedOnly,
+    });
+    if (fallback.route === "skip") {
+      counters.skipped++;
+      log.info({ ...ctx, reason: fallback.reason }, "Backfill skipped after ranged read returned nothing");
+      return;
+    }
+    log.warn(
+      { ...ctx, reason: fallback.reason, bytes: totalSize.toString() },
+      "Backfill falling back to FULL DOWNLOAD"
+    );
+    entries = await downloadAndRead(client, pkg, parts);
+    pathTaken = "download";
+  }
+
+  if (!entries || entries.length === 0) {
+    counters.skipped++;
+    log.warn({ ...ctx, pathTaken }, "Reader returned 0 entries — archive may be encrypted or corrupt");
+    return;
+  }
+
+  await writeListing(pkg, entries, ctx);
+  if (pathTaken === "ranged") counters.listedRanged++;
+  else counters.listedDownload++;
+  log.info({ ...ctx, fileCount: entries.length, pathTaken }, "Backfilled file list");
+}
+
+/**
+ * Write `destMessageIds` for a package that had none. Guarded on the array still
+ * being empty so a value written concurrently — by another worker, or by hand —
+ * is never clobbered. Returns whether a row was actually updated.
+ */
+async function persistRecoveredDestIds(packageId: string, ids: bigint[]): Promise<boolean> {
+  const result = await db.package.updateMany({
+    where: { id: packageId, destMessageIds: { isEmpty: true } },
+    data: { destMessageIds: ids },
+  });
+  return result.count > 0;
+}
+
+/** The original full-download path, kept as the fallback for un-ranged archives. */
+async function downloadAndRead(
+  client: Client,
+  pkg: BackfillPackage,
+  parts: RangedPart[]
+): Promise<FileEntry[]> {
   const tempDir = path.join(config.tempDir, `backfill_${pkg.id}`);
   await mkdir(tempDir, { recursive: true });
-
   try {
     const partPaths: string[] = [];
-
+    for (let i = 0; i < parts.length; i++) {
-    for (let i = 0; i < messageIds.length; i++) {
+      const part = parts[i];
-      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      const localPath = path.join(tempDir, part.fileName || `${pkg.id}.part${i + 1}`);
-      const message = (await client.invoke({
+      await downloadFile(client, part.fileId, localPath, part.fileSize, part.fileName);
-        _: "getMessage",
-        chat_id: chatId,
-        message_id: Number(messageIds[i]),
-      })) as unknown as {
-        content?: { document?: { file_name?: string; document?: { id: number; size: number } } };
-      };
-
-      const doc = message?.content?.document;
-      if (!doc?.document?.id) {
-        throw new Error(`Destination message ${messageIds[i]} has no document`);
-      }
-
-      const fileId = String(doc.document.id);
-      const fileName = doc.file_name ?? `${pkg.id}.part${i + 1}`;
-      const localPath = path.join(tempDir, fileName);
-
-      await downloadFile(
-        client,
-        fileId,
-        localPath,
-        BigInt(doc.document.size),
-        fileName
-      );
-
       partPaths.push(localPath);
     }
 
-    // Run the appropriate reader on the assembled file(s)
+    if (pkg.archiveType === "ZIP") return readZipCentralDirectory(partPaths);
-    let entries: FileEntry[] = [];
+    // unrar / 7z auto-discover sibling parts when in the same directory
-    if (pkg.archiveType === "ZIP") {
+    if (pkg.archiveType === "RAR") return readRarContents(partPaths[0]);
-      entries = await readZipCentralDirectory(partPaths);
+    if (pkg.archiveType === "SEVEN_Z") return read7zContents(partPaths[0]);
-    } else if (pkg.archiveType === "RAR") {
+    return [];
-      // unrar auto-discovers sibling parts when in the same directory
+  } finally {
-      entries = await readRarContents(partPaths[0]);
+    await rm(tempDir, { recursive: true, force: true }).catch(() => {});
-    } else if (pkg.archiveType === "SEVEN_Z") {
-      entries = await read7zContents(partPaths[0]);
-    } else {
-      log.debug({ ...ctx, archiveType: pkg.archiveType }, "Skipping unsupported archive type");
-      return;
-    }
-
-    if (entries.length === 0) {
-      log.warn(ctx, "Reader returned 0 entries — archive may be encrypted or corrupt");
-      return;
   }
+}
 
+async function writeListing(
+  pkg: BackfillPackage,
+  entries: FileEntry[],
+  ctx: { packageId: string; fileName: string }
+): Promise<void> {
   // Also derive slicer tags from the file list so the backfilled packages
   // gain the same search/filter context as newly-ingested ones.
   const slicerTags = extractSlicerTags(entries);
@@ -270,11 +506,6 @@ async function processOnePackage(
       data: { fileCount: entries.length, tags: mergedTags },
     });
   });
-
-    log.info({ ...ctx, fileCount: entries.length }, "Backfilled file list");
-  } finally {
-    await rm(tempDir, { recursive: true, force: true }).catch(() => {});
-  }
 }
 
 /**

worker/src/dest-index.test.ts

@@ -0,0 +1,111 @@
+import { describe, it, expect } from "vitest";
+import { buildDestIndex, resolveDestPartSet } from "./dest-index.js";
+import type { ChatDocument } from "./tdlib/chat-documents.js";
+
+let nextId = 100;
+function doc(fileName: string, opts: { id?: number; size?: number } = {}): ChatDocument {
+  const id = opts.id ?? nextId++;
+  return {
+    id: BigInt(id),
+    fileName,
+    fileId: `f${id}`,
+    fileSize: BigInt(opts.size ?? 1024),
+    date: new Date("2026-01-01T00:00:00Z"),
+  };
+}
+
+describe("buildDestIndex + resolveDestPartSet — spanned ZIP recovery", () => {
+  it("recovers the complete ordered volume set from the .z01 message alone", () => {
+    const z01 = doc("Pack.z01", { id: 10 });
+    const z02 = doc("Pack.z02", { id: 11 });
+    const zip = doc("Pack.zip", { id: 12 });
+    const index = buildDestIndex([z01, z02, zip, doc("Unrelated.zip", { id: 13 })]);
+
+    const resolved = resolveDestPartSet(index, 10n, 3);
+    expect(resolved.ok).toBe(true);
+    if (!resolved.ok) return;
+    // .z01, .z02, then the bare .zip as the FINAL volume — the order the
+    // EOCD-bearing tail read depends on.
+    expect(resolved.parts.map((p) => p.fileName)).toEqual(["Pack.z01", "Pack.z02", "Pack.zip"]);
+    expect(resolved.parts.map((p) => Number(p.id))).toEqual([10, 11, 12]);
+    expect(resolved.kind).toBe("archive-set");
+  });
+
+  it("carries fileId and size through, so no getMessage call is needed per part", () => {
+    const index = buildDestIndex([
+      doc("Pack.z01", { id: 10, size: 500 }),
+      doc("Pack.zip", { id: 11, size: 700 }),
+    ]);
+    const resolved = resolveDestPartSet(index, 10n, 2);
+    expect(resolved.ok).toBe(true);
+    if (!resolved.ok) return;
+    expect(resolved.parts.map((p) => p.fileId)).toEqual(["f10", "f11"]);
+    expect(resolved.parts.map((p) => Number(p.fileSize))).toEqual([500, 700]);
+  });
+});
+
+describe("resolveDestPartSet — refusing to guess", () => {
+  it("reports a missing anchor message instead of inventing a set", () => {
+    const index = buildDestIndex([doc("Pack.z01", { id: 10 }), doc("Pack.zip", { id: 11 })]);
+    const resolved = resolveDestPartSet(index, 999n, 2);
+    expect(resolved).toMatchObject({ ok: false });
+    if (resolved.ok) return;
+    expect(resolved.reason).toMatch(/was not found in the channel scan/);
+  });
+
+  it("refuses a set whose part count disagrees with the package", () => {
+    // Two uploads sharing a base name get merged by groupArchiveSets into one
+    // oversized set — writing that back would mix two archives together.
+    const index = buildDestIndex([
+      doc("Pack.z01", { id: 10 }),
+      doc("Pack.z02", { id: 11 }),
+      doc("Pack.zip", { id: 12 }),
+    ]);
+    const resolved = resolveDestPartSet(index, 10n, 2);
+    expect(resolved).toMatchObject({ ok: false });
+    if (resolved.ok) return;
+    expect(resolved.reason).toMatch(/refusing to write an incomplete or merged set/);
+  });
+
+  it("resolves a genuine single-part package from its one message", () => {
+    const index = buildDestIndex([doc("Solo.zip", { id: 20 })]);
+    const resolved = resolveDestPartSet(index, 20n, 1);
+    expect(resolved).toMatchObject({ ok: true });
+    if (!resolved.ok) return;
+    expect(resolved.parts.map((p) => p.fileName)).toEqual(["Solo.zip"]);
+  });
+});
+
+describe("buildDestIndex — .concat.NNN repacks", () => {
+  it("groups repack chunks that no archive pattern matches", () => {
+    // These names are invisible to archive/detect.ts, so without their own
+    // grouping a repacked package looks identical to one whose messages are gone.
+    const index = buildDestIndex([
+      doc("Pack.concat.003", { id: 32 }),
+      doc("Pack.concat.001", { id: 30 }),
+      doc("Pack.concat.002", { id: 31 }),
+    ]);
+    const resolved = resolveDestPartSet(index, 30n, 3);
+    expect(resolved.ok).toBe(true);
+    if (!resolved.ok) return;
+    expect(resolved.kind).toBe("concat-repack");
+    expect(resolved.parts.map((p) => p.fileName)).toEqual([
+      "Pack.concat.001",
+      "Pack.concat.002",
+      "Pack.concat.003",
+    ]);
+  });
+
+  it("keeps two different repacks apart", () => {
+    const index = buildDestIndex([
+      doc("A.concat.001", { id: 40 }),
+      doc("A.concat.002", { id: 41 }),
+      doc("B.concat.001", { id: 50 }),
+      doc("B.concat.002", { id: 51 }),
+    ]);
+    const a = resolveDestPartSet(index, 40n, 2);
+    expect(a).toMatchObject({ ok: true });
+    if (!a.ok) return;
+    expect(a.parts.map((p) => Number(p.id))).toEqual([40, 41]);
+  });
+});

worker/src/dest-index.ts

@@ -0,0 +1,116 @@
+import { isArchiveAttachment } from "./archive/detect.js";
+import { groupArchiveSets } from "./archive/multipart.js";
+import { isConcatRepackName, concatRepackBase, concatChunkIndex } from "./archive/listing-plan.js";
+import type { ChatDocument } from "./tdlib/chat-documents.js";
+
+/**
+ * An index over one destination-channel scan, used to recover the destination
+ * message ids of packages whose `destMessageIds` array was never populated.
+ *
+ * Two kinds of part set live in here:
+ *
+ *  - `archive-set`   — grouped by `groupArchiveSets`, i.e. the same grouping the
+ *                      ingestion path uses, so a `.z01 … .zip` spanned set or a
+ *                      `.zip.001 …` byte split comes back in upload order.
+ *  - `concat-repack` — `<base>.concat.NNN` chunks, which match no archive
+ *                      pattern at all and so are invisible to `groupArchiveSets`.
+ *                      They still need grouping: a package repacked this way has
+ *                      real destination messages worth recording even though its
+ *                      listing can never be read back.
+ */
+export type DestPartSetKind = "archive-set" | "concat-repack";
+
+export interface DestPartSet {
+  kind: DestPartSetKind;
+  parts: ChatDocument[];
+}
+
+export interface DestIndex {
+  byMessageId: Map<string, ChatDocument>;
+  setByMessageId: Map<string, DestPartSet>;
+  documentCount: number;
+}
+
+export function buildDestIndex(documents: ChatDocument[]): DestIndex {
+  const byMessageId = new Map<string, ChatDocument>();
+  for (const doc of documents) byMessageId.set(doc.id.toString(), doc);
+
+  const setByMessageId = new Map<string, DestPartSet>();
+
+  // Recognized archive names: reuse the ingestion grouping verbatim so the part
+  // order here matches the order the parts were uploaded in.
+  const archives = documents.filter((d) => isArchiveAttachment(d.fileName));
+  for (const set of groupArchiveSets(archives)) {
+    if (set.parts.length === 0) continue;
+    const entry: DestPartSet = { kind: "archive-set", parts: set.parts };
+    for (const part of set.parts) setByMessageId.set(part.id.toString(), entry);
+  }
+
+  // `<base>.concat.NNN` repack chunks, grouped by base and ordered by chunk number.
+  const concatGroups = new Map<string, ChatDocument[]>();
+  for (const doc of documents) {
+    if (!isConcatRepackName(doc.fileName)) continue;
+    const key = concatRepackBase(doc.fileName);
+    const group = concatGroups.get(key) ?? [];
+    group.push(doc);
+    concatGroups.set(key, group);
+  }
+  for (const group of concatGroups.values()) {
+    group.sort((a, b) => concatChunkIndex(a.fileName) - concatChunkIndex(b.fileName));
+    const entry: DestPartSet = { kind: "concat-repack", parts: group };
+    for (const part of group) setByMessageId.set(part.id.toString(), entry);
+  }
+
+  return { byMessageId, setByMessageId, documentCount: documents.length };
+}
+
+export type DestResolution =
+  | { ok: true; kind: DestPartSetKind; parts: ChatDocument[] }
+  | { ok: false; reason: string };
+
+/**
+ * Resolve a package's full destination part set from one known message id.
+ *
+ * Refuses anything it cannot corroborate. In particular the recovered set must
+ * hold exactly `expectedPartCount` parts: the destination channel can legitimately
+ * contain two uploads sharing a base name (a re-post, a duplicate ingestion), and
+ * `groupArchiveSets` merges those into one oversized set. Writing that merged set
+ * back to `destMessageIds` would hand the bot a mix of two archives, so a count
+ * mismatch is reported and the row is left alone.
+ */
+export function resolveDestPartSet(
+  index: DestIndex,
+  anchorMessageId: bigint,
+  expectedPartCount: number
+): DestResolution {
+  const key = anchorMessageId.toString();
+  const anchor = index.byMessageId.get(key);
+  if (!anchor) {
+    return {
+      ok: false,
+      reason: `destination message ${key} was not found in the channel scan (deleted, or outside the scanned range)`,
+    };
+  }
+
+  const set = index.setByMessageId.get(key);
+  if (!set) {
+    if (expectedPartCount === 1) {
+      return { ok: true, kind: "archive-set", parts: [anchor] };
+    }
+    return {
+      ok: false,
+      reason: `destination message ${key} ("${anchor.fileName}") matched no part set, but the package expects ${expectedPartCount} parts`,
+    };
+  }
+
+  if (set.parts.length !== expectedPartCount) {
+    return {
+      ok: false,
+      reason:
+        `resolved ${set.parts.length} destination part(s) for "${anchor.fileName}" but the package records ` +
+        `${expectedPartCount} — refusing to write an incomplete or merged set`,
+    };
+  }
+
+  return { ok: true, kind: set.kind, parts: set.parts };
+}

worker/src/fetch-listener.ts

@@ -538,12 +538,30 @@ function handleManualUpload(uploadId: string): void {
 
 // ── Backfill file-list handler ──
 //
-// Trigger via:
+// A request must say what it wants: a payload with no narrowing selector is
-//   SELECT pg_notify('backfill_filelists', '{"limit":50,"archiveType":"RAR"}');
+// rejected outright rather than sweeping every empty package in the catalogue.
+// See `backfill-scope.ts` for the full contract.
 //
-// Both fields are optional. archiveType filters to one of ZIP/RAR/SEVEN_Z.
+//   -- repair the ZIP-spec spanned sets (.z01 … .zip) that the pre-402c317
-// Default limit is 100. The handler queues so multiple notifications run
+//   -- reader bug left with no file list. Ranged reads only (~64KB each, vs
-// sequentially (no concurrent TDLib downloads competing for the mutex).
+//   -- ~944GB of full downloads), and one destination scan to recover the
+//   -- destMessageIds that were never recorded.
+//   SELECT pg_notify('backfill_filelists', '{
+//     "fileNameLike": "%.z01",
+//     "archiveType": "ZIP",
+//     "limit": 250,
+//     "rangedOnly": true,
+//     "recoverDestIds": true
+//   }');
+//
+//   -- repair a specific handful
+//   SELECT pg_notify('backfill_filelists', '{"packageIds":["ckxyz…","ckabc…"],"rangedOnly":true}');
+//
+//   -- the old broad sweep, now explicit about being one
+//   SELECT pg_notify('backfill_filelists', '{"archiveType":"RAR","limit":50,"allowBroadSweep":true}');
+//
+// The handler queues so multiple notifications run sequentially (no concurrent
+// TDLib downloads competing for the mutex).
 function handleBackfillFilelists(payload: string): void {
   fetchQueue = fetchQueue
     .then(() => processBackfillRequest(payload))

worker/src/provenance-backfill.ts

@@ -100,7 +100,7 @@ export async function resolveCandidateFingerprintEntries(
     if (destParts) {
       const read = tdlibRangeReader(client);
       destEntries =
-        candidate.archiveType === "ZIP" ? await readScannedZipListing(client, destParts)
+        candidate.archiveType === "ZIP" ? await readScannedZipListing(destParts, read)
         : candidate.archiveType === "SEVEN_Z" ? await readSevenZListingRanged(destParts, read)
         : candidate.archiveType === "RAR" ? await readRarListingRanged(destParts, read)
         : null;

worker/src/rebuild.ts

@@ -1,8 +1,7 @@
 import type { Client } from "tdl";
-import { config } from "./util/config.js";
 import { childLogger } from "./util/logger.js";
 import { createTdlibClient, closeTdlibClient } from "./tdlib/client.js";
-import { invokeWithTimeout, MAX_SCAN_PAGES } from "./tdlib/download.js";
+import { scanChatDocuments } from "./tdlib/chat-documents.js";
 import { isArchiveAttachment } from "./archive/detect.js";
 import { extractCreatorFromFileName } from "./archive/creator.js";
 import { groupArchiveSets } from "./archive/multipart.js";
@@ -263,119 +262,38 @@ export async function rebuildPackageDatabase(
 }
 
 /**
- * Scan the destination channel for document messages using searchChatMessages.
+ * Scan the destination channel and keep only the documents whose names
- * Returns archive messages in chronological order (oldest first).
+ * `archive/detect.ts` recognizes. The paging itself lives in
+ * `tdlib/chat-documents.ts` and is shared with the file-list repair path.
  */
 async function scanDestinationChannel(
   client: Client,
   chatId: bigint,
   onProgress?: (messagesScanned: number) => Promise<void>
 ): Promise<TelegramMessage[]> {
+  const scan = await scanChatDocuments(client, chatId, onProgress);
+
   const archives: TelegramMessage[] = [];
-  let currentFromId = 0;
+  for (const doc of scan.documents) {
-  let totalScanned = 0;
+    if (isArchiveAttachment(doc.fileName)) {
-  let pageCount = 0;
+      archives.push(doc);
-  let lastProgressUpdate = 0;
+    } else {
-
+      // Not matched by any pattern in archive/detect.ts, so it is dropped without a
-  // eslint-disable-next-line no-constant-condition
+      // packages/skipped_packages row. Grep "unrecognized attachment" to find naming
-  while (true) {
+      // schemes we do not handle yet.
-    if (pageCount >= MAX_SCAN_PAGES) {
+      log.debug(
-      log.warn(
+        { chatId: chatId.toString(), messageId: Number(doc.id), fileName: doc.fileName },
-        { chatId: chatId.toString(), pageCount, totalScanned },
+        "Skipping unrecognized attachment (no archive/document pattern matched)"
-        "Hit max page limit for destination scan, stopping"
       );
-      break;
     }
-    pageCount++;
-
-    const previousFromId = currentFromId;
-
-    const result = await invokeWithTimeout<{
-      messages?: {
-        id: number;
-        date: number;
-        content: {
-          _: string;
-          document?: {
-            file_name?: string;
-            document?: {
-              id: number;
-              size: number;
-            };
-          };
-        };
-      }[];
-    }>(client, {
-      _: "searchChatMessages",
-      chat_id: Number(chatId),
-      // No topic context for a flat destination scan. TDLib 1.8.64+ replaced
-      // `message_thread_id` / `saved_messages_topic_id` with a single
-      // optional `topic_id`; for a flat scan we just omit it.
-      query: "",
-      from_message_id: currentFromId,
-      offset: 0,
-      limit: 100,
-      filter: { _: "searchMessagesFilterDocument" },
-      sender_id: null,
-    });
-
-    if (!result.messages || result.messages.length === 0) break;
-
-    totalScanned += result.messages.length;
-
-    for (const msg of result.messages) {
-      const doc = msg.content?.document;
-      if (doc?.file_name && doc.document && isArchiveAttachment(doc.file_name)) {
-        archives.push({
-          id: BigInt(msg.id),
-          fileName: doc.file_name,
-          fileId: String(doc.document.id),
-          fileSize: BigInt(doc.document.size),
-          date: new Date(msg.date * 1000),
-        });
-      }
-    }
-
-    // Throttle progress updates to every 2 seconds
-    const now = Date.now();
-    if (onProgress && now - lastProgressUpdate >= 2000) {
-      lastProgressUpdate = now;
-      await onProgress(totalScanned);
-    }
-
-    currentFromId = result.messages[result.messages.length - 1].id;
-
-    // Stuck detection
-    if (currentFromId === previousFromId) {
-      log.warn(
-        { chatId: chatId.toString(), currentFromId, totalScanned },
-        "Pagination stuck, breaking"
-      );
-      break;
-    }
-
-    if (result.messages.length < 100) break;
-
-    await sleep(config.apiDelayMs);
-  }
-
-  // Final progress update
-  if (onProgress) {
-    await onProgress(totalScanned);
   }
 
   log.info(
-    {
+    { chatId: chatId.toString(), archives: archives.length, totalScanned: scan.totalScanned, pages: scan.pages },
-      chatId: chatId.toString(),
-      archives: archives.length,
-      totalScanned,
-      pages: pageCount,
-    },
     "Destination channel scan complete"
   );
 
-  // Reverse to chronological order (oldest first)
+  return archives;
-  return archives.reverse();
 }
 
 /**
@@ -406,7 +324,3 @@ async function updateRebuildProgress(
     }
   }
 }
-
-function sleep(ms: number): Promise<void> {
-  return new Promise((resolve) => setTimeout(resolve, ms));
-}

worker/src/tdlib/chat-documents.ts

@@ -0,0 +1,146 @@
+import type { Client } from "tdl";
+import { config } from "../util/config.js";
+import { childLogger } from "../util/logger.js";
+import { invokeWithTimeout, MAX_SCAN_PAGES } from "./download.js";
+
+const log = childLogger("chat-documents");
+
+/** One document message in a chat, with everything a ranged read needs. */
+export interface ChatDocument {
+  id: bigint;
+  fileName: string;
+  fileId: string;
+  fileSize: bigint;
+  date: Date;
+}
+
+export interface ChatDocumentScan {
+  /** Every document message with a file name, oldest first. */
+  documents: ChatDocument[];
+  /** Total messages returned by the search, including ones without a document. */
+  totalScanned: number;
+  pages: number;
+  /** True when the scan stopped on MAX_SCAN_PAGES rather than running out. */
+  truncated: boolean;
+}
+
+/**
+ * Page through every document message in a chat.
+ *
+ * `searchChatMessages` rather than `getChatHistory` because the destination
+ * channel may be a hidden-history supergroup, where history reads come back
+ * empty.
+ *
+ * Deliberately returns **all** documents and leaves filtering to the caller.
+ * The rebuild path wants only names `archive/detect.ts` recognizes; the repair
+ * path specifically needs the ones it does *not* — a `<base>.concat.NNN` chunk
+ * matches no archive pattern, and if the scan dropped those the repair could
+ * not tell "this package was repacked into an unlistable concatenation" apart
+ * from "its destination messages are gone".
+ */
+export async function scanChatDocuments(
+  client: Client,
+  chatId: bigint,
+  onProgress?: (messagesScanned: number) => Promise<void> | void
+): Promise<ChatDocumentScan> {
+  const documents: ChatDocument[] = [];
+  let currentFromId = 0;
+  let totalScanned = 0;
+  let pageCount = 0;
+  let lastProgressUpdate = 0;
+  let truncated = false;
+
+  for (;;) {
+    if (pageCount >= MAX_SCAN_PAGES) {
+      log.warn(
+        { chatId: chatId.toString(), pageCount, totalScanned },
+        "Hit max page limit for chat document scan, stopping"
+      );
+      truncated = true;
+      break;
+    }
+    pageCount++;
+
+    const previousFromId = currentFromId;
+
+    const result = await invokeWithTimeout<{
+      messages?: {
+        id: number;
+        date: number;
+        content: {
+          _: string;
+          document?: {
+            file_name?: string;
+            document?: { id: number; size: number };
+          };
+        };
+      }[];
+    }>(client, {
+      _: "searchChatMessages",
+      chat_id: Number(chatId),
+      // No topic context for a flat scan. TDLib 1.8.64+ replaced
+      // `message_thread_id` / `saved_messages_topic_id` with a single optional
+      // `topic_id`; for a flat scan we just omit it.
+      query: "",
+      from_message_id: currentFromId,
+      offset: 0,
+      limit: 100,
+      filter: { _: "searchMessagesFilterDocument" },
+      sender_id: null,
+    });
+
+    if (!result.messages || result.messages.length === 0) break;
+
+    totalScanned += result.messages.length;
+
+    for (const msg of result.messages) {
+      const doc = msg.content?.document;
+      if (doc?.file_name && doc.document) {
+        documents.push({
+          id: BigInt(msg.id),
+          fileName: doc.file_name,
+          fileId: String(doc.document.id),
+          fileSize: BigInt(doc.document.size),
+          date: new Date(msg.date * 1000),
+        });
+      }
+    }
+
+    // Throttle progress updates to every 2 seconds
+    const now = Date.now();
+    if (onProgress && now - lastProgressUpdate >= 2000) {
+      lastProgressUpdate = now;
+      await onProgress(totalScanned);
+    }
+
+    currentFromId = result.messages[result.messages.length - 1].id;
+
+    // Stuck detection
+    if (currentFromId === previousFromId) {
+      log.warn(
+        { chatId: chatId.toString(), currentFromId, totalScanned },
+        "Pagination stuck, breaking"
+      );
+      break;
+    }
+
+    if (result.messages.length < 100) break;
+
+    await sleep(config.apiDelayMs);
+  }
+
+  if (onProgress) await onProgress(totalScanned);
+
+  log.info(
+    { chatId: chatId.toString(), documents: documents.length, totalScanned, pages: pageCount, truncated },
+    "Chat document scan complete"
+  );
+
+  // Reverse to chronological order (oldest first)
+  documents.reverse();
+  return { documents, totalScanned, pages: pageCount, truncated };
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}

worker/src/tdlib/download.ts

@@ -267,6 +267,15 @@ export async function getChannelMessages(
           });
           continue;
         }
+        if (doc?.file_name) {
+          // Not matched by any pattern in archive/detect.ts, so it is dropped without a
+          // packages/skipped_packages row. Grep "unrecognized attachment" to find naming
+          // schemes we do not handle yet.
+          log.debug(
+            { chatId: chatId.toString(), messageId: msg.id, fileName: doc.file_name },
+            "Skipping unrecognized attachment (no archive/document pattern matched)"
+          );
+        }
 
         // Check for photo messages (potential previews)
         const photo = msg.content?.photo;

worker/src/tdlib/topics.ts

@@ -280,6 +280,15 @@ export async function getTopicMessages(
         });
         continue;
       }
+      if (doc?.file_name) {
+        // Not matched by any pattern in archive/detect.ts, so it is dropped without a
+        // packages/skipped_packages row. Grep "unrecognized attachment" to find naming
+        // schemes we do not handle yet.
+        log.debug(
+          { chatId: chatId.toString(), topicId: topicId.toString(), messageId: msg.id, fileName: doc.file_name },
+          "Skipping unrecognized attachment (no archive/document pattern matched)"
+        );
+      }
 
       // Check for photo messages (potential previews)
       const photo = msg.content?.photo;