feat(worker): dispatch 7z ranged listing + size-capped full-download fallback

worker/src/archive/ranged/fallback.test.ts

@@ -0,0 +1,27 @@
+import { describe, it, expect, vi } from "vitest";
+
+// logLevel is required here too (not just maxZipSizeMB/tempDir) because this
+// mock replaces the config module for the whole test-file graph, including
+// util/logger.ts's module-level `pino({ level: config.logLevel })` call —
+// pino throws at import time if level is undefined.
+vi.mock("../../util/config.js", () => ({ config: { maxZipSizeMB: 1, tempDir: "/tmp", logLevel: "info" } }));
+const created: unknown[] = [];
+vi.mock("../../db/client.js", () => ({
+  db: { systemNotification: { create: async (a: unknown) => { created.push(a); } } },
+}));
+
+import { fullDownloadListing } from "./fallback.js";
+
+describe("fullDownloadListing", () => {
+  it("refuses to download over the size cap and records a notification", async () => {
+    const res = await fullDownloadListing({
+      client: {} as never,
+      parts: [{ fileId: "1", fileSize: 2n * 1024n * 1024n * 1024n, fileName: "big.rar" }],
+      archiveType: "RAR",
+      totalSize: 2n * 1024n * 1024n * 1024n,
+      fileName: "big.rar",
+    });
+    expect(res).toBeNull();
+    expect(created).toHaveLength(1);
+  });
+});

worker/src/archive/ranged/fallback.ts

@@ -0,0 +1,55 @@
+import { mkdtemp, rm } from "fs/promises";
+import path from "path";
+import type { Client } from "tdl";
+import { config } from "../../util/config.js";
+import { db } from "../../db/client.js";
+import { childLogger } from "../../util/logger.js";
+import { downloadFile } from "../../tdlib/download.js";
+import { read7zContents } from "../sevenz-reader.js";
+import { readRarContents } from "../rar-reader.js";
+import type { FileEntry } from "../zip-reader.js";
+import type { RangedPart } from "./sevenz-ranged.js";
+
+const log = childLogger("ranged-fallback");
+
+export async function fullDownloadListing(args: {
+  client: Client;
+  parts: RangedPart[];
+  archiveType: string;
+  totalSize: bigint;
+  fileName: string;
+}): Promise<FileEntry[] | null> {
+  const capBytes = BigInt(config.maxZipSizeMB) * 1024n * 1024n;
+  if (args.totalSize > capBytes) {
+    await db.systemNotification.create({
+      data: {
+        type: "INTEGRITY_AUDIT",
+        severity: "WARNING",
+        title: `Listing skipped (over size cap): ${args.fileName}`,
+        message: `Ranged listing failed and the archive (${args.totalSize} bytes) exceeds WORKER_MAX_ZIP_SIZE_MB; not downloaded. Inner files left unindexed.`,
+        context: { fileName: args.fileName, archiveType: args.archiveType },
+      },
+    });
+    log.warn({ fileName: args.fileName }, "fallback skipped — over size cap");
+    return null;
+  }
+  const dir = await mkdtemp(path.join(config.tempDir, "fallback-"));
+  const paths: string[] = [];
+  try {
+    for (const p of args.parts) {
+      const dest = path.join(dir, p.fileName);
+      await downloadFile(args.client, p.fileId, dest, p.fileSize, p.fileName, () => {});
+      paths.push(dest);
+    }
+    const entries =
+      args.archiveType === "SEVEN_Z" ? await read7zContents(paths[0])
+      : args.archiveType === "RAR" ? await readRarContents(paths[0])
+      : [];
+    return entries.length > 0 ? entries : null;
+  } catch (err) {
+    log.warn({ err, fileName: args.fileName }, "full-download fallback failed");
+    return null;
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+}

worker/src/provenance-backfill.ts

@@ -11,6 +11,9 @@ import {
   type PlaceholderCandidate,
 } from "./db/queries.js";
 import type { FileEntry } from "./archive/zip-reader.js";
+import { readSevenZListingRanged, type RangedPart } from "./archive/ranged/sevenz-ranged.js";
+import { tdlibRangeReader } from "./archive/ranged/range-reader.js";
+import { fullDownloadListing } from "./archive/ranged/fallback.js";
 import type { Client } from "tdl";
 
 const log = childLogger("provenance-backfill");
@@ -27,7 +30,7 @@ export interface BackfillArgs {
   sourceCaption: string | null;
   remoteUniqueId: string | null;
   creator: string | null;
-  scannedParts: { fileId: string; fileSize: bigint }[];
+  scannedParts: RangedPart[];
   previewData?: Buffer | null;
   previewMsgId?: bigint | null;
 }
@@ -94,6 +97,18 @@ async function readZipListingFromDestination(
   }
 }
 
+async function readScannedListingRanged(
+  archiveType: string,
+  client: Client,
+  parts: RangedPart[],
+): Promise<FileEntry[] | null> {
+  const read = tdlibRangeReader(client);
+  if (archiveType === "ZIP") return readScannedZipListing(client, parts);
+  if (archiveType === "SEVEN_Z") return readSevenZListingRanged(parts, read);
+  // RAR enabled in Task 8.
+  return null;
+}
+
 /**
  * Build the CRC fingerprint entries for a placeholder candidate: start from
  * its stored PackageFile CRCs, and if those are incomplete (e.g. a rebuild
@@ -143,9 +158,17 @@ export async function tryProvenanceBackfill(
   const candidates = await findPlaceholderCandidates(args.destChannelId, args.fileName, args.fileSize);
   if (candidates.length === 0) return { backfilled: false };
 
-  let scannedEntries: FileEntry[] | null = null;
-  if (args.archiveType === "ZIP") {
-    scannedEntries = await readScannedZipListing(args.client, args.scannedParts);
+  let scannedEntries: FileEntry[] | null = await readScannedListingRanged(
+    args.archiveType, args.client, args.scannedParts,
+  );
+  // Cheap ranged read failed — fall back to a size-capped full download so the
+  // listing still gets indexed. Only worth it when the candidate lacks a listing.
+  if (!scannedEntries && candidates.some((c) => c.fileCount === 0)) {
+    const totalSize = args.scannedParts.reduce((s, p) => s + p.fileSize, 0n);
+    scannedEntries = await fullDownloadListing({
+      client: args.client, parts: args.scannedParts, archiveType: args.archiveType,
+      totalSize, fileName: args.fileName,
+    });
   }
 
   let chosen = candidates[0];

worker/src/worker.ts

@@ -1673,7 +1673,7 @@ async function processOneArchiveSet(
         sourceCaption: archiveSet.parts[0].caption ?? null,
         remoteUniqueId: archiveSet.parts[0].remoteUniqueId ?? null,
         creator: derivedCreator,
-        scannedParts: archiveSet.parts.map((p) => ({ fileId: p.fileId, fileSize: p.fileSize })),
+        scannedParts: archiveSet.parts.map((p) => ({ fileId: p.fileId, fileSize: p.fileSize, fileName: p.fileName })),
         previewData: null,
         previewMsgId: preview?.id ?? null,
       });