diff --git a/worker/src/archive/ranged/fallback.test.ts b/worker/src/archive/ranged/fallback.test.ts new file mode 100644 index 0000000..ae96164 --- /dev/null +++ b/worker/src/archive/ranged/fallback.test.ts @@ -0,0 +1,27 @@ +import { describe, it, expect, vi } from "vitest"; + +// logLevel is required here too (not just maxZipSizeMB/tempDir) because this +// mock replaces the config module for the whole test-file graph, including +// util/logger.ts's module-level `pino({ level: config.logLevel })` call — +// pino throws at import time if level is undefined. +vi.mock("../../util/config.js", () => ({ config: { maxZipSizeMB: 1, tempDir: "/tmp", logLevel: "info" } })); +const created: unknown[] = []; +vi.mock("../../db/client.js", () => ({ + db: { systemNotification: { create: async (a: unknown) => { created.push(a); } } }, +})); + +import { fullDownloadListing } from "./fallback.js"; + +describe("fullDownloadListing", () => { + it("refuses to download over the size cap and records a notification", async () => { + const res = await fullDownloadListing({ + client: {} as never, + parts: [{ fileId: "1", fileSize: 2n * 1024n * 1024n * 1024n, fileName: "big.rar" }], + archiveType: "RAR", + totalSize: 2n * 1024n * 1024n * 1024n, + fileName: "big.rar", + }); + expect(res).toBeNull(); + expect(created).toHaveLength(1); + }); +}); diff --git a/worker/src/archive/ranged/fallback.ts b/worker/src/archive/ranged/fallback.ts new file mode 100644 index 0000000..7c714ef --- /dev/null +++ b/worker/src/archive/ranged/fallback.ts @@ -0,0 +1,55 @@ +import { mkdtemp, rm } from "fs/promises"; +import path from "path"; +import type { Client } from "tdl"; +import { config } from "../../util/config.js"; +import { db } from "../../db/client.js"; +import { childLogger } from "../../util/logger.js"; +import { downloadFile } from "../../tdlib/download.js"; +import { read7zContents } from "../sevenz-reader.js"; +import { readRarContents } from "../rar-reader.js"; +import type { FileEntry } from "../zip-reader.js"; +import type { RangedPart } from "./sevenz-ranged.js"; + +const log = childLogger("ranged-fallback"); + +export async function fullDownloadListing(args: { + client: Client; + parts: RangedPart[]; + archiveType: string; + totalSize: bigint; + fileName: string; +}): Promise { + const capBytes = BigInt(config.maxZipSizeMB) * 1024n * 1024n; + if (args.totalSize > capBytes) { + await db.systemNotification.create({ + data: { + type: "INTEGRITY_AUDIT", + severity: "WARNING", + title: `Listing skipped (over size cap): ${args.fileName}`, + message: `Ranged listing failed and the archive (${args.totalSize} bytes) exceeds WORKER_MAX_ZIP_SIZE_MB; not downloaded. Inner files left unindexed.`, + context: { fileName: args.fileName, archiveType: args.archiveType }, + }, + }); + log.warn({ fileName: args.fileName }, "fallback skipped — over size cap"); + return null; + } + const dir = await mkdtemp(path.join(config.tempDir, "fallback-")); + const paths: string[] = []; + try { + for (const p of args.parts) { + const dest = path.join(dir, p.fileName); + await downloadFile(args.client, p.fileId, dest, p.fileSize, p.fileName, () => {}); + paths.push(dest); + } + const entries = + args.archiveType === "SEVEN_Z" ? await read7zContents(paths[0]) + : args.archiveType === "RAR" ? await readRarContents(paths[0]) + : []; + return entries.length > 0 ? entries : null; + } catch (err) { + log.warn({ err, fileName: args.fileName }, "full-download fallback failed"); + return null; + } finally { + await rm(dir, { recursive: true, force: true }); + } +} diff --git a/worker/src/provenance-backfill.ts b/worker/src/provenance-backfill.ts index 6808974..5ada39a 100644 --- a/worker/src/provenance-backfill.ts +++ b/worker/src/provenance-backfill.ts @@ -11,6 +11,9 @@ import { type PlaceholderCandidate, } from "./db/queries.js"; import type { FileEntry } from "./archive/zip-reader.js"; +import { readSevenZListingRanged, type RangedPart } from "./archive/ranged/sevenz-ranged.js"; +import { tdlibRangeReader } from "./archive/ranged/range-reader.js"; +import { fullDownloadListing } from "./archive/ranged/fallback.js"; import type { Client } from "tdl"; const log = childLogger("provenance-backfill"); @@ -27,7 +30,7 @@ export interface BackfillArgs { sourceCaption: string | null; remoteUniqueId: string | null; creator: string | null; - scannedParts: { fileId: string; fileSize: bigint }[]; + scannedParts: RangedPart[]; previewData?: Buffer | null; previewMsgId?: bigint | null; } @@ -94,6 +97,18 @@ async function readZipListingFromDestination( } } +async function readScannedListingRanged( + archiveType: string, + client: Client, + parts: RangedPart[], +): Promise { + const read = tdlibRangeReader(client); + if (archiveType === "ZIP") return readScannedZipListing(client, parts); + if (archiveType === "SEVEN_Z") return readSevenZListingRanged(parts, read); + // RAR enabled in Task 8. + return null; +} + /** * Build the CRC fingerprint entries for a placeholder candidate: start from * its stored PackageFile CRCs, and if those are incomplete (e.g. a rebuild @@ -143,9 +158,17 @@ export async function tryProvenanceBackfill( const candidates = await findPlaceholderCandidates(args.destChannelId, args.fileName, args.fileSize); if (candidates.length === 0) return { backfilled: false }; - let scannedEntries: FileEntry[] | null = null; - if (args.archiveType === "ZIP") { - scannedEntries = await readScannedZipListing(args.client, args.scannedParts); + let scannedEntries: FileEntry[] | null = await readScannedListingRanged( + args.archiveType, args.client, args.scannedParts, + ); + // Cheap ranged read failed — fall back to a size-capped full download so the + // listing still gets indexed. Only worth it when the candidate lacks a listing. + if (!scannedEntries && candidates.some((c) => c.fileCount === 0)) { + const totalSize = args.scannedParts.reduce((s, p) => s + p.fileSize, 0n); + scannedEntries = await fullDownloadListing({ + client: args.client, parts: args.scannedParts, archiveType: args.archiveType, + totalSize, fileName: args.fileName, + }); } let chosen = candidates[0]; diff --git a/worker/src/worker.ts b/worker/src/worker.ts index 8a461b8..0017de0 100644 --- a/worker/src/worker.ts +++ b/worker/src/worker.ts @@ -1673,7 +1673,7 @@ async function processOneArchiveSet( sourceCaption: archiveSet.parts[0].caption ?? null, remoteUniqueId: archiveSet.parts[0].remoteUniqueId ?? null, creator: derivedCreator, - scannedParts: archiveSet.parts.map((p) => ({ fileId: p.fileId, fileSize: p.fileSize })), + scannedParts: archiveSet.parts.map((p) => ({ fileId: p.fileId, fileSize: p.fileSize, fileName: p.fileName })), previewData: null, previewMsgId: preview?.id ?? null, });