feat(worker): scoped, ranged-first file-list repair for spanned ZIP sets

The 402c317 reader fix stops new spanned sets (.z01 … .zip) being indexed
with an empty file list, but leaves 194 pre-existing packages with
fileCount = 0. Nothing could repair them: the backfill re-downloaded full
archive bytes (~944GB for these), could only be scoped to "every empty
package of type X" (4,330 packages / 5.4TB for ZIP), and 154 of the 194
have an empty destMessageIds array — so it fell back to [destMessageId],
which is the first uploaded part. A lone .z01 has no central directory,
so those could never list no matter how much was downloaded.

Three changes address that:

Scoping (backfill-scope.ts). The payload now takes packageIds and/or a
restricted fileNameLike pattern, and a request with no narrowing selector
is rejected rather than defaulted into a full sweep — omitting a field can
only narrow the job or fail it. The unscoped sweep still exists but has to
ask for itself via allowBroadSweep. Unknown fields are an error too, so a
typo'd selector can't silently widen the scope.

Ranged-first reading (archive/listing-plan.ts). A file list lives in tens
of kilobytes of an archive's header or tail, so the repair reads it with
readScannedListingRanged and only falls back to downloadFile when ranged
reading genuinely cannot work — never, when rangedOnly is set. The route
taken is logged per package so the cost is visible rather than inferred.
Ranged reads go through downloadFileRange, which is already FLOOD_WAIT
aware, and the batch still runs under the account's TDLib mutex.

The planner also refuses the cases no reader can serve. When a source
volume exceeded the upload cap, worker.ts concatenated every volume and
re-split it into <base>.concat.NNN. For a byte split that round-trips
losslessly, but a concatenation of spanned ZIP or RAR volumes is not a
valid archive in any format — such a destination copy is permanently
unlistable, and it is skipped with that reason instead of spending API
calls failing.

destMessageIds recovery (dest-index.ts, tdlib/chat-documents.ts). The
destination-channel paging is lifted out of rebuild.ts and shared, so
there is one scanner rather than a third variant. It now returns every
document and leaves filtering to callers, because a .concat.NNN chunk
matches no archive pattern — with the old filter a repacked package was
indistinguishable from one whose messages had been deleted. One scan per
batch recovers the complete ordered part set for every candidate, and its
fileIds and sizes remove the per-part getMessage as a side effect. A
recovered set is persisted only when its part count matches the package:
the channel can hold two uploads sharing a base name, which groupArchiveSets
merges, and writing that back would hand the bot a mix of two archives. A
package whose volumes cannot be corroborated is left untouched and logged.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
2026-08-23 00:01:11 +02:00
co-authored by Claude Sonnet 5
parent e1fb053fe0
commit b7ecf56745
10 changed files with 1474 additions and 287 deletions
+130
View File
@@ -0,0 +1,130 @@
import { describe, it, expect } from "vitest";
import {
planListingRead,
planRangedFallback,
classifySourceShape,
isConcatRepackName,
concatRepackBase,
concatChunkIndex,
isVolumeSet,
} from "./listing-plan.js";
const GB = 1024n * 1024n * 1024n;
/** Defaults for a plan input; individual tests override what they care about. */
function input(over: Partial<Parameters<typeof planListingRead>[0]>) {
return {
archiveType: "ZIP",
sourceFileName: "Pack.z01",
destFileNames: ["Pack.z01", "Pack.zip"],
totalSize: 10n * GB,
maxDownloadBytes: 200n * GB,
rangedOnly: false,
...over,
};
}
describe("classifySourceShape", () => {
it("separates spanned ZIP volumes from a raw byte split", () => {
expect(classifySourceShape("Pack.z01")).toBe("spanned-zip");
expect(classifySourceShape("Pack.z12")).toBe("spanned-zip");
expect(classifySourceShape("Pack.zip.001")).toBe("byte-split");
expect(classifySourceShape("Pack.7z.001")).toBe("byte-split");
});
it("recognizes RAR volume sets and lone archives", () => {
expect(classifySourceShape("Pack.part1.rar")).toBe("rar-volume-set");
expect(classifySourceShape("Pack.r00")).toBe("rar-volume-set");
expect(classifySourceShape("Pack.zip")).toBe("single");
expect(classifySourceShape("notes.txt")).toBe("unknown");
});
it("marks exactly the layouts whose volumes are independent containers", () => {
expect(isVolumeSet("spanned-zip")).toBe(true);
expect(isVolumeSet("rar-volume-set")).toBe(true);
expect(isVolumeSet("byte-split")).toBe(false);
expect(isVolumeSet("single")).toBe(false);
});
});
describe("concat repack naming", () => {
it("recognizes the repack chunk names the uploader produces", () => {
expect(isConcatRepackName("Pack.concat.001")).toBe(true);
expect(isConcatRepackName("Pack.concat.017")).toBe(true);
expect(isConcatRepackName("Pack.concat")).toBe(true);
expect(isConcatRepackName("Pack.z01")).toBe(false);
expect(isConcatRepackName("Pack.zip.001")).toBe(false);
// "concat" appearing mid-name must not count
expect(isConcatRepackName("concat-models.zip")).toBe(false);
});
it("groups and orders chunks of one repack", () => {
expect(concatRepackBase("Pack.concat.002")).toBe("pack.concat");
expect(concatRepackBase("Pack.concat")).toBe("pack.concat");
expect(concatChunkIndex("Pack.concat.017")).toBe(17);
expect(concatChunkIndex("Pack.concat")).toBe(0);
});
});
describe("planListingRead", () => {
it("routes a spanned ZIP set to the ranged read", () => {
const plan = planListingRead(input({}));
expect(plan.route).toBe("ranged");
expect(plan.reason).toContain("spanned-zip");
});
it("skips a concatenated spanned ZIP set — no reader can ever list it", () => {
const plan = planListingRead(
input({ sourceFileName: "Pack.z01", destFileNames: ["Pack.concat.001", "Pack.concat.002"] })
);
expect(plan.route).toBe("skip");
expect(plan.reason).toContain("not a valid archive");
});
it("skips a concatenated RAR volume set for the same reason", () => {
const plan = planListingRead(
input({
archiveType: "RAR",
sourceFileName: "Pack.part1.rar",
destFileNames: ["Pack.concat.001", "Pack.concat.002"],
})
);
expect(plan.route).toBe("skip");
expect(plan.reason).toContain("rar-volume-set");
});
it("still reads a concatenated BYTE SPLIT — re-cutting one stream is lossless", () => {
const plan = planListingRead(
input({ sourceFileName: "Pack.zip.001", destFileNames: ["Pack.concat.001", "Pack.concat.002"] })
);
expect(plan.route).toBe("ranged");
expect(plan.reason).toContain("byte split");
});
it("skips archive types with no file-list reader without touching the API", () => {
expect(planListingRead(input({ archiveType: "DOCUMENT" }))).toMatchObject({ route: "skip" });
});
it("skips when no destination part could be resolved", () => {
expect(planListingRead(input({ destFileNames: [] }))).toMatchObject({ route: "skip" });
});
});
describe("planRangedFallback", () => {
it("refuses to download when rangedOnly is set, however large the archive", () => {
const plan = planRangedFallback({ totalSize: 35n * GB, maxDownloadBytes: 200n * GB, rangedOnly: true });
expect(plan.route).toBe("skip");
expect(plan.reason).toContain("rangedOnly");
});
it("refuses to download past the size cap", () => {
const plan = planRangedFallback({ totalSize: 300n * GB, maxDownloadBytes: 200n * GB, rangedOnly: false });
expect(plan.route).toBe("skip");
expect(plan.reason).toContain("exceeds the download cap");
});
it("falls back to a download when it is allowed and affordable", () => {
const plan = planRangedFallback({ totalSize: 2n * GB, maxDownloadBytes: 200n * GB, rangedOnly: false });
expect(plan.route).toBe("download");
});
});