admin/dragonsstash
1
0
Fork 0
mirror of https://github.com/xCyanGrizzly/DragonsStash.git synced 2026-09-21 13:31:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(worker): scoped, ranged-first file-list repair for spanned ZIP sets

Browse Source 
The 402c317 reader fix stops new spanned sets (.z01 … .zip) being indexed
with an empty file list, but leaves 194 pre-existing packages with
fileCount = 0. Nothing could repair them: the backfill re-downloaded full
archive bytes (~944GB for these), could only be scoped to "every empty
package of type X" (4,330 packages / 5.4TB for ZIP), and 154 of the 194
have an empty destMessageIds array — so it fell back to [destMessageId],
which is the first uploaded part. A lone .z01 has no central directory,
so those could never list no matter how much was downloaded.

Three changes address that:

Scoping (backfill-scope.ts). The payload now takes packageIds and/or a
restricted fileNameLike pattern, and a request with no narrowing selector
is rejected rather than defaulted into a full sweep — omitting a field can
only narrow the job or fail it. The unscoped sweep still exists but has to
ask for itself via allowBroadSweep. Unknown fields are an error too, so a
typo'd selector can't silently widen the scope.

Ranged-first reading (archive/listing-plan.ts). A file list lives in tens
of kilobytes of an archive's header or tail, so the repair reads it with
readScannedListingRanged and only falls back to downloadFile when ranged
reading genuinely cannot work — never, when rangedOnly is set. The route
taken is logged per package so the cost is visible rather than inferred.
Ranged reads go through downloadFileRange, which is already FLOOD_WAIT
aware, and the batch still runs under the account's TDLib mutex.

The planner also refuses the cases no reader can serve. When a source
volume exceeded the upload cap, worker.ts concatenated every volume and
re-split it into <base>.concat.NNN. For a byte split that round-trips
losslessly, but a concatenation of spanned ZIP or RAR volumes is not a
valid archive in any format — such a destination copy is permanently
unlistable, and it is skipped with that reason instead of spending API
calls failing.

destMessageIds recovery (dest-index.ts, tdlib/chat-documents.ts). The
destination-channel paging is lifted out of rebuild.ts and shared, so
there is one scanner rather than a third variant. It now returns every
document and leaves filtering to callers, because a .concat.NNN chunk
matches no archive pattern — with the old filter a repacked package was
indistinguishable from one whose messages had been deleted. One scan per
batch recovers the complete ordered part set for every candidate, and its
fileIds and sizes remove the per-part getMessage as a side effect. A
recovered set is persisted only when its part count matches the package:
the channel can hold two uploads sharing a base name, which groupArchiveSets
merges, and writing that back would hand the bot a mix of two archives. A
package whose volumes cannot be corroborated is left untouched and logged.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
admin
2026-08-23 00:01:11 +02:00
co-authored by Claude Sonnet 5
parent e1fb053fe0
commit b7ecf56745
10 changed files with 1474 additions and 287 deletions
Download Patch File Download Diff File Expand all files Collapse all files
worker/src/rebuild.ts
+18 -112
View File
@@ -1,8 +1,7 @@
import type { Client } from "tdl";
import { config } from "./util/config.js";
import { childLogger } from "./util/logger.js";
import { createTdlibClient, closeTdlibClient } from "./tdlib/client.js";
import { invokeWithTimeout, MAX_SCAN_PAGES } from "./tdlib/download.js";
import { scanChatDocuments } from "./tdlib/chat-documents.js";
import { isArchiveAttachment } from "./archive/detect.js";
import { extractCreatorFromFileName } from "./archive/creator.js";
import { groupArchiveSets } from "./archive/multipart.js";
@@ -263,127 +262,38 @@ export async function rebuildPackageDatabase(
}
/**
* Scan the destination channel for document messages using searchChatMessages.
* Returns archive messages in chronological order (oldest first).
* Scan the destination channel and keep only the documents whose names
* `archive/detect.ts` recognizes. The paging itself lives in
* `tdlib/chat-documents.ts` and is shared with the file-list repair path.
*/
async function scanDestinationChannel(
client: Client,
chatId: bigint,
onProgress?: (messagesScanned: number) => Promise<void>
): Promise<TelegramMessage[]> {
const scan = await scanChatDocuments(client, chatId, onProgress);
const archives: TelegramMessage[] = [];
let currentFromId = 0;
let totalScanned = 0;
let pageCount = 0;
let lastProgressUpdate = 0;
// eslint-disable-next-line no-constant-condition
while (true) {
if (pageCount >= MAX_SCAN_PAGES) {
log.warn(
{ chatId: chatId.toString(), pageCount, totalScanned },
"Hit max page limit for destination scan, stopping"
for (const doc of scan.documents) {
if (isArchiveAttachment(doc.fileName)) {
archives.push(doc);
} else {
// Not matched by any pattern in archive/detect.ts, so it is dropped without a
// packages/skipped_packages row. Grep "unrecognized attachment" to find naming
// schemes we do not handle yet.
log.debug(
{ chatId: chatId.toString(), messageId: Number(doc.id), fileName: doc.fileName },
"Skipping unrecognized attachment (no archive/document pattern matched)"
);
break;
}
pageCount++;
const previousFromId = currentFromId;
const result = await invokeWithTimeout<{
messages?: {
id: number;
date: number;
content: {
_: string;
document?: {
file_name?: string;
document?: {
id: number;
size: number;
};
};
};
}[];
}>(client, {
_: "searchChatMessages",
chat_id: Number(chatId),
// No topic context for a flat destination scan. TDLib 1.8.64+ replaced
// `message_thread_id` / `saved_messages_topic_id` with a single
// optional `topic_id`; for a flat scan we just omit it.
query: "",
from_message_id: currentFromId,
offset: 0,
limit: 100,
filter: { _: "searchMessagesFilterDocument" },
sender_id: null,
});
if (!result.messages || result.messages.length === 0) break;
totalScanned += result.messages.length;
for (const msg of result.messages) {
const doc = msg.content?.document;
if (doc?.file_name && doc.document && isArchiveAttachment(doc.file_name)) {
archives.push({
id: BigInt(msg.id),
fileName: doc.file_name,
fileId: String(doc.document.id),
fileSize: BigInt(doc.document.size),
date: new Date(msg.date * 1000),
});
} else if (doc?.file_name) {
// Not matched by any pattern in archive/detect.ts, so it is dropped without a
// packages/skipped_packages row. Grep "unrecognized attachment" to find naming
// schemes we do not handle yet.
log.debug(
{ chatId: chatId.toString(), messageId: msg.id, fileName: doc.file_name },
"Skipping unrecognized attachment (no archive/document pattern matched)"
);
}
}
// Throttle progress updates to every 2 seconds
const now = Date.now();
if (onProgress && now - lastProgressUpdate >= 2000) {
lastProgressUpdate = now;
await onProgress(totalScanned);
}
currentFromId = result.messages[result.messages.length - 1].id;
// Stuck detection
if (currentFromId === previousFromId) {
log.warn(
{ chatId: chatId.toString(), currentFromId, totalScanned },
"Pagination stuck, breaking"
);
break;
}
if (result.messages.length < 100) break;
await sleep(config.apiDelayMs);
}
// Final progress update
if (onProgress) {
await onProgress(totalScanned);
}
log.info(
{
chatId: chatId.toString(),
archives: archives.length,
totalScanned,
pages: pageCount,
},
{ chatId: chatId.toString(), archives: archives.length, totalScanned: scan.totalScanned, pages: scan.pages },
"Destination channel scan complete"
);
// Reverse to chronological order (oldest first)
return archives.reverse();
return archives;
}
/**
@@ -414,7 +324,3 @@ async function updateRebuildProgress(
}
}
}
function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}