mirror of
https://github.com/xCyanGrizzly/DragonsStash.git
synced 2026-09-21 05:21:43 +00:00
perf(worker): run startup upload-recovery batches with bounded concurrency
continuous-integration/drone/push Build is passing
continuous-integration/drone/push Build is passing
Telegram soft-throttles sustained sequential getMessages calls from a user account with growing per-call latency (no FLOOD_WAIT, so our retry wrapper never sees it). On a large package count this made the once-per-startup destination-message verification pass take 90+ minutes and block the scheduler/fetch-listener from starting. Running up to 3 batches concurrently cuts wall-clock time well under real per-account rate limits. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01JMm2E4ecmATJo8HuBx92NP
This commit is contained in:
+26
-5
@@ -84,15 +84,26 @@ export async function recoverIncompleteUploads(): Promise<void> {
|
||||
// Batch size for getMessages. TDLib accepts up to ~100 IDs per call.
|
||||
// Using 100 means 20k packages → ~200 round-trips instead of 20k.
|
||||
const BATCH_SIZE = 100;
|
||||
// Telegram soft-throttles sustained sequential history reads from user
|
||||
// accounts — no FLOOD_WAIT, just growing per-call latency the longer a
|
||||
// single in-flight request stream runs. A small number of batches in
|
||||
// flight at once cuts wall-clock time substantially without approaching
|
||||
// real per-account rate limits.
|
||||
const CONCURRENCY = 3;
|
||||
|
||||
const tdlibClient = client;
|
||||
const destChannelInfo = destChannel;
|
||||
const batches: (typeof packages)[] = [];
|
||||
for (const [, channelPackages] of byChannel) {
|
||||
// Group packages by destChannelId (already done) — within each group,
|
||||
// process in batches via getMessages (plural).
|
||||
for (let i = 0; i < channelPackages.length; i += BATCH_SIZE) {
|
||||
const batch = channelPackages.slice(i, i + BATCH_SIZE);
|
||||
batches.push(channelPackages.slice(i, i + BATCH_SIZE));
|
||||
}
|
||||
}
|
||||
|
||||
async function processBatch(batch: typeof packages) {
|
||||
const batchResults = await verifyMessagesBatch(
|
||||
client,
|
||||
destChannel.telegramId,
|
||||
tdlibClient,
|
||||
destChannelInfo.telegramId,
|
||||
batch.map((p) => p.destMessageId!)
|
||||
);
|
||||
|
||||
@@ -143,7 +154,17 @@ export async function recoverIncompleteUploads(): Promise<void> {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let nextBatchIndex = 0;
|
||||
async function poolWorker() {
|
||||
while (nextBatchIndex < batches.length) {
|
||||
const batch = batches[nextBatchIndex++];
|
||||
await processBatch(batch);
|
||||
}
|
||||
}
|
||||
await Promise.all(
|
||||
Array.from({ length: Math.min(CONCURRENCY, batches.length) }, () => poolWorker())
|
||||
);
|
||||
|
||||
log.info(
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user