mirror of
https://github.com/xCyanGrizzly/DragonsStash.git
synced 2026-07-22 23:12:02 +00:00
Compare commits
7 Commits
59038889ae
...
77aeb4cc00
| Author | SHA1 | Date | |
|---|---|---|---|
| 77aeb4cc00 | |||
| 3b327eb3f3 | |||
| 379bf246cd | |||
| 7a79b52baf | |||
| 26e2cba69d | |||
| 84cc8d995b | |||
| d99a506b10 |
@@ -0,0 +1,8 @@
|
||||
-- AlterTable: track how many times the worker has tried each skipped source message.
|
||||
-- Existing rows default to 1 (they represent a single past attempt that the worker
|
||||
-- chose to record). Future failures increment via upsertSkippedPackage.
|
||||
ALTER TABLE "skipped_packages" ADD COLUMN "attemptCount" INTEGER NOT NULL DEFAULT 1;
|
||||
|
||||
-- AlterEnum: add CHANNEL_ACCESS_LOST so the worker can surface a notification
|
||||
-- when a source channel becomes inaccessible (account removed, channel deleted, etc.)
|
||||
ALTER TYPE "NotificationType" ADD VALUE 'CHANNEL_ACCESS_LOST';
|
||||
@@ -741,6 +741,12 @@ model SkippedPackage {
|
||||
sourceTopicId BigInt?
|
||||
isMultipart Boolean @default(false)
|
||||
partCount Int @default(1)
|
||||
/// How many times the worker has tried to process this source message.
|
||||
/// The worker auto-retries failures across cycles up to a configurable cap
|
||||
/// (WORKER_MAX_SKIP_ATTEMPTS, default 5). After the cap, the watermark is
|
||||
/// allowed to advance past the failure so cycles aren't pinned forever;
|
||||
/// the user can manually retry via the UI to reset and try again.
|
||||
attemptCount Int @default(1)
|
||||
accountId String
|
||||
account TelegramAccount @relation(fields: [accountId], references: [id], onDelete: Cascade)
|
||||
createdAt DateTime @default(now())
|
||||
@@ -830,6 +836,7 @@ enum NotificationType {
|
||||
DOWNLOAD_FAILED
|
||||
GROUPING_CONFLICT
|
||||
INTEGRITY_AUDIT
|
||||
CHANNEL_ACCESS_LOST
|
||||
}
|
||||
|
||||
enum NotificationSeverity {
|
||||
|
||||
@@ -20,6 +20,7 @@ export interface SkippedRow {
|
||||
sourceChannel: { id: string; title: string };
|
||||
isMultipart: boolean;
|
||||
partCount: number;
|
||||
attemptCount: number;
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
@@ -107,9 +108,22 @@ export function getSkippedColumns({
|
||||
),
|
||||
accessorFn: (row) => row.sourceChannel.title,
|
||||
},
|
||||
{
|
||||
accessorKey: "attemptCount",
|
||||
header: ({ column }) => <DataTableColumnHeader column={column} title="Attempts" />,
|
||||
cell: ({ row }) => {
|
||||
const count = row.original.attemptCount;
|
||||
const variant = count >= 5 ? "destructive" : count > 1 ? "secondary" : "outline";
|
||||
return (
|
||||
<Badge variant={variant} className="text-[10px]">
|
||||
{count}
|
||||
</Badge>
|
||||
);
|
||||
},
|
||||
},
|
||||
{
|
||||
accessorKey: "createdAt",
|
||||
header: ({ column }) => <DataTableColumnHeader column={column} title="Skipped" />,
|
||||
header: ({ column }) => <DataTableColumnHeader column={column} title="Last Skipped" />,
|
||||
cell: ({ row }) => (
|
||||
<span className="text-sm text-muted-foreground">
|
||||
{new Date(row.original.createdAt).toLocaleDateString()}
|
||||
|
||||
@@ -589,6 +589,7 @@ export async function listSkippedPackages(options: {
|
||||
sourceMessageId: s.sourceMessageId.toString(),
|
||||
isMultipart: s.isMultipart,
|
||||
partCount: s.partCount,
|
||||
attemptCount: s.attemptCount,
|
||||
createdAt: s.createdAt.toISOString(),
|
||||
}));
|
||||
|
||||
|
||||
@@ -55,6 +55,8 @@ export interface SkippedPackageItem {
|
||||
sourceMessageId: string;
|
||||
isMultipart: boolean;
|
||||
partCount: number;
|
||||
/** How many times the worker has tried this source message across cycles. */
|
||||
attemptCount: number;
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
|
||||
@@ -625,6 +625,7 @@ export async function upsertSkippedPackage(data: {
|
||||
errorMessage: data.errorMessage ?? null,
|
||||
fileName: data.fileName,
|
||||
fileSize: data.fileSize,
|
||||
attemptCount: { increment: 1 },
|
||||
createdAt: new Date(),
|
||||
},
|
||||
create: {
|
||||
@@ -642,6 +643,26 @@ export async function upsertSkippedPackage(data: {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Return source-message IDs in a channel whose SkippedPackage attemptCount has
|
||||
* reached or exceeded the cap — these are treated as "permanently failed for
|
||||
* now" so the watermark can advance past them. The user can manually retry via
|
||||
* the UI to reset the SkippedPackage record.
|
||||
*/
|
||||
export async function getCappedSkippedMessageIds(
|
||||
sourceChannelId: string,
|
||||
cap: number
|
||||
): Promise<Set<bigint>> {
|
||||
const rows = await db.skippedPackage.findMany({
|
||||
where: {
|
||||
sourceChannelId,
|
||||
attemptCount: { gte: cap },
|
||||
},
|
||||
select: { sourceMessageId: true },
|
||||
});
|
||||
return new Set(rows.map((r) => r.sourceMessageId));
|
||||
}
|
||||
|
||||
export async function deleteSkippedPackage(
|
||||
sourceChannelId: string,
|
||||
sourceMessageId: bigint
|
||||
|
||||
@@ -119,25 +119,45 @@ async function sendWithRetry(
|
||||
continue;
|
||||
}
|
||||
|
||||
// Stall or timeout — retry with a cooldown
|
||||
// Stall or timeout — fail fast and let the caller recreate the TDLib
|
||||
// client. Retrying on the same degraded event stream wastes ~15 min
|
||||
// per attempt because the underlying issue (missing send-success
|
||||
// events) is client-level, not transient. The set ends up in
|
||||
// SkippedPackage and the caller's watermark cap ensures it gets
|
||||
// retried next cycle on a fresh client.
|
||||
const errMsg = err instanceof Error ? err.message : "";
|
||||
if (errMsg.includes("stalled") || errMsg.includes("timed out")) {
|
||||
if (!isLastAttempt) {
|
||||
log.warn(
|
||||
{ fileName, attempt: attempt + 1, maxRetries: MAX_UPLOAD_RETRIES },
|
||||
"Upload stalled/timed out — retrying"
|
||||
{ fileName, attempt: attempt + 1 },
|
||||
"Upload stalled — failing fast so caller can recreate TDLib client"
|
||||
);
|
||||
await sleep(10_000);
|
||||
continue;
|
||||
}
|
||||
// All stall retries exhausted — throw UploadStallError so the caller
|
||||
// knows the TDLib client's event stream is likely degraded and can
|
||||
// recreate the client before continuing.
|
||||
throw new UploadStallError(
|
||||
`Upload stalled after ${MAX_UPLOAD_RETRIES} retries for ${fileName}`
|
||||
`Upload stalled for ${fileName}: ${errMsg}`
|
||||
);
|
||||
}
|
||||
|
||||
// Transient Telegram server-side error (HTTP 5xx returned via
|
||||
// updateMessageSendFailed). These are NOT FLOOD_WAIT, NOT stalls — just
|
||||
// TG having a bad moment. They typically resolve on a short backoff, so
|
||||
// retry up to MAX_UPLOAD_RETRIES with linear backoff before giving up.
|
||||
const lowerMsg = errMsg.toLowerCase();
|
||||
const isTransientServerError =
|
||||
lowerMsg.includes("internal server error") ||
|
||||
lowerMsg.includes("internal error") ||
|
||||
lowerMsg.includes("server error") ||
|
||||
lowerMsg.includes("bad gateway") ||
|
||||
lowerMsg.includes("service unavailable") ||
|
||||
lowerMsg.includes("gateway timeout");
|
||||
if (isTransientServerError && !isLastAttempt) {
|
||||
const backoffMs = 15_000 * (attempt + 1) + Math.random() * 5_000;
|
||||
log.warn(
|
||||
{ fileName, attempt: attempt + 1, maxRetries: MAX_UPLOAD_RETRIES, backoffMs: Math.round(backoffMs) },
|
||||
`Transient Telegram server error — retrying after backoff`
|
||||
);
|
||||
await sleep(backoffMs);
|
||||
continue;
|
||||
}
|
||||
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
@@ -168,6 +188,12 @@ async function sendAndWaitForUpload(
|
||||
let lastProgressBytes = 0;
|
||||
let lastProgressTime = Date.now();
|
||||
|
||||
// Events for our message can arrive before `sendMessage` resolves
|
||||
// (TDLib emits them while our .then() is still in the microtask queue).
|
||||
// Buffer them and replay once tempMsgId is known.
|
||||
let pendingSuccess: { oldMsgId: number; finalId: number } | null = null;
|
||||
let pendingFailure: { oldMsgId: number; errorMsg: string; code?: number } | null = null;
|
||||
|
||||
// Timeout: 20 minutes per GB, minimum 15 minutes
|
||||
const timeoutMs = Math.max(
|
||||
15 * 60_000,
|
||||
@@ -204,6 +230,26 @@ async function sendAndWaitForUpload(
|
||||
}
|
||||
}, 30_000);
|
||||
|
||||
const completeWithSuccess = (finalId: number) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
cleanup();
|
||||
log.info(
|
||||
{ fileName, tempMsgId, finalMsgId: finalId },
|
||||
"Upload confirmed by Telegram"
|
||||
);
|
||||
resolve(BigInt(finalId));
|
||||
};
|
||||
|
||||
const completeWithFailure = (errorMsg: string, code?: number) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
cleanup();
|
||||
const error = new Error(`Upload failed for ${fileName}: ${errorMsg}`);
|
||||
(error as Error & { code?: number }).code = code;
|
||||
reject(error);
|
||||
};
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const handleUpdate = (update: any) => {
|
||||
// Track upload progress via updateFile events
|
||||
@@ -234,33 +280,29 @@ async function sendAndWaitForUpload(
|
||||
// The money event: upload succeeded, we get the final server message ID
|
||||
if (update?._ === "updateMessageSendSucceeded") {
|
||||
const msg = update.message;
|
||||
const oldMsgId = update.old_message_id;
|
||||
if (tempMsgId !== null && oldMsgId === tempMsgId) {
|
||||
if (!settled) {
|
||||
settled = true;
|
||||
cleanup();
|
||||
const finalId = BigInt(msg.id);
|
||||
log.info(
|
||||
{ fileName, tempMsgId, finalMsgId: Number(finalId) },
|
||||
"Upload confirmed by Telegram"
|
||||
);
|
||||
resolve(finalId);
|
||||
const oldMsgId: number = update.old_message_id;
|
||||
if (tempMsgId === null) {
|
||||
// Race: event arrived before our .then() assigned tempMsgId.
|
||||
// Buffer it and process once tempMsgId is known.
|
||||
pendingSuccess = { oldMsgId, finalId: msg.id };
|
||||
return;
|
||||
}
|
||||
if (oldMsgId === tempMsgId) {
|
||||
completeWithSuccess(msg.id);
|
||||
}
|
||||
}
|
||||
|
||||
// Upload failed
|
||||
if (update?._ === "updateMessageSendFailed") {
|
||||
const oldMsgId = update.old_message_id;
|
||||
if (tempMsgId !== null && oldMsgId === tempMsgId) {
|
||||
if (!settled) {
|
||||
settled = true;
|
||||
cleanup();
|
||||
const errorMsg = update.error?.message ?? "Unknown upload error";
|
||||
const error = new Error(`Upload failed for ${fileName}: ${errorMsg}`);
|
||||
(error as Error & { code?: number }).code = update.error?.code;
|
||||
reject(error);
|
||||
const oldMsgId: number = update.old_message_id;
|
||||
const errorMsg: string = update.error?.message ?? "Unknown upload error";
|
||||
const code: number | undefined = update.error?.code;
|
||||
if (tempMsgId === null) {
|
||||
pendingFailure = { oldMsgId, errorMsg, code };
|
||||
return;
|
||||
}
|
||||
if (oldMsgId === tempMsgId) {
|
||||
completeWithFailure(errorMsg, code);
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -304,6 +346,13 @@ async function sendAndWaitForUpload(
|
||||
{ fileName, tempMsgId },
|
||||
"Message queued, waiting for upload confirmation"
|
||||
);
|
||||
|
||||
// Replay any event that arrived before we knew tempMsgId
|
||||
if (pendingSuccess && pendingSuccess.oldMsgId === tempMsgId) {
|
||||
completeWithSuccess(pendingSuccess.finalId);
|
||||
} else if (pendingFailure && pendingFailure.oldMsgId === tempMsgId) {
|
||||
completeWithFailure(pendingFailure.errorMsg, pendingFailure.code);
|
||||
}
|
||||
})
|
||||
.catch((err) => {
|
||||
if (!settled) {
|
||||
|
||||
@@ -20,4 +20,8 @@ export const config = {
|
||||
apiDelayMs: 1000,
|
||||
/** Max retries for rate-limited requests */
|
||||
maxRetries: 5,
|
||||
/** After this many failed attempts on the same source message, the worker
|
||||
* stops auto-retrying and lets the watermark advance past it. The user can
|
||||
* manually retry via the UI to reset and try again. */
|
||||
maxSkipAttempts: parseInt(process.env.WORKER_MAX_SKIP_ATTEMPTS ?? "5", 10),
|
||||
} as const;
|
||||
|
||||
@@ -30,6 +30,7 @@ import {
|
||||
getUploadedPackageByHash,
|
||||
upsertSkippedPackage,
|
||||
deleteSkippedPackage,
|
||||
getCappedSkippedMessageIds,
|
||||
} from "./db/queries.js";
|
||||
import type { ActivityUpdate } from "./db/queries.js";
|
||||
import { createTdlibClient, closeTdlibClient } from "./tdlib/client.js";
|
||||
@@ -408,6 +409,47 @@ export async function runWorkerForAccount(
|
||||
throw new Error("No global destination channel configured — set one in the admin UI");
|
||||
}
|
||||
|
||||
// ── Ensure TDLib knows about the destination chat ──
|
||||
// Source channels get an explicit getChat below, but the destination was
|
||||
// previously only loaded via loadChats — which can miss it if the account
|
||||
// archived/moved it. Failing here surfaces the problem clearly instead of
|
||||
// letting every upload fail with a cryptic "Chat not found".
|
||||
try {
|
||||
await client.invoke({
|
||||
_: "getChat",
|
||||
chat_id: Number(destChannel.telegramId),
|
||||
});
|
||||
} catch (destErr) {
|
||||
accountLog.error(
|
||||
{ err: destErr, destChannel: destChannel.title, telegramId: destChannel.telegramId.toString() },
|
||||
"Destination chat is not accessible to this account — uploads will fail. Re-join via invite link or remove this account."
|
||||
);
|
||||
// Surface as a persistent notification so the admin sees it in the UI
|
||||
try {
|
||||
await db.systemNotification.create({
|
||||
data: {
|
||||
type: "UPLOAD_FAILED",
|
||||
severity: "ERROR",
|
||||
title: `Destination chat unreachable for ${account.phone}`,
|
||||
message: `Account ${account.phone} cannot access the destination chat "${destChannel.title}". Uploads for this account will fail until access is restored. Re-join via the invite link in admin settings.`,
|
||||
context: {
|
||||
accountId: account.id,
|
||||
accountPhone: account.phone,
|
||||
destChannelId: destChannel.id,
|
||||
destChannelTitle: destChannel.title,
|
||||
},
|
||||
},
|
||||
});
|
||||
} catch {
|
||||
// Best-effort notification
|
||||
}
|
||||
// Skip this account's ingestion cycle entirely — there's no point
|
||||
// scanning + downloading if we can't upload.
|
||||
throw new Error(
|
||||
`Destination chat "${destChannel.title}" is not accessible to account ${account.phone}`
|
||||
);
|
||||
}
|
||||
|
||||
const totalChannels = channelMappings.length;
|
||||
|
||||
if (totalChannels === 0) {
|
||||
@@ -560,13 +602,33 @@ export async function runWorkerForAccount(
|
||||
pipelineCtx.sourceTopicId = topic.topicId;
|
||||
pipelineCtx.channelTitle = `${channel.title} › ${topic.name}`;
|
||||
|
||||
const maxProcessedId = await processArchiveSets(pipelineCtx, scanResult, run.id, progress?.lastProcessedMessageId);
|
||||
const { maxProcessedId, minFailedId } = await processArchiveSets(
|
||||
pipelineCtx,
|
||||
scanResult,
|
||||
run.id,
|
||||
progress?.lastProcessedMessageId,
|
||||
// Incremental watermark advance — saves progress per-set so a
|
||||
// worker restart mid-scan doesn't lose all work.
|
||||
async (messageId) => {
|
||||
await upsertTopicProgress(
|
||||
mapping.id,
|
||||
topic.topicId,
|
||||
topic.name,
|
||||
messageId
|
||||
);
|
||||
}
|
||||
);
|
||||
// Sync client back in case it was recreated during upload stall recovery
|
||||
client = pipelineCtx.client;
|
||||
|
||||
// Advance progress: use archive watermark if available, fall back to scan watermark
|
||||
const topicWatermark = maxProcessedId ?? scanResult.maxScannedMessageId;
|
||||
if (topicWatermark) {
|
||||
// Final watermark write at the end of the scan (covers the
|
||||
// no-archives-found and all-failures-with-fallback cases).
|
||||
// The incremental updates above already handle the success path.
|
||||
let topicWatermark = maxProcessedId ?? scanResult.maxScannedMessageId;
|
||||
if (minFailedId !== null && topicWatermark !== null && topicWatermark >= minFailedId) {
|
||||
topicWatermark = minFailedId - 1n;
|
||||
}
|
||||
if (topicWatermark !== null) {
|
||||
await upsertTopicProgress(
|
||||
mapping.id,
|
||||
topic.topicId,
|
||||
@@ -639,13 +701,28 @@ export async function runWorkerForAccount(
|
||||
pipelineCtx.sourceTopicId = null;
|
||||
pipelineCtx.channelTitle = channel.title;
|
||||
|
||||
const maxProcessedId = await processArchiveSets(pipelineCtx, scanResult, run.id, mapping.lastProcessedMessageId);
|
||||
const { maxProcessedId, minFailedId } = await processArchiveSets(
|
||||
pipelineCtx,
|
||||
scanResult,
|
||||
run.id,
|
||||
mapping.lastProcessedMessageId,
|
||||
// Incremental watermark advance — saves progress per-set so a
|
||||
// worker restart mid-scan doesn't lose all work.
|
||||
async (messageId) => {
|
||||
await updateLastProcessedMessage(mapping.id, messageId);
|
||||
}
|
||||
);
|
||||
// Sync client back in case it was recreated during upload stall recovery
|
||||
client = pipelineCtx.client;
|
||||
|
||||
// Advance progress: use archive watermark if available, fall back to scan watermark
|
||||
const channelWatermark = maxProcessedId ?? scanResult.maxScannedMessageId;
|
||||
if (channelWatermark) {
|
||||
// Final watermark write at the end of the scan (covers the
|
||||
// no-archives-found and all-failures-with-fallback cases).
|
||||
// The incremental updates above already handle the success path.
|
||||
let channelWatermark = maxProcessedId ?? scanResult.maxScannedMessageId;
|
||||
if (minFailedId !== null && channelWatermark !== null && channelWatermark >= minFailedId) {
|
||||
channelWatermark = minFailedId - 1n;
|
||||
}
|
||||
if (channelWatermark !== null) {
|
||||
await updateLastProcessedMessage(mapping.id, channelWatermark);
|
||||
}
|
||||
}
|
||||
@@ -654,6 +731,50 @@ export async function runWorkerForAccount(
|
||||
{ err: channelErr, channelId: channel.id, title: channel.title },
|
||||
"Failed to process channel, skipping to next"
|
||||
);
|
||||
|
||||
// If the channel is no longer accessible (account got removed,
|
||||
// channel deleted, etc.), surface a persistent notification so the
|
||||
// admin can decide what to do. Dedupe by (channelId, accountId)
|
||||
// within the last 24h so we don't flood the notifications list every
|
||||
// cycle.
|
||||
const errMsg = channelErr instanceof Error ? channelErr.message : String(channelErr);
|
||||
const isAccessError =
|
||||
errMsg.includes("Can't access the chat") ||
|
||||
errMsg.includes("CHAT_FORBIDDEN") ||
|
||||
errMsg.includes("CHANNEL_PRIVATE") ||
|
||||
errMsg.includes("Chat not found");
|
||||
if (isAccessError) {
|
||||
try {
|
||||
const recent = await db.systemNotification.findFirst({
|
||||
where: {
|
||||
type: "CHANNEL_ACCESS_LOST",
|
||||
context: { path: ["channelId"], equals: channel.id },
|
||||
createdAt: { gte: new Date(Date.now() - 24 * 60 * 60 * 1000) },
|
||||
},
|
||||
select: { id: true },
|
||||
});
|
||||
if (!recent) {
|
||||
await db.systemNotification.create({
|
||||
data: {
|
||||
type: "CHANNEL_ACCESS_LOST",
|
||||
severity: "WARNING",
|
||||
title: `Lost access to "${channel.title}" for ${account.phone}`,
|
||||
message: `Account ${account.phone} can no longer access source channel "${channel.title}". The worker is skipping this channel every cycle. Re-join the channel or unlink it from this account in admin settings.`,
|
||||
context: {
|
||||
channelId: channel.id,
|
||||
channelTitle: channel.title,
|
||||
telegramId: channel.telegramId.toString(),
|
||||
accountId: account.id,
|
||||
accountPhone: account.phone,
|
||||
errorMessage: errMsg.slice(0, 200),
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
} catch {
|
||||
// Best-effort notification
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -704,8 +825,13 @@ async function processArchiveSets(
|
||||
ctx: PipelineContext,
|
||||
scanResult: ChannelScanResult,
|
||||
ingestionRunId: string,
|
||||
lastProcessedMessageId?: bigint | null
|
||||
): Promise<bigint | null> {
|
||||
lastProcessedMessageId?: bigint | null,
|
||||
/** Called after each successful set with a safe watermark value (capped
|
||||
* below any failed message ID in this scan). Used by the caller to
|
||||
* advance the channel/topic watermark incrementally — otherwise a long
|
||||
* scan that gets killed by worker restart loses all progress. */
|
||||
onWatermarkAdvance?: (messageId: bigint) => Promise<void>
|
||||
): Promise<{ maxProcessedId: bigint | null; minFailedId: bigint | null }> {
|
||||
const { client, runId, channelTitle, channel, throttled, counters, accountLog } = ctx;
|
||||
|
||||
// Group into archive sets
|
||||
@@ -726,6 +852,26 @@ async function processArchiveSets(
|
||||
}
|
||||
}
|
||||
|
||||
// Filter out sets whose source message has hit the auto-retry cap — these are
|
||||
// treated as "give up for now" so the watermark can advance past them.
|
||||
// Removing them from archiveSets means they are NOT tracked in minFailedId,
|
||||
// so the caller's watermark cap won't pin progress below them. The
|
||||
// SkippedPackage record stays so the user can manually retry via the UI.
|
||||
const cappedIds = await getCappedSkippedMessageIds(channel.id, config.maxSkipAttempts);
|
||||
if (cappedIds.size > 0) {
|
||||
const beforeCap = archiveSets.length;
|
||||
archiveSets = archiveSets.filter(
|
||||
(set) => !cappedIds.has(set.parts[0].id)
|
||||
);
|
||||
const cappedSkipped = beforeCap - archiveSets.length;
|
||||
if (cappedSkipped > 0) {
|
||||
accountLog.warn(
|
||||
{ cappedSkipped, cap: config.maxSkipAttempts, remaining: archiveSets.length },
|
||||
"Skipping archive sets that hit the auto-retry attempt cap — watermark will advance past them"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
counters.zipsFound += archiveSets.length;
|
||||
|
||||
// Match preview photos to archive sets
|
||||
@@ -754,8 +900,11 @@ async function processArchiveSets(
|
||||
messagesScanned: counters.messagesScanned,
|
||||
});
|
||||
|
||||
// Track the highest message ID that was successfully processed
|
||||
// Track the highest message ID that was successfully processed and the
|
||||
// lowest message ID of any failed set. The caller uses minFailedId to cap
|
||||
// the watermark so failures get retried on the next cycle.
|
||||
let maxProcessedId: bigint | null = null;
|
||||
let minFailedId: bigint | null = null;
|
||||
const indexedPackageRefs: IndexedPackageRef[] = [];
|
||||
|
||||
for (let setIdx = 0; setIdx < archiveSets.length; setIdx++) {
|
||||
@@ -787,6 +936,27 @@ async function processArchiveSets(
|
||||
maxProcessedId = setMaxId;
|
||||
}
|
||||
|
||||
// Persist watermark immediately so a worker restart or cycle timeout
|
||||
// doesn't throw away progress. We only advance below minFailedId so a
|
||||
// later-encountered failure (out of order, e.g., multipart spanning)
|
||||
// doesn't get buried by an earlier success in this scan. In practice
|
||||
// sets are processed oldest-first, so setMaxId rarely exceeds
|
||||
// minFailedId, but the cap keeps the invariant if it ever does.
|
||||
if (onWatermarkAdvance) {
|
||||
const safeWatermark =
|
||||
minFailedId !== null && setMaxId >= minFailedId
|
||||
? minFailedId - 1n
|
||||
: setMaxId;
|
||||
if (safeWatermark > 0n) {
|
||||
await onWatermarkAdvance(safeWatermark).catch((err) => {
|
||||
accountLog.warn(
|
||||
{ err, setMaxId: setMaxId.toString() },
|
||||
"Failed to persist incremental watermark (will retry at end of scan)"
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Reset stall counter on any successful upload
|
||||
ctx.consecutiveStalls = 0;
|
||||
} catch (setErr) {
|
||||
@@ -796,6 +966,16 @@ async function processArchiveSets(
|
||||
"Archive set failed, watermark will not advance past this set"
|
||||
);
|
||||
|
||||
// Record the lowest part ID of this set as a failure boundary so the
|
||||
// caller can cap the watermark below it and the next scan re-picks it up.
|
||||
const setMinId = archiveSets[setIdx].parts.reduce(
|
||||
(min, p) => (p.id < min ? p.id : min),
|
||||
archiveSets[setIdx].parts[0].id
|
||||
);
|
||||
if (minFailedId === null || setMinId < minFailedId) {
|
||||
minFailedId = setMinId;
|
||||
}
|
||||
|
||||
// ── TDLib client recreation on repeated upload stalls ──
|
||||
// When the TDLib event stream degrades, uploads complete (bytes sent)
|
||||
// but confirmations never arrive. Retrying with the same broken client
|
||||
@@ -852,11 +1032,15 @@ async function processArchiveSets(
|
||||
try {
|
||||
const archiveSet = archiveSets[setIdx];
|
||||
const totalSize = archiveSet.parts.reduce((sum, p) => sum + p.fileSize, 0n);
|
||||
const errMsg = setErr instanceof Error ? setErr.message : String(setErr);
|
||||
const rawErrMsg = setErr instanceof Error ? setErr.message : String(setErr);
|
||||
// Prefix with [<phone>] so the SkippedPackage / notification UI shows
|
||||
// which account hit the error — important when two accounts share a
|
||||
// source channel and only one is failing.
|
||||
const errMsg = `[${ctx.accountPhone}] ${rawErrMsg}`;
|
||||
await upsertSkippedPackage({
|
||||
fileName: archiveSet.parts[0].fileName,
|
||||
fileSize: totalSize,
|
||||
reason: inferSkipReason(errMsg),
|
||||
reason: inferSkipReason(rawErrMsg),
|
||||
errorMessage: errMsg,
|
||||
sourceChannelId: ctx.channel.id,
|
||||
sourceMessageId: archiveSet.parts[0].id,
|
||||
@@ -868,7 +1052,7 @@ async function processArchiveSets(
|
||||
// Also create a persistent notification
|
||||
await db.systemNotification.create({
|
||||
data: {
|
||||
type: inferSkipReason(errMsg) === "UPLOAD_FAILED" ? "UPLOAD_FAILED" : "DOWNLOAD_FAILED",
|
||||
type: inferSkipReason(rawErrMsg) === "UPLOAD_FAILED" ? "UPLOAD_FAILED" : "DOWNLOAD_FAILED",
|
||||
severity: "WARNING",
|
||||
title: `Failed to process ${archiveSet.parts[0].fileName}`,
|
||||
message: errMsg,
|
||||
@@ -877,7 +1061,8 @@ async function processArchiveSets(
|
||||
sourceChannelId: ctx.channel.id,
|
||||
sourceMessageId: Number(archiveSet.parts[0].id),
|
||||
channelTitle: ctx.channelTitle,
|
||||
reason: inferSkipReason(errMsg),
|
||||
accountPhone: ctx.accountPhone,
|
||||
reason: inferSkipReason(rawErrMsg),
|
||||
},
|
||||
},
|
||||
});
|
||||
@@ -929,7 +1114,7 @@ async function processArchiveSets(
|
||||
await detectGroupingConflicts(channel.id, indexedPackageRefs);
|
||||
}
|
||||
|
||||
return maxProcessedId;
|
||||
return { maxProcessedId, minFailedId };
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user