mirror of
https://github.com/xCyanGrizzly/DragonsStash.git
synced 2026-07-22 23:12:02 +00:00
Compare commits
7 Commits
59038889ae
...
77aeb4cc00
| Author | SHA1 | Date | |
|---|---|---|---|
| 77aeb4cc00 | |||
| 3b327eb3f3 | |||
| 379bf246cd | |||
| 7a79b52baf | |||
| 26e2cba69d | |||
| 84cc8d995b | |||
| d99a506b10 |
@@ -0,0 +1,8 @@
|
|||||||
|
-- AlterTable: track how many times the worker has tried each skipped source message.
|
||||||
|
-- Existing rows default to 1 (they represent a single past attempt that the worker
|
||||||
|
-- chose to record). Future failures increment via upsertSkippedPackage.
|
||||||
|
ALTER TABLE "skipped_packages" ADD COLUMN "attemptCount" INTEGER NOT NULL DEFAULT 1;
|
||||||
|
|
||||||
|
-- AlterEnum: add CHANNEL_ACCESS_LOST so the worker can surface a notification
|
||||||
|
-- when a source channel becomes inaccessible (account removed, channel deleted, etc.)
|
||||||
|
ALTER TYPE "NotificationType" ADD VALUE 'CHANNEL_ACCESS_LOST';
|
||||||
@@ -741,6 +741,12 @@ model SkippedPackage {
|
|||||||
sourceTopicId BigInt?
|
sourceTopicId BigInt?
|
||||||
isMultipart Boolean @default(false)
|
isMultipart Boolean @default(false)
|
||||||
partCount Int @default(1)
|
partCount Int @default(1)
|
||||||
|
/// How many times the worker has tried to process this source message.
|
||||||
|
/// The worker auto-retries failures across cycles up to a configurable cap
|
||||||
|
/// (WORKER_MAX_SKIP_ATTEMPTS, default 5). After the cap, the watermark is
|
||||||
|
/// allowed to advance past the failure so cycles aren't pinned forever;
|
||||||
|
/// the user can manually retry via the UI to reset and try again.
|
||||||
|
attemptCount Int @default(1)
|
||||||
accountId String
|
accountId String
|
||||||
account TelegramAccount @relation(fields: [accountId], references: [id], onDelete: Cascade)
|
account TelegramAccount @relation(fields: [accountId], references: [id], onDelete: Cascade)
|
||||||
createdAt DateTime @default(now())
|
createdAt DateTime @default(now())
|
||||||
@@ -830,6 +836,7 @@ enum NotificationType {
|
|||||||
DOWNLOAD_FAILED
|
DOWNLOAD_FAILED
|
||||||
GROUPING_CONFLICT
|
GROUPING_CONFLICT
|
||||||
INTEGRITY_AUDIT
|
INTEGRITY_AUDIT
|
||||||
|
CHANNEL_ACCESS_LOST
|
||||||
}
|
}
|
||||||
|
|
||||||
enum NotificationSeverity {
|
enum NotificationSeverity {
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ export interface SkippedRow {
|
|||||||
sourceChannel: { id: string; title: string };
|
sourceChannel: { id: string; title: string };
|
||||||
isMultipart: boolean;
|
isMultipart: boolean;
|
||||||
partCount: number;
|
partCount: number;
|
||||||
|
attemptCount: number;
|
||||||
createdAt: string;
|
createdAt: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -107,9 +108,22 @@ export function getSkippedColumns({
|
|||||||
),
|
),
|
||||||
accessorFn: (row) => row.sourceChannel.title,
|
accessorFn: (row) => row.sourceChannel.title,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
accessorKey: "attemptCount",
|
||||||
|
header: ({ column }) => <DataTableColumnHeader column={column} title="Attempts" />,
|
||||||
|
cell: ({ row }) => {
|
||||||
|
const count = row.original.attemptCount;
|
||||||
|
const variant = count >= 5 ? "destructive" : count > 1 ? "secondary" : "outline";
|
||||||
|
return (
|
||||||
|
<Badge variant={variant} className="text-[10px]">
|
||||||
|
{count}
|
||||||
|
</Badge>
|
||||||
|
);
|
||||||
|
},
|
||||||
|
},
|
||||||
{
|
{
|
||||||
accessorKey: "createdAt",
|
accessorKey: "createdAt",
|
||||||
header: ({ column }) => <DataTableColumnHeader column={column} title="Skipped" />,
|
header: ({ column }) => <DataTableColumnHeader column={column} title="Last Skipped" />,
|
||||||
cell: ({ row }) => (
|
cell: ({ row }) => (
|
||||||
<span className="text-sm text-muted-foreground">
|
<span className="text-sm text-muted-foreground">
|
||||||
{new Date(row.original.createdAt).toLocaleDateString()}
|
{new Date(row.original.createdAt).toLocaleDateString()}
|
||||||
|
|||||||
@@ -589,6 +589,7 @@ export async function listSkippedPackages(options: {
|
|||||||
sourceMessageId: s.sourceMessageId.toString(),
|
sourceMessageId: s.sourceMessageId.toString(),
|
||||||
isMultipart: s.isMultipart,
|
isMultipart: s.isMultipart,
|
||||||
partCount: s.partCount,
|
partCount: s.partCount,
|
||||||
|
attemptCount: s.attemptCount,
|
||||||
createdAt: s.createdAt.toISOString(),
|
createdAt: s.createdAt.toISOString(),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
|
|||||||
@@ -55,6 +55,8 @@ export interface SkippedPackageItem {
|
|||||||
sourceMessageId: string;
|
sourceMessageId: string;
|
||||||
isMultipart: boolean;
|
isMultipart: boolean;
|
||||||
partCount: number;
|
partCount: number;
|
||||||
|
/** How many times the worker has tried this source message across cycles. */
|
||||||
|
attemptCount: number;
|
||||||
createdAt: string;
|
createdAt: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -625,6 +625,7 @@ export async function upsertSkippedPackage(data: {
|
|||||||
errorMessage: data.errorMessage ?? null,
|
errorMessage: data.errorMessage ?? null,
|
||||||
fileName: data.fileName,
|
fileName: data.fileName,
|
||||||
fileSize: data.fileSize,
|
fileSize: data.fileSize,
|
||||||
|
attemptCount: { increment: 1 },
|
||||||
createdAt: new Date(),
|
createdAt: new Date(),
|
||||||
},
|
},
|
||||||
create: {
|
create: {
|
||||||
@@ -642,6 +643,26 @@ export async function upsertSkippedPackage(data: {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return source-message IDs in a channel whose SkippedPackage attemptCount has
|
||||||
|
* reached or exceeded the cap — these are treated as "permanently failed for
|
||||||
|
* now" so the watermark can advance past them. The user can manually retry via
|
||||||
|
* the UI to reset the SkippedPackage record.
|
||||||
|
*/
|
||||||
|
export async function getCappedSkippedMessageIds(
|
||||||
|
sourceChannelId: string,
|
||||||
|
cap: number
|
||||||
|
): Promise<Set<bigint>> {
|
||||||
|
const rows = await db.skippedPackage.findMany({
|
||||||
|
where: {
|
||||||
|
sourceChannelId,
|
||||||
|
attemptCount: { gte: cap },
|
||||||
|
},
|
||||||
|
select: { sourceMessageId: true },
|
||||||
|
});
|
||||||
|
return new Set(rows.map((r) => r.sourceMessageId));
|
||||||
|
}
|
||||||
|
|
||||||
export async function deleteSkippedPackage(
|
export async function deleteSkippedPackage(
|
||||||
sourceChannelId: string,
|
sourceChannelId: string,
|
||||||
sourceMessageId: bigint
|
sourceMessageId: bigint
|
||||||
|
|||||||
@@ -119,25 +119,45 @@ async function sendWithRetry(
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stall or timeout — retry with a cooldown
|
// Stall or timeout — fail fast and let the caller recreate the TDLib
|
||||||
|
// client. Retrying on the same degraded event stream wastes ~15 min
|
||||||
|
// per attempt because the underlying issue (missing send-success
|
||||||
|
// events) is client-level, not transient. The set ends up in
|
||||||
|
// SkippedPackage and the caller's watermark cap ensures it gets
|
||||||
|
// retried next cycle on a fresh client.
|
||||||
const errMsg = err instanceof Error ? err.message : "";
|
const errMsg = err instanceof Error ? err.message : "";
|
||||||
if (errMsg.includes("stalled") || errMsg.includes("timed out")) {
|
if (errMsg.includes("stalled") || errMsg.includes("timed out")) {
|
||||||
if (!isLastAttempt) {
|
|
||||||
log.warn(
|
log.warn(
|
||||||
{ fileName, attempt: attempt + 1, maxRetries: MAX_UPLOAD_RETRIES },
|
{ fileName, attempt: attempt + 1 },
|
||||||
"Upload stalled/timed out — retrying"
|
"Upload stalled — failing fast so caller can recreate TDLib client"
|
||||||
);
|
);
|
||||||
await sleep(10_000);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
// All stall retries exhausted — throw UploadStallError so the caller
|
|
||||||
// knows the TDLib client's event stream is likely degraded and can
|
|
||||||
// recreate the client before continuing.
|
|
||||||
throw new UploadStallError(
|
throw new UploadStallError(
|
||||||
`Upload stalled after ${MAX_UPLOAD_RETRIES} retries for ${fileName}`
|
`Upload stalled for ${fileName}: ${errMsg}`
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Transient Telegram server-side error (HTTP 5xx returned via
|
||||||
|
// updateMessageSendFailed). These are NOT FLOOD_WAIT, NOT stalls — just
|
||||||
|
// TG having a bad moment. They typically resolve on a short backoff, so
|
||||||
|
// retry up to MAX_UPLOAD_RETRIES with linear backoff before giving up.
|
||||||
|
const lowerMsg = errMsg.toLowerCase();
|
||||||
|
const isTransientServerError =
|
||||||
|
lowerMsg.includes("internal server error") ||
|
||||||
|
lowerMsg.includes("internal error") ||
|
||||||
|
lowerMsg.includes("server error") ||
|
||||||
|
lowerMsg.includes("bad gateway") ||
|
||||||
|
lowerMsg.includes("service unavailable") ||
|
||||||
|
lowerMsg.includes("gateway timeout");
|
||||||
|
if (isTransientServerError && !isLastAttempt) {
|
||||||
|
const backoffMs = 15_000 * (attempt + 1) + Math.random() * 5_000;
|
||||||
|
log.warn(
|
||||||
|
{ fileName, attempt: attempt + 1, maxRetries: MAX_UPLOAD_RETRIES, backoffMs: Math.round(backoffMs) },
|
||||||
|
`Transient Telegram server error — retrying after backoff`
|
||||||
|
);
|
||||||
|
await sleep(backoffMs);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
throw err;
|
throw err;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -168,6 +188,12 @@ async function sendAndWaitForUpload(
|
|||||||
let lastProgressBytes = 0;
|
let lastProgressBytes = 0;
|
||||||
let lastProgressTime = Date.now();
|
let lastProgressTime = Date.now();
|
||||||
|
|
||||||
|
// Events for our message can arrive before `sendMessage` resolves
|
||||||
|
// (TDLib emits them while our .then() is still in the microtask queue).
|
||||||
|
// Buffer them and replay once tempMsgId is known.
|
||||||
|
let pendingSuccess: { oldMsgId: number; finalId: number } | null = null;
|
||||||
|
let pendingFailure: { oldMsgId: number; errorMsg: string; code?: number } | null = null;
|
||||||
|
|
||||||
// Timeout: 20 minutes per GB, minimum 15 minutes
|
// Timeout: 20 minutes per GB, minimum 15 minutes
|
||||||
const timeoutMs = Math.max(
|
const timeoutMs = Math.max(
|
||||||
15 * 60_000,
|
15 * 60_000,
|
||||||
@@ -204,6 +230,26 @@ async function sendAndWaitForUpload(
|
|||||||
}
|
}
|
||||||
}, 30_000);
|
}, 30_000);
|
||||||
|
|
||||||
|
const completeWithSuccess = (finalId: number) => {
|
||||||
|
if (settled) return;
|
||||||
|
settled = true;
|
||||||
|
cleanup();
|
||||||
|
log.info(
|
||||||
|
{ fileName, tempMsgId, finalMsgId: finalId },
|
||||||
|
"Upload confirmed by Telegram"
|
||||||
|
);
|
||||||
|
resolve(BigInt(finalId));
|
||||||
|
};
|
||||||
|
|
||||||
|
const completeWithFailure = (errorMsg: string, code?: number) => {
|
||||||
|
if (settled) return;
|
||||||
|
settled = true;
|
||||||
|
cleanup();
|
||||||
|
const error = new Error(`Upload failed for ${fileName}: ${errorMsg}`);
|
||||||
|
(error as Error & { code?: number }).code = code;
|
||||||
|
reject(error);
|
||||||
|
};
|
||||||
|
|
||||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||||
const handleUpdate = (update: any) => {
|
const handleUpdate = (update: any) => {
|
||||||
// Track upload progress via updateFile events
|
// Track upload progress via updateFile events
|
||||||
@@ -234,33 +280,29 @@ async function sendAndWaitForUpload(
|
|||||||
// The money event: upload succeeded, we get the final server message ID
|
// The money event: upload succeeded, we get the final server message ID
|
||||||
if (update?._ === "updateMessageSendSucceeded") {
|
if (update?._ === "updateMessageSendSucceeded") {
|
||||||
const msg = update.message;
|
const msg = update.message;
|
||||||
const oldMsgId = update.old_message_id;
|
const oldMsgId: number = update.old_message_id;
|
||||||
if (tempMsgId !== null && oldMsgId === tempMsgId) {
|
if (tempMsgId === null) {
|
||||||
if (!settled) {
|
// Race: event arrived before our .then() assigned tempMsgId.
|
||||||
settled = true;
|
// Buffer it and process once tempMsgId is known.
|
||||||
cleanup();
|
pendingSuccess = { oldMsgId, finalId: msg.id };
|
||||||
const finalId = BigInt(msg.id);
|
return;
|
||||||
log.info(
|
|
||||||
{ fileName, tempMsgId, finalMsgId: Number(finalId) },
|
|
||||||
"Upload confirmed by Telegram"
|
|
||||||
);
|
|
||||||
resolve(finalId);
|
|
||||||
}
|
}
|
||||||
|
if (oldMsgId === tempMsgId) {
|
||||||
|
completeWithSuccess(msg.id);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Upload failed
|
// Upload failed
|
||||||
if (update?._ === "updateMessageSendFailed") {
|
if (update?._ === "updateMessageSendFailed") {
|
||||||
const oldMsgId = update.old_message_id;
|
const oldMsgId: number = update.old_message_id;
|
||||||
if (tempMsgId !== null && oldMsgId === tempMsgId) {
|
const errorMsg: string = update.error?.message ?? "Unknown upload error";
|
||||||
if (!settled) {
|
const code: number | undefined = update.error?.code;
|
||||||
settled = true;
|
if (tempMsgId === null) {
|
||||||
cleanup();
|
pendingFailure = { oldMsgId, errorMsg, code };
|
||||||
const errorMsg = update.error?.message ?? "Unknown upload error";
|
return;
|
||||||
const error = new Error(`Upload failed for ${fileName}: ${errorMsg}`);
|
|
||||||
(error as Error & { code?: number }).code = update.error?.code;
|
|
||||||
reject(error);
|
|
||||||
}
|
}
|
||||||
|
if (oldMsgId === tempMsgId) {
|
||||||
|
completeWithFailure(errorMsg, code);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
@@ -304,6 +346,13 @@ async function sendAndWaitForUpload(
|
|||||||
{ fileName, tempMsgId },
|
{ fileName, tempMsgId },
|
||||||
"Message queued, waiting for upload confirmation"
|
"Message queued, waiting for upload confirmation"
|
||||||
);
|
);
|
||||||
|
|
||||||
|
// Replay any event that arrived before we knew tempMsgId
|
||||||
|
if (pendingSuccess && pendingSuccess.oldMsgId === tempMsgId) {
|
||||||
|
completeWithSuccess(pendingSuccess.finalId);
|
||||||
|
} else if (pendingFailure && pendingFailure.oldMsgId === tempMsgId) {
|
||||||
|
completeWithFailure(pendingFailure.errorMsg, pendingFailure.code);
|
||||||
|
}
|
||||||
})
|
})
|
||||||
.catch((err) => {
|
.catch((err) => {
|
||||||
if (!settled) {
|
if (!settled) {
|
||||||
|
|||||||
@@ -20,4 +20,8 @@ export const config = {
|
|||||||
apiDelayMs: 1000,
|
apiDelayMs: 1000,
|
||||||
/** Max retries for rate-limited requests */
|
/** Max retries for rate-limited requests */
|
||||||
maxRetries: 5,
|
maxRetries: 5,
|
||||||
|
/** After this many failed attempts on the same source message, the worker
|
||||||
|
* stops auto-retrying and lets the watermark advance past it. The user can
|
||||||
|
* manually retry via the UI to reset and try again. */
|
||||||
|
maxSkipAttempts: parseInt(process.env.WORKER_MAX_SKIP_ATTEMPTS ?? "5", 10),
|
||||||
} as const;
|
} as const;
|
||||||
|
|||||||
@@ -30,6 +30,7 @@ import {
|
|||||||
getUploadedPackageByHash,
|
getUploadedPackageByHash,
|
||||||
upsertSkippedPackage,
|
upsertSkippedPackage,
|
||||||
deleteSkippedPackage,
|
deleteSkippedPackage,
|
||||||
|
getCappedSkippedMessageIds,
|
||||||
} from "./db/queries.js";
|
} from "./db/queries.js";
|
||||||
import type { ActivityUpdate } from "./db/queries.js";
|
import type { ActivityUpdate } from "./db/queries.js";
|
||||||
import { createTdlibClient, closeTdlibClient } from "./tdlib/client.js";
|
import { createTdlibClient, closeTdlibClient } from "./tdlib/client.js";
|
||||||
@@ -408,6 +409,47 @@ export async function runWorkerForAccount(
|
|||||||
throw new Error("No global destination channel configured — set one in the admin UI");
|
throw new Error("No global destination channel configured — set one in the admin UI");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── Ensure TDLib knows about the destination chat ──
|
||||||
|
// Source channels get an explicit getChat below, but the destination was
|
||||||
|
// previously only loaded via loadChats — which can miss it if the account
|
||||||
|
// archived/moved it. Failing here surfaces the problem clearly instead of
|
||||||
|
// letting every upload fail with a cryptic "Chat not found".
|
||||||
|
try {
|
||||||
|
await client.invoke({
|
||||||
|
_: "getChat",
|
||||||
|
chat_id: Number(destChannel.telegramId),
|
||||||
|
});
|
||||||
|
} catch (destErr) {
|
||||||
|
accountLog.error(
|
||||||
|
{ err: destErr, destChannel: destChannel.title, telegramId: destChannel.telegramId.toString() },
|
||||||
|
"Destination chat is not accessible to this account — uploads will fail. Re-join via invite link or remove this account."
|
||||||
|
);
|
||||||
|
// Surface as a persistent notification so the admin sees it in the UI
|
||||||
|
try {
|
||||||
|
await db.systemNotification.create({
|
||||||
|
data: {
|
||||||
|
type: "UPLOAD_FAILED",
|
||||||
|
severity: "ERROR",
|
||||||
|
title: `Destination chat unreachable for ${account.phone}`,
|
||||||
|
message: `Account ${account.phone} cannot access the destination chat "${destChannel.title}". Uploads for this account will fail until access is restored. Re-join via the invite link in admin settings.`,
|
||||||
|
context: {
|
||||||
|
accountId: account.id,
|
||||||
|
accountPhone: account.phone,
|
||||||
|
destChannelId: destChannel.id,
|
||||||
|
destChannelTitle: destChannel.title,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
// Best-effort notification
|
||||||
|
}
|
||||||
|
// Skip this account's ingestion cycle entirely — there's no point
|
||||||
|
// scanning + downloading if we can't upload.
|
||||||
|
throw new Error(
|
||||||
|
`Destination chat "${destChannel.title}" is not accessible to account ${account.phone}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
const totalChannels = channelMappings.length;
|
const totalChannels = channelMappings.length;
|
||||||
|
|
||||||
if (totalChannels === 0) {
|
if (totalChannels === 0) {
|
||||||
@@ -560,13 +602,33 @@ export async function runWorkerForAccount(
|
|||||||
pipelineCtx.sourceTopicId = topic.topicId;
|
pipelineCtx.sourceTopicId = topic.topicId;
|
||||||
pipelineCtx.channelTitle = `${channel.title} › ${topic.name}`;
|
pipelineCtx.channelTitle = `${channel.title} › ${topic.name}`;
|
||||||
|
|
||||||
const maxProcessedId = await processArchiveSets(pipelineCtx, scanResult, run.id, progress?.lastProcessedMessageId);
|
const { maxProcessedId, minFailedId } = await processArchiveSets(
|
||||||
|
pipelineCtx,
|
||||||
|
scanResult,
|
||||||
|
run.id,
|
||||||
|
progress?.lastProcessedMessageId,
|
||||||
|
// Incremental watermark advance — saves progress per-set so a
|
||||||
|
// worker restart mid-scan doesn't lose all work.
|
||||||
|
async (messageId) => {
|
||||||
|
await upsertTopicProgress(
|
||||||
|
mapping.id,
|
||||||
|
topic.topicId,
|
||||||
|
topic.name,
|
||||||
|
messageId
|
||||||
|
);
|
||||||
|
}
|
||||||
|
);
|
||||||
// Sync client back in case it was recreated during upload stall recovery
|
// Sync client back in case it was recreated during upload stall recovery
|
||||||
client = pipelineCtx.client;
|
client = pipelineCtx.client;
|
||||||
|
|
||||||
// Advance progress: use archive watermark if available, fall back to scan watermark
|
// Final watermark write at the end of the scan (covers the
|
||||||
const topicWatermark = maxProcessedId ?? scanResult.maxScannedMessageId;
|
// no-archives-found and all-failures-with-fallback cases).
|
||||||
if (topicWatermark) {
|
// The incremental updates above already handle the success path.
|
||||||
|
let topicWatermark = maxProcessedId ?? scanResult.maxScannedMessageId;
|
||||||
|
if (minFailedId !== null && topicWatermark !== null && topicWatermark >= minFailedId) {
|
||||||
|
topicWatermark = minFailedId - 1n;
|
||||||
|
}
|
||||||
|
if (topicWatermark !== null) {
|
||||||
await upsertTopicProgress(
|
await upsertTopicProgress(
|
||||||
mapping.id,
|
mapping.id,
|
||||||
topic.topicId,
|
topic.topicId,
|
||||||
@@ -639,13 +701,28 @@ export async function runWorkerForAccount(
|
|||||||
pipelineCtx.sourceTopicId = null;
|
pipelineCtx.sourceTopicId = null;
|
||||||
pipelineCtx.channelTitle = channel.title;
|
pipelineCtx.channelTitle = channel.title;
|
||||||
|
|
||||||
const maxProcessedId = await processArchiveSets(pipelineCtx, scanResult, run.id, mapping.lastProcessedMessageId);
|
const { maxProcessedId, minFailedId } = await processArchiveSets(
|
||||||
|
pipelineCtx,
|
||||||
|
scanResult,
|
||||||
|
run.id,
|
||||||
|
mapping.lastProcessedMessageId,
|
||||||
|
// Incremental watermark advance — saves progress per-set so a
|
||||||
|
// worker restart mid-scan doesn't lose all work.
|
||||||
|
async (messageId) => {
|
||||||
|
await updateLastProcessedMessage(mapping.id, messageId);
|
||||||
|
}
|
||||||
|
);
|
||||||
// Sync client back in case it was recreated during upload stall recovery
|
// Sync client back in case it was recreated during upload stall recovery
|
||||||
client = pipelineCtx.client;
|
client = pipelineCtx.client;
|
||||||
|
|
||||||
// Advance progress: use archive watermark if available, fall back to scan watermark
|
// Final watermark write at the end of the scan (covers the
|
||||||
const channelWatermark = maxProcessedId ?? scanResult.maxScannedMessageId;
|
// no-archives-found and all-failures-with-fallback cases).
|
||||||
if (channelWatermark) {
|
// The incremental updates above already handle the success path.
|
||||||
|
let channelWatermark = maxProcessedId ?? scanResult.maxScannedMessageId;
|
||||||
|
if (minFailedId !== null && channelWatermark !== null && channelWatermark >= minFailedId) {
|
||||||
|
channelWatermark = minFailedId - 1n;
|
||||||
|
}
|
||||||
|
if (channelWatermark !== null) {
|
||||||
await updateLastProcessedMessage(mapping.id, channelWatermark);
|
await updateLastProcessedMessage(mapping.id, channelWatermark);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -654,6 +731,50 @@ export async function runWorkerForAccount(
|
|||||||
{ err: channelErr, channelId: channel.id, title: channel.title },
|
{ err: channelErr, channelId: channel.id, title: channel.title },
|
||||||
"Failed to process channel, skipping to next"
|
"Failed to process channel, skipping to next"
|
||||||
);
|
);
|
||||||
|
|
||||||
|
// If the channel is no longer accessible (account got removed,
|
||||||
|
// channel deleted, etc.), surface a persistent notification so the
|
||||||
|
// admin can decide what to do. Dedupe by (channelId, accountId)
|
||||||
|
// within the last 24h so we don't flood the notifications list every
|
||||||
|
// cycle.
|
||||||
|
const errMsg = channelErr instanceof Error ? channelErr.message : String(channelErr);
|
||||||
|
const isAccessError =
|
||||||
|
errMsg.includes("Can't access the chat") ||
|
||||||
|
errMsg.includes("CHAT_FORBIDDEN") ||
|
||||||
|
errMsg.includes("CHANNEL_PRIVATE") ||
|
||||||
|
errMsg.includes("Chat not found");
|
||||||
|
if (isAccessError) {
|
||||||
|
try {
|
||||||
|
const recent = await db.systemNotification.findFirst({
|
||||||
|
where: {
|
||||||
|
type: "CHANNEL_ACCESS_LOST",
|
||||||
|
context: { path: ["channelId"], equals: channel.id },
|
||||||
|
createdAt: { gte: new Date(Date.now() - 24 * 60 * 60 * 1000) },
|
||||||
|
},
|
||||||
|
select: { id: true },
|
||||||
|
});
|
||||||
|
if (!recent) {
|
||||||
|
await db.systemNotification.create({
|
||||||
|
data: {
|
||||||
|
type: "CHANNEL_ACCESS_LOST",
|
||||||
|
severity: "WARNING",
|
||||||
|
title: `Lost access to "${channel.title}" for ${account.phone}`,
|
||||||
|
message: `Account ${account.phone} can no longer access source channel "${channel.title}". The worker is skipping this channel every cycle. Re-join the channel or unlink it from this account in admin settings.`,
|
||||||
|
context: {
|
||||||
|
channelId: channel.id,
|
||||||
|
channelTitle: channel.title,
|
||||||
|
telegramId: channel.telegramId.toString(),
|
||||||
|
accountId: account.id,
|
||||||
|
accountPhone: account.phone,
|
||||||
|
errorMessage: errMsg.slice(0, 200),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Best-effort notification
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -704,8 +825,13 @@ async function processArchiveSets(
|
|||||||
ctx: PipelineContext,
|
ctx: PipelineContext,
|
||||||
scanResult: ChannelScanResult,
|
scanResult: ChannelScanResult,
|
||||||
ingestionRunId: string,
|
ingestionRunId: string,
|
||||||
lastProcessedMessageId?: bigint | null
|
lastProcessedMessageId?: bigint | null,
|
||||||
): Promise<bigint | null> {
|
/** Called after each successful set with a safe watermark value (capped
|
||||||
|
* below any failed message ID in this scan). Used by the caller to
|
||||||
|
* advance the channel/topic watermark incrementally — otherwise a long
|
||||||
|
* scan that gets killed by worker restart loses all progress. */
|
||||||
|
onWatermarkAdvance?: (messageId: bigint) => Promise<void>
|
||||||
|
): Promise<{ maxProcessedId: bigint | null; minFailedId: bigint | null }> {
|
||||||
const { client, runId, channelTitle, channel, throttled, counters, accountLog } = ctx;
|
const { client, runId, channelTitle, channel, throttled, counters, accountLog } = ctx;
|
||||||
|
|
||||||
// Group into archive sets
|
// Group into archive sets
|
||||||
@@ -726,6 +852,26 @@ async function processArchiveSets(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Filter out sets whose source message has hit the auto-retry cap — these are
|
||||||
|
// treated as "give up for now" so the watermark can advance past them.
|
||||||
|
// Removing them from archiveSets means they are NOT tracked in minFailedId,
|
||||||
|
// so the caller's watermark cap won't pin progress below them. The
|
||||||
|
// SkippedPackage record stays so the user can manually retry via the UI.
|
||||||
|
const cappedIds = await getCappedSkippedMessageIds(channel.id, config.maxSkipAttempts);
|
||||||
|
if (cappedIds.size > 0) {
|
||||||
|
const beforeCap = archiveSets.length;
|
||||||
|
archiveSets = archiveSets.filter(
|
||||||
|
(set) => !cappedIds.has(set.parts[0].id)
|
||||||
|
);
|
||||||
|
const cappedSkipped = beforeCap - archiveSets.length;
|
||||||
|
if (cappedSkipped > 0) {
|
||||||
|
accountLog.warn(
|
||||||
|
{ cappedSkipped, cap: config.maxSkipAttempts, remaining: archiveSets.length },
|
||||||
|
"Skipping archive sets that hit the auto-retry attempt cap — watermark will advance past them"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
counters.zipsFound += archiveSets.length;
|
counters.zipsFound += archiveSets.length;
|
||||||
|
|
||||||
// Match preview photos to archive sets
|
// Match preview photos to archive sets
|
||||||
@@ -754,8 +900,11 @@ async function processArchiveSets(
|
|||||||
messagesScanned: counters.messagesScanned,
|
messagesScanned: counters.messagesScanned,
|
||||||
});
|
});
|
||||||
|
|
||||||
// Track the highest message ID that was successfully processed
|
// Track the highest message ID that was successfully processed and the
|
||||||
|
// lowest message ID of any failed set. The caller uses minFailedId to cap
|
||||||
|
// the watermark so failures get retried on the next cycle.
|
||||||
let maxProcessedId: bigint | null = null;
|
let maxProcessedId: bigint | null = null;
|
||||||
|
let minFailedId: bigint | null = null;
|
||||||
const indexedPackageRefs: IndexedPackageRef[] = [];
|
const indexedPackageRefs: IndexedPackageRef[] = [];
|
||||||
|
|
||||||
for (let setIdx = 0; setIdx < archiveSets.length; setIdx++) {
|
for (let setIdx = 0; setIdx < archiveSets.length; setIdx++) {
|
||||||
@@ -787,6 +936,27 @@ async function processArchiveSets(
|
|||||||
maxProcessedId = setMaxId;
|
maxProcessedId = setMaxId;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Persist watermark immediately so a worker restart or cycle timeout
|
||||||
|
// doesn't throw away progress. We only advance below minFailedId so a
|
||||||
|
// later-encountered failure (out of order, e.g., multipart spanning)
|
||||||
|
// doesn't get buried by an earlier success in this scan. In practice
|
||||||
|
// sets are processed oldest-first, so setMaxId rarely exceeds
|
||||||
|
// minFailedId, but the cap keeps the invariant if it ever does.
|
||||||
|
if (onWatermarkAdvance) {
|
||||||
|
const safeWatermark =
|
||||||
|
minFailedId !== null && setMaxId >= minFailedId
|
||||||
|
? minFailedId - 1n
|
||||||
|
: setMaxId;
|
||||||
|
if (safeWatermark > 0n) {
|
||||||
|
await onWatermarkAdvance(safeWatermark).catch((err) => {
|
||||||
|
accountLog.warn(
|
||||||
|
{ err, setMaxId: setMaxId.toString() },
|
||||||
|
"Failed to persist incremental watermark (will retry at end of scan)"
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Reset stall counter on any successful upload
|
// Reset stall counter on any successful upload
|
||||||
ctx.consecutiveStalls = 0;
|
ctx.consecutiveStalls = 0;
|
||||||
} catch (setErr) {
|
} catch (setErr) {
|
||||||
@@ -796,6 +966,16 @@ async function processArchiveSets(
|
|||||||
"Archive set failed, watermark will not advance past this set"
|
"Archive set failed, watermark will not advance past this set"
|
||||||
);
|
);
|
||||||
|
|
||||||
|
// Record the lowest part ID of this set as a failure boundary so the
|
||||||
|
// caller can cap the watermark below it and the next scan re-picks it up.
|
||||||
|
const setMinId = archiveSets[setIdx].parts.reduce(
|
||||||
|
(min, p) => (p.id < min ? p.id : min),
|
||||||
|
archiveSets[setIdx].parts[0].id
|
||||||
|
);
|
||||||
|
if (minFailedId === null || setMinId < minFailedId) {
|
||||||
|
minFailedId = setMinId;
|
||||||
|
}
|
||||||
|
|
||||||
// ── TDLib client recreation on repeated upload stalls ──
|
// ── TDLib client recreation on repeated upload stalls ──
|
||||||
// When the TDLib event stream degrades, uploads complete (bytes sent)
|
// When the TDLib event stream degrades, uploads complete (bytes sent)
|
||||||
// but confirmations never arrive. Retrying with the same broken client
|
// but confirmations never arrive. Retrying with the same broken client
|
||||||
@@ -852,11 +1032,15 @@ async function processArchiveSets(
|
|||||||
try {
|
try {
|
||||||
const archiveSet = archiveSets[setIdx];
|
const archiveSet = archiveSets[setIdx];
|
||||||
const totalSize = archiveSet.parts.reduce((sum, p) => sum + p.fileSize, 0n);
|
const totalSize = archiveSet.parts.reduce((sum, p) => sum + p.fileSize, 0n);
|
||||||
const errMsg = setErr instanceof Error ? setErr.message : String(setErr);
|
const rawErrMsg = setErr instanceof Error ? setErr.message : String(setErr);
|
||||||
|
// Prefix with [<phone>] so the SkippedPackage / notification UI shows
|
||||||
|
// which account hit the error — important when two accounts share a
|
||||||
|
// source channel and only one is failing.
|
||||||
|
const errMsg = `[${ctx.accountPhone}] ${rawErrMsg}`;
|
||||||
await upsertSkippedPackage({
|
await upsertSkippedPackage({
|
||||||
fileName: archiveSet.parts[0].fileName,
|
fileName: archiveSet.parts[0].fileName,
|
||||||
fileSize: totalSize,
|
fileSize: totalSize,
|
||||||
reason: inferSkipReason(errMsg),
|
reason: inferSkipReason(rawErrMsg),
|
||||||
errorMessage: errMsg,
|
errorMessage: errMsg,
|
||||||
sourceChannelId: ctx.channel.id,
|
sourceChannelId: ctx.channel.id,
|
||||||
sourceMessageId: archiveSet.parts[0].id,
|
sourceMessageId: archiveSet.parts[0].id,
|
||||||
@@ -868,7 +1052,7 @@ async function processArchiveSets(
|
|||||||
// Also create a persistent notification
|
// Also create a persistent notification
|
||||||
await db.systemNotification.create({
|
await db.systemNotification.create({
|
||||||
data: {
|
data: {
|
||||||
type: inferSkipReason(errMsg) === "UPLOAD_FAILED" ? "UPLOAD_FAILED" : "DOWNLOAD_FAILED",
|
type: inferSkipReason(rawErrMsg) === "UPLOAD_FAILED" ? "UPLOAD_FAILED" : "DOWNLOAD_FAILED",
|
||||||
severity: "WARNING",
|
severity: "WARNING",
|
||||||
title: `Failed to process ${archiveSet.parts[0].fileName}`,
|
title: `Failed to process ${archiveSet.parts[0].fileName}`,
|
||||||
message: errMsg,
|
message: errMsg,
|
||||||
@@ -877,7 +1061,8 @@ async function processArchiveSets(
|
|||||||
sourceChannelId: ctx.channel.id,
|
sourceChannelId: ctx.channel.id,
|
||||||
sourceMessageId: Number(archiveSet.parts[0].id),
|
sourceMessageId: Number(archiveSet.parts[0].id),
|
||||||
channelTitle: ctx.channelTitle,
|
channelTitle: ctx.channelTitle,
|
||||||
reason: inferSkipReason(errMsg),
|
accountPhone: ctx.accountPhone,
|
||||||
|
reason: inferSkipReason(rawErrMsg),
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
@@ -929,7 +1114,7 @@ async function processArchiveSets(
|
|||||||
await detectGroupingConflicts(channel.id, indexedPackageRefs);
|
await detectGroupingConflicts(channel.id, indexedPackageRefs);
|
||||||
}
|
}
|
||||||
|
|
||||||
return maxProcessedId;
|
return { maxProcessedId, minFailedId };
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
Reference in New Issue
Block a user