Fix worker getting stuck during sync: add timeouts, stuck detection, and safety limits

- Add invokeWithTimeout wrapper for TDLib API calls (2min timeout per call)
- Add stuck detection to getChannelMessages: break if from_message_id doesn't advance
- Add stuck detection to getTopicMessages: same protection for topic scanning
- Add stuck detection to getForumTopicList: break if pagination offsets don't advance
- Add max page limit (5000) to all scanning loops to prevent infinite pagination
- Add mutex wait timeout (30min) to prevent indefinite blocking when holder hangs
- Add cycle timeout (4h default, configurable via WORKER_CYCLE_TIMEOUT_MINUTES)
- Fix end-of-page detection to use actual limit value instead of hardcoded 100

Co-authored-by: xCyanGrizzly <53275238+xCyanGrizzly@users.noreply.github.com>

worker/dist/util/config.d.ts

@@ -0,0 +1,18 @@
+export declare const config: {
+    readonly databaseUrl: string;
+    readonly workerIntervalMinutes: number;
+    readonly tempDir: string;
+    readonly tdlibStateDir: string;
+    readonly maxZipSizeMB: number;
+    readonly logLevel: "debug" | "info" | "warn" | "error";
+    readonly telegramApiId: number;
+    readonly telegramApiHash: string;
+    /** Maximum jitter added to scheduler interval (in minutes) */
+    readonly jitterMinutes: 5;
+    /** Maximum time span for multipart archive parts (in hours). 0 = no limit. */
+    readonly multipartTimeoutHours: number;
+    /** Delay between Telegram API calls (in ms) to avoid rate limits */
+    readonly apiDelayMs: 1000;
+    /** Max retries for rate-limited requests */
+    readonly maxRetries: 5;
+};

worker/dist/util/config.js

@@ -0,0 +1,19 @@
+export const config = {
+    databaseUrl: process.env.DATABASE_URL ?? "",
+    workerIntervalMinutes: parseInt(process.env.WORKER_INTERVAL_MINUTES ?? "60", 10),
+    tempDir: process.env.WORKER_TEMP_DIR ?? "/tmp/zips",
+    tdlibStateDir: process.env.TDLIB_STATE_DIR ?? "/data/tdlib",
+    maxZipSizeMB: parseInt(process.env.WORKER_MAX_ZIP_SIZE_MB ?? "4096", 10),
+    logLevel: (process.env.LOG_LEVEL ?? "info"),
+    telegramApiId: parseInt(process.env.TELEGRAM_API_ID ?? "0", 10),
+    telegramApiHash: process.env.TELEGRAM_API_HASH ?? "",
+    /** Maximum jitter added to scheduler interval (in minutes) */
+    jitterMinutes: 5,
+    /** Maximum time span for multipart archive parts (in hours). 0 = no limit. */
+    multipartTimeoutHours: parseInt(process.env.MULTIPART_TIMEOUT_HOURS ?? "0", 10),
+    /** Delay between Telegram API calls (in ms) to avoid rate limits */
+    apiDelayMs: 1000,
+    /** Max retries for rate-limited requests */
+    maxRetries: 5,
+};
+//# sourceMappingURL=config.js.map

worker/dist/util/config.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/util/config.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,MAAM,GAAG;IACpB,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,EAAE;IAC3C,qBAAqB,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB,IAAI,IAAI,EAAE,EAAE,CAAC;IAChF,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,WAAW;IACnD,aAAa,EAAE,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,aAAa;IAC3D,YAAY,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,sBAAsB,IAAI,MAAM,EAAE,EAAE,CAAC;IACxE,QAAQ,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,MAAM,CAAwC;IAClF,aAAa,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,GAAG,EAAE,EAAE,CAAC;IAC/D,eAAe,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,EAAE;IACpD,8DAA8D;IAC9D,aAAa,EAAE,CAAC;IAChB,8EAA8E;IAC9E,qBAAqB,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB,IAAI,GAAG,EAAE,EAAE,CAAC;IAC/E,oEAAoE;IACpE,UAAU,EAAE,IAAI;IAChB,4CAA4C;IAC5C,UAAU,EAAE,CAAC;CACL,CAAC"}

worker/dist/util/logger.d.ts

@@ -0,0 +1,3 @@
+import pino from "pino";
+export declare const logger: pino.Logger<never, boolean>;
+export declare function childLogger(name: string, extra?: Record<string, unknown>): pino.Logger<never, boolean>;

worker/dist/util/logger.js

@@ -0,0 +1,12 @@
+import pino from "pino";
+import { config } from "./config.js";
+export const logger = pino({
+    level: config.logLevel,
+    transport: config.logLevel === "debug"
+        ? { target: "pino/file", options: { destination: 1 } }
+        : undefined,
+});
+export function childLogger(name, extra) {
+    return logger.child({ module: name, ...extra });
+}
+//# sourceMappingURL=logger.js.map

worker/dist/util/logger.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"logger.js","sourceRoot":"","sources":["../../src/util/logger.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC,MAAM,CAAC,MAAM,MAAM,GAAG,IAAI,CAAC;IACzB,KAAK,EAAE,MAAM,CAAC,QAAQ;IACtB,SAAS,EACP,MAAM,CAAC,QAAQ,KAAK,OAAO;QACzB,CAAC,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,EAAE;QACtD,CAAC,CAAC,SAAS;CAChB,CAAC,CAAC;AAEH,MAAM,UAAU,WAAW,CAAC,IAAY,EAAE,KAA+B;IACvE,OAAO,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,KAAK,EAAE,CAAC,CAAC;AAClD,CAAC"}

worker/dist/util/mutex.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Ensures only one TDLib client runs at a time across the entire worker process.
+ * Both the scheduler (auth, ingestion) and the fetch listener acquire this
+ * before creating any TDLib client.
+ *
+ * Includes a wait timeout to prevent indefinite blocking if the current holder hangs.
+ */
+export declare function withTdlibMutex<T>(label: string, fn: () => Promise<T>): Promise<T>;

worker/dist/util/mutex.js

@@ -0,0 +1,61 @@
+import { childLogger } from "./logger.js";
+const log = childLogger("mutex");
+let locked = false;
+let holder = "";
+const queue = [];
+/**
+ * Maximum time to wait for the TDLib mutex (ms).
+ * If the mutex is not available within this time, the operation is rejected.
+ * Default: 30 minutes (long enough for large downloads, short enough to detect hangs).
+ */
+const MUTEX_WAIT_TIMEOUT_MS = 30 * 60 * 1000;
+/**
+ * Ensures only one TDLib client runs at a time across the entire worker process.
+ * Both the scheduler (auth, ingestion) and the fetch listener acquire this
+ * before creating any TDLib client.
+ *
+ * Includes a wait timeout to prevent indefinite blocking if the current holder hangs.
+ */
+export async function withTdlibMutex(label, fn) {
+    if (locked) {
+        log.info({ waiting: label, holder }, "Waiting for TDLib mutex");
+        await new Promise((resolve, reject) => {
+            const entry = { resolve, reject, label };
+            queue.push(entry);
+            // Timeout: reject if we've been waiting too long
+            const timer = setTimeout(() => {
+                const idx = queue.indexOf(entry);
+                if (idx !== -1) {
+                    queue.splice(idx, 1);
+                    reject(new Error(`TDLib mutex wait timeout after ${MUTEX_WAIT_TIMEOUT_MS / 60_000}min ` +
+                        `(waiting: ${label}, holder: ${holder})`));
+                }
+            }, MUTEX_WAIT_TIMEOUT_MS);
+            // Wrap resolve to clear the timer
+            const origResolve = entry.resolve;
+            entry.resolve = () => {
+                clearTimeout(timer);
+                origResolve();
+            };
+        });
+    }
+    locked = true;
+    holder = label;
+    log.debug({ label }, "TDLib mutex acquired");
+    try {
+        return await fn();
+    }
+    finally {
+        locked = false;
+        holder = "";
+        const next = queue.shift();
+        if (next) {
+            log.debug({ next: next.label }, "TDLib mutex releasing to next waiter");
+            next.resolve();
+        }
+        else {
+            log.debug({ label }, "TDLib mutex released");
+        }
+    }
+}
+//# sourceMappingURL=mutex.js.map

worker/dist/util/mutex.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mutex.js","sourceRoot":"","sources":["../../src/util/mutex.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAE1C,MAAM,GAAG,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;AAEjC,IAAI,MAAM,GAAG,KAAK,CAAC;AACnB,IAAI,MAAM,GAAG,EAAE,CAAC;AAChB,MAAM,KAAK,GAAgF,EAAE,CAAC;AAE9F;;;;GAIG;AACH,MAAM,qBAAqB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAE7C;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,KAAa,EACb,EAAoB;IAEpB,IAAI,MAAM,EAAE,CAAC;QACX,GAAG,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,yBAAyB,CAAC,CAAC;QAChE,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC1C,MAAM,KAAK,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;YACzC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAElB,iDAAiD;YACjD,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;gBAC5B,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;gBACjC,IAAI,GAAG,KAAK,CAAC,CAAC,EAAE,CAAC;oBACf,KAAK,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;oBACrB,MAAM,CAAC,IAAI,KAAK,CACd,kCAAkC,qBAAqB,GAAG,MAAM,MAAM;wBACtE,aAAa,KAAK,aAAa,MAAM,GAAG,CACzC,CAAC,CAAC;gBACL,CAAC;YACH,CAAC,EAAE,qBAAqB,CAAC,CAAC;YAE1B,kCAAkC;YAClC,MAAM,WAAW,GAAG,KAAK,CAAC,OAAO,CAAC;YAClC,KAAK,CAAC,OAAO,GAAG,GAAG,EAAE;gBACnB,YAAY,CAAC,KAAK,CAAC,CAAC;gBACpB,WAAW,EAAE,CAAC;YAChB,CAAC,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IAED,MAAM,GAAG,IAAI,CAAC;IACd,MAAM,GAAG,KAAK,CAAC;IACf,GAAG,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,EAAE,sBAAsB,CAAC,CAAC;IAE7C,IAAI,CAAC;QACH,OAAO,MAAM,EAAE,EAAE,CAAC;IACpB,CAAC;YAAS,CAAC;QACT,MAAM,GAAG,KAAK,CAAC;QACf,MAAM,GAAG,EAAE,CAAC;QACZ,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;QAC3B,IAAI,IAAI,EAAE,CAAC;YACT,GAAG,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,KAAK,EAAE,EAAE,sCAAsC,CAAC,CAAC;YACxE,IAAI,CAAC,OAAO,EAAE,CAAC;QACjB,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,EAAE,sBAAsB,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;AACH,CAAC"}