Fix worker getting stuck during sync: add timeouts, stuck detection, and safety limits

- Add invokeWithTimeout wrapper for TDLib API calls (2min timeout per call)
- Add stuck detection to getChannelMessages: break if from_message_id doesn't advance
- Add stuck detection to getTopicMessages: same protection for topic scanning
- Add stuck detection to getForumTopicList: break if pagination offsets don't advance
- Add max page limit (5000) to all scanning loops to prevent infinite pagination
- Add mutex wait timeout (30min) to prevent indefinite blocking when holder hangs
- Add cycle timeout (4h default, configurable via WORKER_CYCLE_TIMEOUT_MINUTES)
- Fix end-of-page detection to use actual limit value instead of hardcoded 100

Co-authored-by: xCyanGrizzly <53275238+xCyanGrizzly@users.noreply.github.com>

worker/dist/util/mutex.js

@@ -0,0 +1,61 @@
+import { childLogger } from "./logger.js";
+const log = childLogger("mutex");
+let locked = false;
+let holder = "";
+const queue = [];
+/**
+ * Maximum time to wait for the TDLib mutex (ms).
+ * If the mutex is not available within this time, the operation is rejected.
+ * Default: 30 minutes (long enough for large downloads, short enough to detect hangs).
+ */
+const MUTEX_WAIT_TIMEOUT_MS = 30 * 60 * 1000;
+/**
+ * Ensures only one TDLib client runs at a time across the entire worker process.
+ * Both the scheduler (auth, ingestion) and the fetch listener acquire this
+ * before creating any TDLib client.
+ *
+ * Includes a wait timeout to prevent indefinite blocking if the current holder hangs.
+ */
+export async function withTdlibMutex(label, fn) {
+    if (locked) {
+        log.info({ waiting: label, holder }, "Waiting for TDLib mutex");
+        await new Promise((resolve, reject) => {
+            const entry = { resolve, reject, label };
+            queue.push(entry);
+            // Timeout: reject if we've been waiting too long
+            const timer = setTimeout(() => {
+                const idx = queue.indexOf(entry);
+                if (idx !== -1) {
+                    queue.splice(idx, 1);
+                    reject(new Error(`TDLib mutex wait timeout after ${MUTEX_WAIT_TIMEOUT_MS / 60_000}min ` +
+                        `(waiting: ${label}, holder: ${holder})`));
+                }
+            }, MUTEX_WAIT_TIMEOUT_MS);
+            // Wrap resolve to clear the timer
+            const origResolve = entry.resolve;
+            entry.resolve = () => {
+                clearTimeout(timer);
+                origResolve();
+            };
+        });
+    }
+    locked = true;
+    holder = label;
+    log.debug({ label }, "TDLib mutex acquired");
+    try {
+        return await fn();
+    }
+    finally {
+        locked = false;
+        holder = "";
+        const next = queue.shift();
+        if (next) {
+            log.debug({ next: next.label }, "TDLib mutex releasing to next waiter");
+            next.resolve();
+        }
+        else {
+            log.debug({ label }, "TDLib mutex released");
+        }
+    }
+}
+//# sourceMappingURL=mutex.js.map