Fix worker getting stuck during sync: add timeouts, stuck detection, and safety limits

- Add invokeWithTimeout wrapper for TDLib API calls (2min timeout per call)
- Add stuck detection to getChannelMessages: break if from_message_id doesn't advance
- Add stuck detection to getTopicMessages: same protection for topic scanning
- Add stuck detection to getForumTopicList: break if pagination offsets don't advance
- Add max page limit (5000) to all scanning loops to prevent infinite pagination
- Add mutex wait timeout (30min) to prevent indefinite blocking when holder hangs
- Add cycle timeout (4h default, configurable via WORKER_CYCLE_TIMEOUT_MINUTES)
- Fix end-of-page detection to use actual limit value instead of hardcoded 100

Co-authored-by: xCyanGrizzly <53275238+xCyanGrizzly@users.noreply.github.com>

worker/dist/archive/creator.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Extract a creator name from common archive file naming patterns.
+ *
+ * Priority in the worker: topic name > filename extraction.
+ * This is the fallback when no forum topic name is available.
+ *
+ * Patterns handled (split on ` - `):
+ *   "Mammoth Factory - 2026-01.zip"        → "Mammoth Factory"
+ *   "Artist Name - Pack Title.part01.rar"   → "Artist Name"
+ *   "some_random_file.zip"                  → null
+ */
+export declare function extractCreatorFromFileName(fileName: string): string | null;

worker/dist/archive/creator.js

@@ -0,0 +1,21 @@
+/**
+ * Extract a creator name from common archive file naming patterns.
+ *
+ * Priority in the worker: topic name > filename extraction.
+ * This is the fallback when no forum topic name is available.
+ *
+ * Patterns handled (split on ` - `):
+ *   "Mammoth Factory - 2026-01.zip"        → "Mammoth Factory"
+ *   "Artist Name - Pack Title.part01.rar"   → "Artist Name"
+ *   "some_random_file.zip"                  → null
+ */
+export function extractCreatorFromFileName(fileName) {
+    // Strip archive extensions (.zip, .rar, .part01.rar, .z01, etc.)
+    const bare = fileName.replace(/(\.(part\d+\.rar|z\d{2}|zip|rar))+$/i, "");
+    const idx = bare.indexOf(" - ");
+    if (idx <= 0)
+        return null;
+    const creator = bare.slice(0, idx).trim();
+    return creator.length > 0 ? creator : null;
+}
+//# sourceMappingURL=creator.js.map

worker/dist/archive/creator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"creator.js","sourceRoot":"","sources":["../../src/archive/creator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AACH,MAAM,UAAU,0BAA0B,CAAC,QAAgB;IACzD,iEAAiE;IACjE,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,sCAAsC,EAAE,EAAE,CAAC,CAAC;IAE1E,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAChC,IAAI,GAAG,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IAE1B,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IAC1C,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;AAC7C,CAAC"}

worker/dist/archive/detect.d.ts

@@ -0,0 +1,15 @@
+export type ArchiveFormat = "ZIP" | "RAR";
+export interface MultipartInfo {
+    baseName: string;
+    partNumber: number;
+    format: ArchiveFormat;
+    pattern: "ZIP_NUMBERED" | "ZIP_LEGACY" | "RAR_PART" | "RAR_LEGACY" | "SINGLE";
+}
+/**
+ * Detect if a filename is an archive and extract multipart info.
+ */
+export declare function detectArchive(fileName: string): MultipartInfo | null;
+/**
+ * Check if a filename looks like any archive attachment we should process.
+ */
+export declare function isArchiveAttachment(fileName: string): boolean;

worker/dist/archive/detect.js

@@ -0,0 +1,77 @@
+const patterns = [
+    // pack.zip.001, pack.zip.002
+    {
+        regex: /^(.+\.zip)\.(\d{3,})$/i,
+        format: "ZIP",
+        pattern: "ZIP_NUMBERED",
+        getBaseName: (m) => m[1],
+        getPartNumber: (m) => parseInt(m[2], 10),
+    },
+    // pack.z01, pack.z02 (legacy split — final part is pack.zip)
+    {
+        regex: /^(.+)\.z(\d{2,})$/i,
+        format: "ZIP",
+        pattern: "ZIP_LEGACY",
+        getBaseName: (m) => m[1],
+        getPartNumber: (m) => parseInt(m[2], 10),
+    },
+    // pack.part1.rar, pack.part2.rar
+    {
+        regex: /^(.+)\.part(\d+)\.rar$/i,
+        format: "RAR",
+        pattern: "RAR_PART",
+        getBaseName: (m) => m[1],
+        getPartNumber: (m) => parseInt(m[2], 10),
+    },
+    // pack.r00, pack.r01 (legacy split — final part is pack.rar)
+    {
+        regex: /^(.+)\.r(\d{2,})$/i,
+        format: "RAR",
+        pattern: "RAR_LEGACY",
+        getBaseName: (m) => m[1],
+        getPartNumber: (m) => parseInt(m[2], 10),
+    },
+];
+/**
+ * Detect if a filename is an archive and extract multipart info.
+ */
+export function detectArchive(fileName) {
+    // Check multipart patterns first
+    for (const p of patterns) {
+        const match = fileName.match(p.regex);
+        if (match) {
+            return {
+                baseName: p.getBaseName(match),
+                partNumber: p.getPartNumber(match),
+                format: p.format,
+                pattern: p.pattern,
+            };
+        }
+    }
+    // Single .zip file — could be a standalone or the final part of a ZIP_LEGACY set
+    if (/\.zip$/i.test(fileName)) {
+        return {
+            baseName: fileName.replace(/\.zip$/i, ""),
+            partNumber: -1, // -1 signals "could be single or final legacy part"
+            format: "ZIP",
+            pattern: "SINGLE",
+        };
+    }
+    // Single .rar file — could be standalone or final part of RAR_LEGACY set
+    if (/\.rar$/i.test(fileName)) {
+        return {
+            baseName: fileName.replace(/\.rar$/i, ""),
+            partNumber: -1,
+            format: "RAR",
+            pattern: "SINGLE",
+        };
+    }
+    return null;
+}
+/**
+ * Check if a filename looks like any archive attachment we should process.
+ */
+export function isArchiveAttachment(fileName) {
+    return detectArchive(fileName) !== null;
+}
+//# sourceMappingURL=detect.js.map

worker/dist/archive/detect.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"detect.js","sourceRoot":"","sources":["../../src/archive/detect.ts"],"names":[],"mappings":"AASA,MAAM,QAAQ,GAMR;IACJ,6BAA6B;IAC7B;QACE,KAAK,EAAE,wBAAwB;QAC/B,MAAM,EAAE,KAAK;QACb,OAAO,EAAE,cAAc;QACvB,WAAW,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACxB,aAAa,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;KACzC;IACD,6DAA6D;IAC7D;QACE,KAAK,EAAE,oBAAoB;QAC3B,MAAM,EAAE,KAAK;QACb,OAAO,EAAE,YAAY;QACrB,WAAW,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACxB,aAAa,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;KACzC;IACD,iCAAiC;IACjC;QACE,KAAK,EAAE,yBAAyB;QAChC,MAAM,EAAE,KAAK;QACb,OAAO,EAAE,UAAU;QACnB,WAAW,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACxB,aAAa,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;KACzC;IACD,6DAA6D;IAC7D;QACE,KAAK,EAAE,oBAAoB;QAC3B,MAAM,EAAE,KAAK;QACb,OAAO,EAAE,YAAY;QACrB,WAAW,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACxB,aAAa,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;KACzC;CACF,CAAC;AAEF;;GAEG;AACH,MAAM,UAAU,aAAa,CAAC,QAAgB;IAC5C,iCAAiC;IACjC,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QACtC,IAAI,KAAK,EAAE,CAAC;YACV,OAAO;gBACL,QAAQ,EAAE,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC;gBAC9B,UAAU,EAAE,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC;gBAClC,MAAM,EAAE,CAAC,CAAC,MAAM;gBAChB,OAAO,EAAE,CAAC,CAAC,OAAO;aACnB,CAAC;QACJ,CAAC;IACH,CAAC;IAED,iFAAiF;IACjF,IAAI,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC7B,OAAO;YACL,QAAQ,EAAE,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC;YACzC,UAAU,EAAE,CAAC,CAAC,EAAE,oDAAoD;YACpE,MAAM,EAAE,KAAK;YACb,OAAO,EAAE,QAAQ;SAClB,CAAC;IACJ,CAAC;IAED,yEAAyE;IACzE,IAAI,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC7B,OAAO;YACL,QAAQ,EAAE,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC;YACzC,UAAU,EAAE,CAAC,CAAC;YACd,MAAM,EAAE,KAAK;YACb,OAAO,EAAE,QAAQ;SAClB,CAAC;IACJ,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,QAAgB;IAClD,OAAO,aAAa,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC;AAC1C,CAAC"}

worker/dist/archive/hash.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Compute SHA-256 hash of one or more files by streaming them in order.
+ * Memory usage: O(1) — reads in 64KB chunks regardless of total size.
+ * For multipart archives, pass all parts sorted by part number.
+ */
+export declare function hashParts(filePaths: string[]): Promise<string>;

worker/dist/archive/hash.js

@@ -0,0 +1,22 @@
+import { createReadStream } from "fs";
+import { createHash } from "crypto";
+import { pipeline } from "stream/promises";
+import { PassThrough } from "stream";
+/**
+ * Compute SHA-256 hash of one or more files by streaming them in order.
+ * Memory usage: O(1) — reads in 64KB chunks regardless of total size.
+ * For multipart archives, pass all parts sorted by part number.
+ */
+export async function hashParts(filePaths) {
+    const hash = createHash("sha256");
+    for (const filePath of filePaths) {
+        await pipeline(createReadStream(filePath), new PassThrough({
+            transform(chunk, _encoding, callback) {
+                hash.update(chunk);
+                callback();
+            },
+        }));
+    }
+    return hash.digest("hex");
+}
+//# sourceMappingURL=hash.js.map

worker/dist/archive/hash.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"hash.js","sourceRoot":"","sources":["../../src/archive/hash.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,IAAI,CAAC;AACtC,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAC3C,OAAO,EAAE,WAAW,EAAE,MAAM,QAAQ,CAAC;AAErC;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,SAAmB;IACjD,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IAClC,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,MAAM,QAAQ,CACZ,gBAAgB,CAAC,QAAQ,CAAC,EAC1B,IAAI,WAAW,CAAC;YACd,SAAS,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ;gBAClC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACnB,QAAQ,EAAE,CAAC;YACb,CAAC;SACF,CAAC,CACH,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC5B,CAAC"}

worker/dist/archive/multipart.d.ts

@@ -0,0 +1,19 @@
+import { type ArchiveFormat } from "./detect.js";
+export interface TelegramMessage {
+    id: bigint;
+    fileName: string;
+    fileId: string;
+    fileSize: bigint;
+    date: Date;
+}
+export interface ArchiveSet {
+    type: ArchiveFormat;
+    baseName: string;
+    parts: TelegramMessage[];
+    isMultipart: boolean;
+}
+/**
+ * Group messages into archive sets (single files + multipart groups).
+ * Messages should be pre-filtered to only include archive attachments.
+ */
+export declare function groupArchiveSets(messages: TelegramMessage[]): ArchiveSet[];

worker/dist/archive/multipart.js

@@ -0,0 +1,74 @@
+import { detectArchive } from "./detect.js";
+import { config } from "../util/config.js";
+import { childLogger } from "../util/logger.js";
+const log = childLogger("multipart");
+/**
+ * Group messages into archive sets (single files + multipart groups).
+ * Messages should be pre-filtered to only include archive attachments.
+ */
+export function groupArchiveSets(messages) {
+    // Detect and annotate each message
+    const annotated = [];
+    for (const msg of messages) {
+        const info = detectArchive(msg.fileName);
+        if (info) {
+            annotated.push({ msg, info });
+        }
+    }
+    // Group by baseName + format
+    const groups = new Map();
+    for (const item of annotated) {
+        const key = `${item.info.format}:${item.info.baseName.toLowerCase()}`;
+        const group = groups.get(key) ?? [];
+        group.push(item);
+        groups.set(key, group);
+    }
+    const results = [];
+    for (const [, group] of groups) {
+        const format = group[0].info.format;
+        const baseName = group[0].info.baseName;
+        // Separate explicit multipart entries from potential singles
+        const multipartEntries = group.filter((g) => g.info.pattern !== "SINGLE");
+        const singleEntries = group.filter((g) => g.info.pattern === "SINGLE");
+        if (multipartEntries.length > 0) {
+            // This is a multipart set
+            // Check if any single entry is the "final part" of a legacy split
+            const allEntries = [...multipartEntries, ...singleEntries];
+            // Check time span — skip if parts span too long (0 = no limit)
+            if (config.multipartTimeoutHours > 0) {
+                const dates = allEntries.map((e) => e.msg.date.getTime());
+                const span = Math.max(...dates) - Math.min(...dates);
+                const maxSpanMs = config.multipartTimeoutHours * 60 * 60 * 1000;
+                if (span > maxSpanMs) {
+                    log.warn({ baseName, format, span: span / 3600000 }, "Multipart set spans too long, skipping");
+                    continue;
+                }
+            }
+            // Sort by part number (singles get a very high number so they come last — they're the final part)
+            allEntries.sort((a, b) => {
+                const aNum = a.info.partNumber === -1 ? 999999 : a.info.partNumber;
+                const bNum = b.info.partNumber === -1 ? 999999 : b.info.partNumber;
+                return aNum - bNum;
+            });
+            results.push({
+                type: format,
+                baseName,
+                parts: allEntries.map((e) => e.msg),
+                isMultipart: true,
+            });
+        }
+        else {
+            // All entries are singles — each is its own archive set
+            for (const entry of singleEntries) {
+                results.push({
+                    type: format,
+                    baseName: entry.info.baseName,
+                    parts: [entry.msg],
+                    isMultipart: false,
+                });
+            }
+        }
+    }
+    return results;
+}
+//# sourceMappingURL=multipart.js.map

worker/dist/archive/multipart.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"multipart.js","sourceRoot":"","sources":["../../src/archive/multipart.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAA0C,MAAM,aAAa,CAAC;AACpF,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,MAAM,GAAG,GAAG,WAAW,CAAC,WAAW,CAAC,CAAC;AAiBrC;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,QAA2B;IAC1D,mCAAmC;IACnC,MAAM,SAAS,GAAoD,EAAE,CAAC;IACtE,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACzC,IAAI,IAAI,EAAE,CAAC;YACT,SAAS,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;IAED,6BAA6B;IAC7B,MAAM,MAAM,GAAG,IAAI,GAAG,EAA2D,CAAC;IAClF,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,EAAE,CAAC;QACtE,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACpC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjB,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACzB,CAAC;IAED,MAAM,OAAO,GAAiB,EAAE,CAAC;IAEjC,KAAK,MAAM,CAAC,EAAE,KAAK,CAAC,IAAI,MAAM,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;QACpC,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC;QAExC,6DAA6D;QAC7D,MAAM,gBAAgB,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC;QAC1E,MAAM,aAAa,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC;QAEvE,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChC,0BAA0B;YAC1B,kEAAkE;YAClE,MAAM,UAAU,GAAG,CAAC,GAAG,gBAAgB,EAAE,GAAG,aAAa,CAAC,CAAC;YAE3D,+DAA+D;YAC/D,IAAI,MAAM,CAAC,qBAAqB,GAAG,CAAC,EAAE,CAAC;gBACrC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;gBAC1D,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC;gBACrD,MAAM,SAAS,GAAG,MAAM,CAAC,qBAAqB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;gBAEhE,IAAI,IAAI,GAAG,SAAS,EAAE,CAAC;oBACrB,GAAG,CAAC,IAAI,CACN,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,GAAG,OAAO,EAAE,EAC1C,wCAAwC,CACzC,CAAC;oBACF,SAAS;gBACX,CAAC;YACH,CAAC;YAED,kGAAkG;YAClG,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;gBACvB,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,UAAU,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC;gBACnE,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,UAAU,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC;gBACnE,OAAO,IAAI,GAAG,IAAI,CAAC;YACrB,CAAC,CAAC,CAAC;YAEH,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,MAAM;gBACZ,QAAQ;gBACR,KAAK,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;gBACnC,WAAW,EAAE,IAAI;aAClB,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,wDAAwD;YACxD,KAAK,MAAM,KAAK,IAAI,aAAa,EAAE,CAAC;gBAClC,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,MAAM;oBACZ,QAAQ,EAAE,KAAK,CAAC,IAAI,CAAC,QAAQ;oBAC7B,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC;oBAClB,WAAW,EAAE,KAAK;iBACnB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"}

worker/dist/archive/rar-reader.d.ts

@@ -0,0 +1,6 @@
+import type { FileEntry } from "./zip-reader.js";
+/**
+ * Parse output of `unrar l -v <file>` to extract file metadata.
+ * unrar automatically discovers sibling parts when they're co-located.
+ */
+export declare function readRarContents(firstPartPath: string): Promise<FileEntry[]>;

worker/dist/archive/rar-reader.js

@@ -0,0 +1,77 @@
+import { execFile } from "child_process";
+import { promisify } from "util";
+import path from "path";
+import { childLogger } from "../util/logger.js";
+const execFileAsync = promisify(execFile);
+const log = childLogger("rar-reader");
+/**
+ * Parse output of `unrar l -v <file>` to extract file metadata.
+ * unrar automatically discovers sibling parts when they're co-located.
+ */
+export async function readRarContents(firstPartPath) {
+    try {
+        const { stdout } = await execFileAsync("unrar", ["l", "-v", firstPartPath], {
+            timeout: 30000,
+            maxBuffer: 10 * 1024 * 1024, // 10MB for very large archives
+        });
+        return parseUnrarOutput(stdout);
+    }
+    catch (err) {
+        log.warn({ err, file: firstPartPath }, "Failed to read RAR contents");
+        return []; // Fallback: return empty on error
+    }
+}
+/**
+ * Parse the tabular output of `unrar l -v`.
+ *
+ * Example output format:
+ *  Archive: test.rar
+ *  Details: RAR 5
+ *
+ *   Attributes      Size     Packed Ratio   Date   Time   CRC-32  Name
+ *  ----------- ---------  --------- ----- -------- ----- --------  ----
+ *   ...A....      12345      10234  83%  2024-01-15 10:30 DEADBEEF  folder/file.stl
+ *  ----------- ---------  --------- ----- -------- ----- --------  ----
+ */
+function parseUnrarOutput(output) {
+    const entries = [];
+    const lines = output.split("\n");
+    let inFileList = false;
+    let separatorCount = 0;
+    for (const line of lines) {
+        const trimmed = line.trim();
+        // Detect separator lines (------- pattern)
+        if (/^-{5,}/.test(trimmed)) {
+            separatorCount++;
+            if (separatorCount === 1) {
+                inFileList = true;
+            }
+            else if (separatorCount >= 2) {
+                inFileList = false;
+            }
+            continue;
+        }
+        if (!inFileList)
+            continue;
+        // Parse file entry line
+        // Format: Attributes Size Packed Ratio Date Time CRC Name
+        const match = trimmed.match(/^\S+\s+(\d+)\s+(\d+)\s+\d+%\s+\S+\s+\S+\s+([0-9A-Fa-f]+)\s+(.+)$/);
+        if (match) {
+            const [, uncompressedStr, compressedStr, crc32, filePath] = match;
+            // Skip directory entries (typically end with / or have size 0 with dir attributes)
+            if (filePath.endsWith("/") || filePath.endsWith("\\"))
+                continue;
+            const ext = path.extname(filePath).toLowerCase();
+            entries.push({
+                path: filePath,
+                fileName: path.basename(filePath),
+                extension: ext ? ext.slice(1) : null,
+                compressedSize: BigInt(compressedStr),
+                uncompressedSize: BigInt(uncompressedStr),
+                crc32: crc32.toLowerCase(),
+            });
+        }
+    }
+    return entries;
+}
+//# sourceMappingURL=rar-reader.js.map

worker/dist/archive/rar-reader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"rar-reader.js","sourceRoot":"","sources":["../../src/archive/rar-reader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,MAAM,CAAC;AACjC,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAGhD,MAAM,aAAa,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;AAC1C,MAAM,GAAG,GAAG,WAAW,CAAC,YAAY,CAAC,CAAC;AAEtC;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,aAAqB;IAErB,IAAI,CAAC;QACH,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,aAAa,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,IAAI,EAAE,aAAa,CAAC,EAAE;YAC1E,OAAO,EAAE,KAAK;YACd,SAAS,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI,EAAE,+BAA+B;SAC7D,CAAC,CAAC;QAEH,OAAO,gBAAgB,CAAC,MAAM,CAAC,CAAC;IAClC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,aAAa,EAAE,EAAE,6BAA6B,CAAC,CAAC;QACtE,OAAO,EAAE,CAAC,CAAC,kCAAkC;IAC/C,CAAC;AACH,CAAC;AAED;;;;;;;;;;;GAWG;AACH,SAAS,gBAAgB,CAAC,MAAc;IACtC,MAAM,OAAO,GAAgB,EAAE,CAAC;IAChC,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAEjC,IAAI,UAAU,GAAG,KAAK,CAAC;IACvB,IAAI,cAAc,GAAG,CAAC,CAAC;IAEvB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAE5B,2CAA2C;QAC3C,IAAI,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAC3B,cAAc,EAAE,CAAC;YACjB,IAAI,cAAc,KAAK,CAAC,EAAE,CAAC;gBACzB,UAAU,GAAG,IAAI,CAAC;YACpB,CAAC;iBAAM,IAAI,cAAc,IAAI,CAAC,EAAE,CAAC;gBAC/B,UAAU,GAAG,KAAK,CAAC;YACrB,CAAC;YACD,SAAS;QACX,CAAC;QAED,IAAI,CAAC,UAAU;YAAE,SAAS;QAE1B,wBAAwB;QACxB,0DAA0D;QAC1D,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CACzB,kEAAkE,CACnE,CAAC;QAEF,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,CAAC,EAAE,eAAe,EAAE,aAAa,EAAE,KAAK,EAAE,QAAQ,CAAC,GAAG,KAAK,CAAC;YAElE,mFAAmF;YACnF,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC;gBAAE,SAAS;YAEhE,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;YACjD,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,QAAQ;gBACd,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBACjC,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI;gBACpC,cAAc,EAAE,MAAM,CAAC,aAAa,CAAC;gBACrC,gBAAgB,EAAE,MAAM,CAAC,eAAe,CAAC;gBACzC,KAAK,EAAE,KAAK,CAAC,WAAW,EAAE;aAC3B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"}

worker/dist/archive/split.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Split a file into ≤2GB parts using byte-level splitting.
+ * Returns paths to the split parts. If the file is already ≤2GB, returns the original path.
+ */
+export declare function byteLevelSplit(filePath: string): Promise<string[]>;
+/**
+ * Concatenate multiple files into a single output file by streaming
+ * each input sequentially. Used for repacking multipart archives
+ * that have oversized parts (>2GB) before re-splitting.
+ */
+export declare function concatenateFiles(inputPaths: string[], outputPath: string): Promise<void>;

worker/dist/archive/split.js

@@ -0,0 +1,55 @@
+import { createReadStream, createWriteStream } from "fs";
+import { stat } from "fs/promises";
+import path from "path";
+import { pipeline } from "stream/promises";
+import { childLogger } from "../util/logger.js";
+const log = childLogger("split");
+/** 2GB in bytes — Telegram's file size limit */
+const MAX_PART_SIZE = 2n * 1024n * 1024n * 1024n;
+/**
+ * Split a file into ≤2GB parts using byte-level splitting.
+ * Returns paths to the split parts. If the file is already ≤2GB, returns the original path.
+ */
+export async function byteLevelSplit(filePath) {
+    const stats = await stat(filePath);
+    const fileSize = BigInt(stats.size);
+    if (fileSize <= MAX_PART_SIZE) {
+        return [filePath];
+    }
+    const dir = path.dirname(filePath);
+    const baseName = path.basename(filePath);
+    const partSize = Number(MAX_PART_SIZE);
+    const totalParts = Math.ceil(Number(fileSize) / partSize);
+    const parts = [];
+    log.info({ filePath, fileSize: Number(fileSize), totalParts }, "Splitting file");
+    for (let i = 0; i < totalParts; i++) {
+        const partNum = String(i + 1).padStart(3, "0");
+        const partPath = path.join(dir, `${baseName}.${partNum}`);
+        const start = i * partSize;
+        const end = Math.min(start + partSize - 1, Number(fileSize) - 1);
+        await pipeline(createReadStream(filePath, { start, end }), createWriteStream(partPath));
+        parts.push(partPath);
+    }
+    log.info({ filePath, parts: parts.length }, "File split complete");
+    return parts;
+}
+/**
+ * Concatenate multiple files into a single output file by streaming
+ * each input sequentially. Used for repacking multipart archives
+ * that have oversized parts (>2GB) before re-splitting.
+ */
+export async function concatenateFiles(inputPaths, outputPath) {
+    const out = createWriteStream(outputPath);
+    for (let i = 0; i < inputPaths.length; i++) {
+        log.info({ part: i + 1, total: inputPaths.length, file: path.basename(inputPaths[i]) }, "Concatenating part");
+        await pipeline(createReadStream(inputPaths[i]), out, { end: false });
+    }
+    // Close the output stream
+    await new Promise((resolve, reject) => {
+        out.end(() => resolve());
+        out.on("error", reject);
+    });
+    const stats = await stat(outputPath);
+    log.info({ outputPath, totalBytes: stats.size, parts: inputPaths.length }, "Concatenation complete");
+}
+//# sourceMappingURL=split.js.map

worker/dist/archive/split.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"split.js","sourceRoot":"","sources":["../../src/archive/split.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,IAAI,CAAC;AACzD,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AACnC,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAC3C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,MAAM,GAAG,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;AAEjC,gDAAgD;AAChD,MAAM,aAAa,GAAG,EAAE,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,CAAC;AAEjD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,QAAgB;IACnD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;IACnC,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAEpC,IAAI,QAAQ,IAAI,aAAa,EAAE,CAAC;QAC9B,OAAO,CAAC,QAAQ,CAAC,CAAC;IACpB,CAAC;IAED,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACzC,MAAM,QAAQ,GAAG,MAAM,CAAC,aAAa,CAAC,CAAC;IACvC,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,QAAQ,CAAC,CAAC;IAC1D,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,GAAG,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC,EAAE,UAAU,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAEjF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,MAAM,OAAO,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAC/C,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,IAAI,OAAO,EAAE,CAAC,CAAC;QAC1D,MAAM,KAAK,GAAG,CAAC,GAAG,QAAQ,CAAC;QAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,QAAQ,GAAG,CAAC,EAAE,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;QAEjE,MAAM,QAAQ,CACZ,gBAAgB,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,EAC1C,iBAAiB,CAAC,QAAQ,CAAC,CAC5B,CAAC;QAEF,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACvB,CAAC;IAED,GAAG,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,EAAE,qBAAqB,CAAC,CAAC;IACnE,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,UAAoB,EACpB,UAAkB;IAElB,MAAM,GAAG,GAAG,iBAAiB,CAAC,UAAU,CAAC,CAAC;IAE1C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3C,GAAG,CAAC,IAAI,CACN,EAAE,IAAI,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,UAAU,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,EAC7E,oBAAoB,CACrB,CAAC;QACF,MAAM,QAAQ,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC;IACvE,CAAC;IAED,0BAA0B;IAC1B,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC1C,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;QACzB,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAC1B,CAAC,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC;IACrC,GAAG,CAAC,IAAI,CACN,EAAE,UAAU,EAAE,UAAU,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,UAAU,CAAC,MAAM,EAAE,EAChE,wBAAwB,CACzB,CAAC;AACJ,CAAC"}

worker/dist/archive/zip-reader.d.ts

@@ -0,0 +1,15 @@
+export interface FileEntry {
+    path: string;
+    fileName: string;
+    extension: string | null;
+    compressedSize: bigint;
+    uncompressedSize: bigint;
+    crc32: string | null;
+}
+/**
+ * Read the central directory of a ZIP file without extracting any contents.
+ * For multipart ZIPs (.zip.001, .zip.002 etc.), uses a custom random-access
+ * reader that spans all parts seamlessly so yauzl can find the central
+ * directory at the end of the combined data.
+ */
+export declare function readZipCentralDirectory(filePaths: string[]): Promise<FileEntry[]>;

worker/dist/archive/zip-reader.js

@@ -0,0 +1,161 @@
+import yauzl from "yauzl";
+import { open as fsOpen, stat as fsStat } from "fs/promises";
+import path from "path";
+import { Readable } from "stream";
+import { childLogger } from "../util/logger.js";
+const log = childLogger("zip-reader");
+/**
+ * Read the central directory of a ZIP file without extracting any contents.
+ * For multipart ZIPs (.zip.001, .zip.002 etc.), uses a custom random-access
+ * reader that spans all parts seamlessly so yauzl can find the central
+ * directory at the end of the combined data.
+ */
+export async function readZipCentralDirectory(filePaths) {
+    if (filePaths.length === 1) {
+        return readSingleZip(filePaths[0]);
+    }
+    // Multipart: use a spanning random-access reader
+    return readMultipartZip(filePaths);
+}
+/** Read a single (non-split) ZIP file. */
+function readSingleZip(targetFile) {
+    return new Promise((resolve) => {
+        yauzl.open(targetFile, { lazyEntries: true, autoClose: true }, (err, zipFile) => {
+            if (err) {
+                log.warn({ err, file: targetFile }, "Failed to open ZIP for reading");
+                resolve([]);
+                return;
+            }
+            const entries = [];
+            zipFile.readEntry();
+            zipFile.on("entry", (entry) => {
+                if (!entry.fileName.endsWith("/")) {
+                    const ext = path.extname(entry.fileName).toLowerCase();
+                    entries.push({
+                        path: entry.fileName,
+                        fileName: path.basename(entry.fileName),
+                        extension: ext ? ext.slice(1) : null,
+                        compressedSize: BigInt(entry.compressedSize),
+                        uncompressedSize: BigInt(entry.uncompressedSize),
+                        crc32: entry.crc32 !== 0 ? entry.crc32.toString(16).padStart(8, "0") : null,
+                    });
+                }
+                zipFile.readEntry();
+            });
+            zipFile.on("end", () => resolve(entries));
+            zipFile.on("error", (error) => {
+                log.warn({ error, file: targetFile }, "Error reading ZIP entries");
+                resolve(entries);
+            });
+        });
+    });
+}
+/**
+ * Read a multipart split ZIP using yauzl's RandomAccessReader API.
+ * This creates a virtual "file" that spans all parts so yauzl can
+ * seek freely across the entire archive to read the central directory.
+ */
+async function readMultipartZip(filePaths) {
+    // Get sizes of all parts
+    const partSizes = [];
+    for (const fp of filePaths) {
+        const s = await fsStat(fp);
+        partSizes.push(s.size);
+    }
+    const totalSize = partSizes.reduce((a, b) => a + b, 0);
+    log.debug({ parts: filePaths.length, totalSize }, "Reading multipart ZIP via spanning reader");
+    return new Promise((resolve) => {
+        const reader = createMultiPartReader(filePaths, partSizes);
+        yauzl.fromRandomAccessReader(reader, totalSize, { lazyEntries: true, autoClose: true }, (err, zipFile) => {
+            if (err) {
+                log.warn({ err }, "Failed to open multipart ZIP for reading");
+                reader.close(() => { });
+                resolve([]);
+                return;
+            }
+            const entries = [];
+            zipFile.readEntry();
+            zipFile.on("entry", (entry) => {
+                if (!entry.fileName.endsWith("/")) {
+                    const ext = path.extname(entry.fileName).toLowerCase();
+                    entries.push({
+                        path: entry.fileName,
+                        fileName: path.basename(entry.fileName),
+                        extension: ext ? ext.slice(1) : null,
+                        compressedSize: BigInt(entry.compressedSize),
+                        uncompressedSize: BigInt(entry.uncompressedSize),
+                        crc32: entry.crc32 !== 0 ? entry.crc32.toString(16).padStart(8, "0") : null,
+                    });
+                }
+                zipFile.readEntry();
+            });
+            zipFile.on("end", () => {
+                log.info({ entries: entries.length }, "Multipart ZIP entries read");
+                resolve(entries);
+            });
+            zipFile.on("error", (error) => {
+                log.warn({ error }, "Error reading multipart ZIP entries");
+                resolve(entries);
+            });
+        });
+    });
+}
+/**
+ * Create a yauzl RandomAccessReader that reads across multiple split part files.
+ * Maps a global offset to the correct part file and local offset.
+ *
+ * Uses Object.create to properly inherit from yauzl.RandomAccessReader
+ * (whose constructor + prototype is defined at runtime, not as a TS class).
+ */
+function createMultiPartReader(filePaths, partSizes) {
+    // Build cumulative offset table
+    const partOffsets = [];
+    let offset = 0;
+    for (const size of partSizes) {
+        partOffsets.push(offset);
+        offset += size;
+    }
+    // Create an instance by calling the parent constructor
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const reader = new yauzl.RandomAccessReader();
+    // Override _readStreamForRange — yauzl calls this to read a range of bytes
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    reader._readStreamForRange = function (start, end) {
+        const readable = new Readable({ read() { } });
+        readRange(start, end, readable).catch((err) => {
+            readable.destroy(err);
+        });
+        return readable;
+    };
+    async function readRange(start, end, readable) {
+        let remaining = end - start;
+        let globalOffset = start;
+        while (remaining > 0) {
+            // Find which part this offset falls in
+            let partIdx = partOffsets.length - 1;
+            for (let i = 0; i < partOffsets.length; i++) {
+                if (i + 1 < partOffsets.length && globalOffset < partOffsets[i + 1]) {
+                    partIdx = i;
+                    break;
+                }
+            }
+            const localOffset = globalOffset - partOffsets[partIdx];
+            const partRemaining = partSizes[partIdx] - localOffset;
+            const toRead = Math.min(remaining, partRemaining);
+            const fh = await fsOpen(filePaths[partIdx], "r");
+            try {
+                const buf = Buffer.alloc(toRead);
+                const { bytesRead } = await fh.read(buf, 0, toRead, localOffset);
+                readable.push(buf.subarray(0, bytesRead));
+                remaining -= bytesRead;
+                globalOffset += bytesRead;
+            }
+            finally {
+                await fh.close();
+            }
+        }
+        readable.push(null); // Signal end of stream
+    }
+    return reader;
+}
+//# sourceMappingURL=zip-reader.js.map

worker/dist/db/client.d.ts

@@ -0,0 +1,7 @@
+import { PrismaClient } from "@prisma/client";
+import { PrismaPg } from "@prisma/adapter-pg";
+declare const pool: import("pg").Pool;
+export declare const db: PrismaClient<{
+    adapter: PrismaPg;
+}, never, import("@prisma/client/runtime/client").DefaultArgs>;
+export { pool };

worker/dist/db/client.js

@@ -0,0 +1,12 @@
+import { PrismaClient } from "@prisma/client";
+import { PrismaPg } from "@prisma/adapter-pg";
+import pg from "pg";
+import { config } from "../util/config.js";
+const pool = new pg.Pool({
+    connectionString: config.databaseUrl,
+    max: 5,
+});
+const adapter = new PrismaPg(pool);
+export const db = new PrismaClient({ adapter });
+export { pool };
+//# sourceMappingURL=client.js.map

worker/dist/db/client.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"client.js","sourceRoot":"","sources":["../../src/db/client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,MAAM,IAAI,GAAG,IAAI,EAAE,CAAC,IAAI,CAAC;IACvB,gBAAgB,EAAE,MAAM,CAAC,WAAW;IACpC,GAAG,EAAE,CAAC;CACP,CAAC,CAAC;AAEH,MAAM,OAAO,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC;AACnC,MAAM,CAAC,MAAM,EAAE,GAAG,IAAI,YAAY,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC;AAEhD,OAAO,EAAE,IAAI,EAAE,CAAC"}

worker/dist/db/locks.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Try to acquire a PostgreSQL advisory lock for an account.
+ * Returns true if acquired, false if already held by another session.
+ */
+export declare function tryAcquireLock(accountId: string): Promise<boolean>;
+/**
+ * Release the advisory lock for an account.
+ */
+export declare function releaseLock(accountId: string): Promise<void>;

worker/dist/db/locks.js

@@ -0,0 +1,53 @@
+import { pool } from "./client.js";
+import { childLogger } from "../util/logger.js";
+const log = childLogger("locks");
+/**
+ * Derive a stable 32-bit integer lock ID from an account ID string.
+ * PostgreSQL advisory locks use bigint, but we use 32-bit for safety.
+ */
+function hashToLockId(accountId) {
+    let hash = 0;
+    for (let i = 0; i < accountId.length; i++) {
+        const char = accountId.charCodeAt(i);
+        hash = (hash << 5) - hash + char;
+        hash |= 0; // Convert to 32-bit integer
+    }
+    return Math.abs(hash);
+}
+/**
+ * Try to acquire a PostgreSQL advisory lock for an account.
+ * Returns true if acquired, false if already held by another session.
+ */
+export async function tryAcquireLock(accountId) {
+    const lockId = hashToLockId(accountId);
+    const client = await pool.connect();
+    try {
+        const result = await client.query("SELECT pg_try_advisory_lock($1)", [lockId]);
+        const acquired = result.rows[0]?.pg_try_advisory_lock ?? false;
+        if (acquired) {
+            log.debug({ accountId, lockId }, "Advisory lock acquired");
+        }
+        else {
+            log.debug({ accountId, lockId }, "Advisory lock already held");
+        }
+        return acquired;
+    }
+    finally {
+        client.release();
+    }
+}
+/**
+ * Release the advisory lock for an account.
+ */
+export async function releaseLock(accountId) {
+    const lockId = hashToLockId(accountId);
+    const client = await pool.connect();
+    try {
+        await client.query("SELECT pg_advisory_unlock($1)", [lockId]);
+        log.debug({ accountId, lockId }, "Advisory lock released");
+    }
+    finally {
+        client.release();
+    }
+}
+//# sourceMappingURL=locks.js.map

worker/dist/db/locks.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"locks.js","sourceRoot":"","sources":["../../src/db/locks.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AACnC,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,MAAM,GAAG,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;AAEjC;;;GAGG;AACH,SAAS,YAAY,CAAC,SAAiB;IACrC,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,MAAM,IAAI,GAAG,SAAS,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QACrC,IAAI,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;QACjC,IAAI,IAAI,CAAC,CAAC,CAAC,4BAA4B;IACzC,CAAC;IACD,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;AACxB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,SAAiB;IACpD,MAAM,MAAM,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC;IACvC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;IACpC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,KAAK,CAC/B,iCAAiC,EACjC,CAAC,MAAM,CAAC,CACT,CAAC;QACF,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,oBAAoB,IAAI,KAAK,CAAC;QAC/D,IAAI,QAAQ,EAAE,CAAC;YACb,GAAG,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,wBAAwB,CAAC,CAAC;QAC7D,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,4BAA4B,CAAC,CAAC;QACjE,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;YAAS,CAAC;QACT,MAAM,CAAC,OAAO,EAAE,CAAC;IACnB,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,SAAiB;IACjD,MAAM,MAAM,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC;IACvC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;IACpC,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,KAAK,CAAC,+BAA+B,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC;QAC9D,GAAG,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,wBAAwB,CAAC,CAAC;IAC7D,CAAC;YAAS,CAAC;QACT,MAAM,CAAC,OAAO,EAAE,CAAC;IACnB,CAAC;AACH,CAAC"}

worker/dist/db/queries.d.ts

@@ -0,0 +1,356 @@
+import type { ArchiveType, FetchStatus } from "@prisma/client";
+export declare function getActiveAccounts(): Promise<{
+    id: string;
+    phone: string;
+    displayName: string | null;
+    isActive: boolean;
+    authState: import("@prisma/client").$Enums.AuthState;
+    authCode: string | null;
+    lastSeenAt: Date | null;
+    createdAt: Date;
+    updatedAt: Date;
+}[]>;
+export declare function getPendingAccounts(): Promise<{
+    id: string;
+    phone: string;
+    displayName: string | null;
+    isActive: boolean;
+    authState: import("@prisma/client").$Enums.AuthState;
+    authCode: string | null;
+    lastSeenAt: Date | null;
+    createdAt: Date;
+    updatedAt: Date;
+}[]>;
+export declare function hasAnyChannels(): Promise<boolean>;
+export declare function getSourceChannelMappings(accountId: string): Promise<({
+    channel: {
+        id: string;
+        isActive: boolean;
+        createdAt: Date;
+        updatedAt: Date;
+        telegramId: bigint;
+        title: string;
+        type: import("@prisma/client").$Enums.ChannelType;
+        isForum: boolean;
+    };
+} & {
+    accountId: string;
+    id: string;
+    createdAt: Date;
+    channelId: string;
+    role: import("@prisma/client").$Enums.ChannelRole;
+    lastProcessedMessageId: bigint | null;
+})[]>;
+export declare function getGlobalDestinationChannel(): Promise<{
+    id: string;
+    isActive: boolean;
+    createdAt: Date;
+    updatedAt: Date;
+    telegramId: bigint;
+    title: string;
+    type: import("@prisma/client").$Enums.ChannelType;
+    isForum: boolean;
+} | null>;
+export declare function getGlobalSetting(key: string): Promise<string | null>;
+export declare function setGlobalSetting(key: string, value: string): Promise<{
+    updatedAt: Date;
+    key: string;
+    value: string;
+}>;
+export declare function packageExistsByHash(contentHash: string): Promise<boolean>;
+/**
+ * Check if a package already exists for a given source message ID
+ * AND was successfully uploaded to the destination (destMessageId is set).
+ * Used as an early skip before downloading.
+ */
+export declare function packageExistsBySourceMessage(sourceChannelId: string, sourceMessageId: bigint): Promise<boolean>;
+/**
+ * Delete orphaned Package rows that have the same content hash but never
+ * completed the upload (destMessageId is null). Called before creating a
+ * new complete record to avoid unique constraint violations.
+ */
+export declare function deleteOrphanedPackageByHash(contentHash: string): Promise<void>;
+export interface CreatePackageInput {
+    contentHash: string;
+    fileName: string;
+    fileSize: bigint;
+    archiveType: ArchiveType;
+    sourceChannelId: string;
+    sourceMessageId: bigint;
+    sourceTopicId?: bigint | null;
+    destChannelId?: string;
+    destMessageId?: bigint;
+    isMultipart: boolean;
+    partCount: number;
+    ingestionRunId: string;
+    creator?: string | null;
+    previewData?: Buffer | null;
+    previewMsgId?: bigint | null;
+    files: {
+        path: string;
+        fileName: string;
+        extension: string | null;
+        compressedSize: bigint;
+        uncompressedSize: bigint;
+        crc32: string | null;
+    }[];
+}
+export declare function createPackageWithFiles(input: CreatePackageInput): Promise<{
+    id: string;
+    createdAt: Date;
+    contentHash: string;
+    fileName: string;
+    fileSize: bigint;
+    archiveType: import("@prisma/client").$Enums.ArchiveType;
+    creator: string | null;
+    sourceChannelId: string;
+    sourceMessageId: bigint;
+    sourceTopicId: bigint | null;
+    destChannelId: string | null;
+    destMessageId: bigint | null;
+    isMultipart: boolean;
+    partCount: number;
+    fileCount: number;
+    previewData: import("@prisma/client/runtime/client").Bytes | null;
+    previewMsgId: bigint | null;
+    indexedAt: Date;
+    ingestionRunId: string | null;
+}>;
+export declare function createIngestionRun(accountId: string): Promise<{
+    accountId: string;
+    id: string;
+    status: import("@prisma/client").$Enums.IngestionStatus;
+    startedAt: Date;
+    finishedAt: Date | null;
+    messagesScanned: number;
+    zipsFound: number;
+    zipsDuplicate: number;
+    zipsIngested: number;
+    errorMessage: string | null;
+    currentActivity: string | null;
+    currentStep: string | null;
+    currentChannel: string | null;
+    currentFile: string | null;
+    currentFileNum: number | null;
+    totalFiles: number | null;
+    downloadedBytes: bigint | null;
+    totalBytes: bigint | null;
+    downloadPercent: number | null;
+    lastActivityAt: Date | null;
+}>;
+export interface ActivityUpdate {
+    currentActivity: string;
+    currentStep: string;
+    currentChannel?: string | null;
+    currentFile?: string | null;
+    currentFileNum?: number | null;
+    totalFiles?: number | null;
+    downloadedBytes?: bigint | null;
+    totalBytes?: bigint | null;
+    downloadPercent?: number | null;
+    messagesScanned?: number;
+    zipsFound?: number;
+    zipsDuplicate?: number;
+    zipsIngested?: number;
+}
+export declare function updateRunActivity(runId: string, activity: ActivityUpdate): Promise<{
+    accountId: string;
+    id: string;
+    status: import("@prisma/client").$Enums.IngestionStatus;
+    startedAt: Date;
+    finishedAt: Date | null;
+    messagesScanned: number;
+    zipsFound: number;
+    zipsDuplicate: number;
+    zipsIngested: number;
+    errorMessage: string | null;
+    currentActivity: string | null;
+    currentStep: string | null;
+    currentChannel: string | null;
+    currentFile: string | null;
+    currentFileNum: number | null;
+    totalFiles: number | null;
+    downloadedBytes: bigint | null;
+    totalBytes: bigint | null;
+    downloadPercent: number | null;
+    lastActivityAt: Date | null;
+}>;
+export declare function completeIngestionRun(runId: string, counters: {
+    messagesScanned: number;
+    zipsFound: number;
+    zipsDuplicate: number;
+    zipsIngested: number;
+}): Promise<{
+    accountId: string;
+    id: string;
+    status: import("@prisma/client").$Enums.IngestionStatus;
+    startedAt: Date;
+    finishedAt: Date | null;
+    messagesScanned: number;
+    zipsFound: number;
+    zipsDuplicate: number;
+    zipsIngested: number;
+    errorMessage: string | null;
+    currentActivity: string | null;
+    currentStep: string | null;
+    currentChannel: string | null;
+    currentFile: string | null;
+    currentFileNum: number | null;
+    totalFiles: number | null;
+    downloadedBytes: bigint | null;
+    totalBytes: bigint | null;
+    downloadPercent: number | null;
+    lastActivityAt: Date | null;
+}>;
+export declare function failIngestionRun(runId: string, errorMessage: string): Promise<{
+    accountId: string;
+    id: string;
+    status: import("@prisma/client").$Enums.IngestionStatus;
+    startedAt: Date;
+    finishedAt: Date | null;
+    messagesScanned: number;
+    zipsFound: number;
+    zipsDuplicate: number;
+    zipsIngested: number;
+    errorMessage: string | null;
+    currentActivity: string | null;
+    currentStep: string | null;
+    currentChannel: string | null;
+    currentFile: string | null;
+    currentFileNum: number | null;
+    totalFiles: number | null;
+    downloadedBytes: bigint | null;
+    totalBytes: bigint | null;
+    downloadPercent: number | null;
+    lastActivityAt: Date | null;
+}>;
+export declare function updateLastProcessedMessage(mappingId: string, messageId: bigint): Promise<{
+    accountId: string;
+    id: string;
+    createdAt: Date;
+    channelId: string;
+    role: import("@prisma/client").$Enums.ChannelRole;
+    lastProcessedMessageId: bigint | null;
+}>;
+export declare function markStaleRunsAsFailed(): Promise<import("@prisma/client").Prisma.BatchPayload>;
+export declare function updateAccountAuthState(accountId: string, authState: "PENDING" | "AWAITING_CODE" | "AWAITING_PASSWORD" | "AUTHENTICATED" | "EXPIRED", authCode?: string | null): Promise<{
+    id: string;
+    phone: string;
+    displayName: string | null;
+    isActive: boolean;
+    authState: import("@prisma/client").$Enums.AuthState;
+    authCode: string | null;
+    lastSeenAt: Date | null;
+    createdAt: Date;
+    updatedAt: Date;
+}>;
+export declare function getAccountAuthCode(accountId: string): Promise<{
+    authState: import("@prisma/client").$Enums.AuthState;
+    authCode: string | null;
+} | null>;
+export interface UpsertChannelInput {
+    telegramId: bigint;
+    title: string;
+    type: "SOURCE" | "DESTINATION";
+    isForum: boolean;
+    isActive?: boolean;
+}
+/**
+ * Upsert a channel by telegramId. Returns the channel record.
+ * If it already exists, update title and forum status.
+ * New channels default to disabled (isActive: false) so the admin must
+ * explicitly enable them before the worker processes them.
+ * Pass isActive: true for DESTINATION channels that must be active immediately.
+ */
+export declare function upsertChannel(input: UpsertChannelInput): Promise<{
+    id: string;
+    isActive: boolean;
+    createdAt: Date;
+    updatedAt: Date;
+    telegramId: bigint;
+    title: string;
+    type: import("@prisma/client").$Enums.ChannelType;
+    isForum: boolean;
+}>;
+/**
+ * Link an account to a channel if not already linked.
+ * Uses a try/catch on unique constraint to make it idempotent.
+ */
+export declare function ensureAccountChannelLink(accountId: string, channelId: string, role: "READER" | "WRITER"): Promise<{
+    accountId: string;
+    id: string;
+    createdAt: Date;
+    channelId: string;
+    role: import("@prisma/client").$Enums.ChannelRole;
+    lastProcessedMessageId: bigint | null;
+} | null>;
+export declare function setChannelForum(channelId: string, isForum: boolean): Promise<{
+    id: string;
+    isActive: boolean;
+    createdAt: Date;
+    updatedAt: Date;
+    telegramId: bigint;
+    title: string;
+    type: import("@prisma/client").$Enums.ChannelType;
+    isForum: boolean;
+}>;
+export declare function getTopicProgress(mappingId: string): Promise<{
+    id: string;
+    lastProcessedMessageId: bigint | null;
+    accountChannelMapId: string;
+    topicId: bigint;
+    topicName: string | null;
+}[]>;
+export declare function upsertTopicProgress(mappingId: string, topicId: bigint, topicName: string | null, lastProcessedMessageId: bigint): Promise<{
+    id: string;
+    lastProcessedMessageId: bigint | null;
+    accountChannelMapId: string;
+    topicId: bigint;
+    topicName: string | null;
+}>;
+export declare function getChannelFetchRequest(requestId: string): Promise<({
+    account: {
+        id: string;
+        phone: string;
+        displayName: string | null;
+        isActive: boolean;
+        authState: import("@prisma/client").$Enums.AuthState;
+        authCode: string | null;
+        lastSeenAt: Date | null;
+        createdAt: Date;
+        updatedAt: Date;
+    };
+} & {
+    error: string | null;
+    accountId: string;
+    id: string;
+    createdAt: Date;
+    updatedAt: Date;
+    status: import("@prisma/client").$Enums.FetchStatus;
+    resultJson: string | null;
+}) | null>;
+export declare function updateFetchRequestStatus(requestId: string, status: FetchStatus, extra?: {
+    resultJson?: string;
+    error?: string;
+}): Promise<{
+    error: string | null;
+    accountId: string;
+    id: string;
+    createdAt: Date;
+    updatedAt: Date;
+    status: import("@prisma/client").$Enums.FetchStatus;
+    resultJson: string | null;
+}>;
+export declare function getAccountLinkedChannelIds(accountId: string): Promise<Set<string>>;
+export declare function getExistingChannelsByTelegramId(): Promise<Map<string, string>>;
+export declare function getAccountById(accountId: string): Promise<{
+    id: string;
+    phone: string;
+    displayName: string | null;
+    isActive: boolean;
+    authState: import("@prisma/client").$Enums.AuthState;
+    authCode: string | null;
+    lastSeenAt: Date | null;
+    createdAt: Date;
+    updatedAt: Date;
+} | null>;

worker/dist/db/queries.js

@@ -0,0 +1,319 @@
+import { db } from "./client.js";
+export async function getActiveAccounts() {
+    return db.telegramAccount.findMany({
+        where: { isActive: true, authState: "AUTHENTICATED" },
+    });
+}
+export async function getPendingAccounts() {
+    return db.telegramAccount.findMany({
+        where: { isActive: true, authState: "PENDING" },
+    });
+}
+export async function hasAnyChannels() {
+    const count = await db.telegramChannel.count();
+    return count > 0;
+}
+export async function getSourceChannelMappings(accountId) {
+    return db.accountChannelMap.findMany({
+        where: {
+            accountId,
+            role: "READER",
+            channel: { type: "SOURCE", isActive: true },
+        },
+        include: { channel: true },
+    });
+}
+// ── Global destination channel ──
+export async function getGlobalDestinationChannel() {
+    const setting = await db.globalSetting.findUnique({
+        where: { key: "destination_channel_id" },
+    });
+    if (!setting)
+        return null;
+    return db.telegramChannel.findFirst({
+        where: { id: setting.value, type: "DESTINATION", isActive: true },
+    });
+}
+export async function getGlobalSetting(key) {
+    const setting = await db.globalSetting.findUnique({ where: { key } });
+    return setting?.value ?? null;
+}
+export async function setGlobalSetting(key, value) {
+    return db.globalSetting.upsert({
+        where: { key },
+        create: { key, value },
+        update: { value },
+    });
+}
+export async function packageExistsByHash(contentHash) {
+    const pkg = await db.package.findFirst({
+        where: { contentHash, destMessageId: { not: null } },
+        select: { id: true },
+    });
+    return pkg !== null;
+}
+/**
+ * Check if a package already exists for a given source message ID
+ * AND was successfully uploaded to the destination (destMessageId is set).
+ * Used as an early skip before downloading.
+ */
+export async function packageExistsBySourceMessage(sourceChannelId, sourceMessageId) {
+    const pkg = await db.package.findFirst({
+        where: { sourceChannelId, sourceMessageId, destMessageId: { not: null } },
+        select: { id: true },
+    });
+    return pkg !== null;
+}
+/**
+ * Delete orphaned Package rows that have the same content hash but never
+ * completed the upload (destMessageId is null). Called before creating a
+ * new complete record to avoid unique constraint violations.
+ */
+export async function deleteOrphanedPackageByHash(contentHash) {
+    await db.package.deleteMany({
+        where: { contentHash, destMessageId: null },
+    });
+}
+export async function createPackageWithFiles(input) {
+    const pkg = await db.package.create({
+        data: {
+            contentHash: input.contentHash,
+            fileName: input.fileName,
+            fileSize: input.fileSize,
+            archiveType: input.archiveType,
+            sourceChannelId: input.sourceChannelId,
+            sourceMessageId: input.sourceMessageId,
+            sourceTopicId: input.sourceTopicId ?? undefined,
+            destChannelId: input.destChannelId,
+            destMessageId: input.destMessageId,
+            isMultipart: input.isMultipart,
+            partCount: input.partCount,
+            fileCount: input.files.length,
+            ingestionRunId: input.ingestionRunId,
+            creator: input.creator ?? undefined,
+            previewData: input.previewData ? new Uint8Array(input.previewData) : undefined,
+            previewMsgId: input.previewMsgId ?? undefined,
+            files: {
+                create: input.files,
+            },
+        },
+    });
+    // Notify the bot service about the new package (for subscription alerts)
+    try {
+        await db.$queryRawUnsafe(`SELECT pg_notify('new_package', $1)`, JSON.stringify({
+            packageId: pkg.id,
+            fileName: input.fileName,
+            creator: input.creator ?? null,
+        }));
+    }
+    catch {
+        // Best-effort — don't fail the ingestion if notification fails
+    }
+    return pkg;
+}
+export async function createIngestionRun(accountId) {
+    return db.ingestionRun.create({
+        data: {
+            accountId,
+            status: "RUNNING",
+            currentActivity: "Starting ingestion run",
+            currentStep: "initializing",
+            lastActivityAt: new Date(),
+        },
+    });
+}
+export async function updateRunActivity(runId, activity) {
+    return db.ingestionRun.update({
+        where: { id: runId },
+        data: {
+            currentActivity: activity.currentActivity,
+            currentStep: activity.currentStep,
+            currentChannel: activity.currentChannel ?? undefined,
+            currentFile: activity.currentFile ?? undefined,
+            currentFileNum: activity.currentFileNum ?? undefined,
+            totalFiles: activity.totalFiles ?? undefined,
+            downloadedBytes: activity.downloadedBytes ?? undefined,
+            totalBytes: activity.totalBytes ?? undefined,
+            downloadPercent: activity.downloadPercent ?? undefined,
+            lastActivityAt: new Date(),
+            ...(activity.messagesScanned !== undefined && { messagesScanned: activity.messagesScanned }),
+            ...(activity.zipsFound !== undefined && { zipsFound: activity.zipsFound }),
+            ...(activity.zipsDuplicate !== undefined && { zipsDuplicate: activity.zipsDuplicate }),
+            ...(activity.zipsIngested !== undefined && { zipsIngested: activity.zipsIngested }),
+        },
+    });
+}
+const CLEAR_ACTIVITY = {
+    currentActivity: null,
+    currentStep: null,
+    currentChannel: null,
+    currentFile: null,
+    currentFileNum: null,
+    totalFiles: null,
+    downloadedBytes: null,
+    totalBytes: null,
+    downloadPercent: null,
+    lastActivityAt: new Date(),
+};
+export async function completeIngestionRun(runId, counters) {
+    return db.ingestionRun.update({
+        where: { id: runId },
+        data: {
+            status: "COMPLETED",
+            finishedAt: new Date(),
+            ...counters,
+            ...CLEAR_ACTIVITY,
+        },
+    });
+}
+export async function failIngestionRun(runId, errorMessage) {
+    return db.ingestionRun.update({
+        where: { id: runId },
+        data: {
+            status: "FAILED",
+            finishedAt: new Date(),
+            errorMessage,
+            ...CLEAR_ACTIVITY,
+        },
+    });
+}
+export async function updateLastProcessedMessage(mappingId, messageId) {
+    return db.accountChannelMap.update({
+        where: { id: mappingId },
+        data: { lastProcessedMessageId: messageId },
+    });
+}
+export async function markStaleRunsAsFailed() {
+    return db.ingestionRun.updateMany({
+        where: { status: "RUNNING" },
+        data: {
+            status: "FAILED",
+            finishedAt: new Date(),
+            errorMessage: "Worker restarted — run was still marked as RUNNING",
+        },
+    });
+}
+export async function updateAccountAuthState(accountId, authState, authCode) {
+    return db.telegramAccount.update({
+        where: { id: accountId },
+        data: { authState, authCode, lastSeenAt: authState === "AUTHENTICATED" ? new Date() : undefined },
+    });
+}
+export async function getAccountAuthCode(accountId) {
+    const account = await db.telegramAccount.findUnique({
+        where: { id: accountId },
+        select: { authCode: true, authState: true },
+    });
+    return account;
+}
+/**
+ * Upsert a channel by telegramId. Returns the channel record.
+ * If it already exists, update title and forum status.
+ * New channels default to disabled (isActive: false) so the admin must
+ * explicitly enable them before the worker processes them.
+ * Pass isActive: true for DESTINATION channels that must be active immediately.
+ */
+export async function upsertChannel(input) {
+    return db.telegramChannel.upsert({
+        where: { telegramId: input.telegramId },
+        create: {
+            telegramId: input.telegramId,
+            title: input.title,
+            type: input.type,
+            isForum: input.isForum,
+            isActive: input.isActive ?? false,
+        },
+        update: {
+            title: input.title,
+            isForum: input.isForum,
+        },
+    });
+}
+/**
+ * Link an account to a channel if not already linked.
+ * Uses a try/catch on unique constraint to make it idempotent.
+ */
+export async function ensureAccountChannelLink(accountId, channelId, role) {
+    try {
+        return await db.accountChannelMap.create({
+            data: { accountId, channelId, role },
+        });
+    }
+    catch (err) {
+        // Already linked — ignore unique constraint violation
+        if (err instanceof Error && err.message.includes("Unique constraint")) {
+            return null;
+        }
+        throw err;
+    }
+}
+// ── Forum / Topic progress ──
+export async function setChannelForum(channelId, isForum) {
+    return db.telegramChannel.update({
+        where: { id: channelId },
+        data: { isForum },
+    });
+}
+export async function getTopicProgress(mappingId) {
+    return db.topicProgress.findMany({
+        where: { accountChannelMapId: mappingId },
+    });
+}
+export async function upsertTopicProgress(mappingId, topicId, topicName, lastProcessedMessageId) {
+    return db.topicProgress.upsert({
+        where: {
+            accountChannelMapId_topicId: {
+                accountChannelMapId: mappingId,
+                topicId,
+            },
+        },
+        create: {
+            accountChannelMapId: mappingId,
+            topicId,
+            topicName,
+            lastProcessedMessageId,
+        },
+        update: {
+            topicName,
+            lastProcessedMessageId,
+        },
+    });
+}
+// ── Channel fetch requests (DB-mediated communication with web app) ──
+export async function getChannelFetchRequest(requestId) {
+    return db.channelFetchRequest.findUnique({
+        where: { id: requestId },
+        include: { account: true },
+    });
+}
+export async function updateFetchRequestStatus(requestId, status, extra) {
+    return db.channelFetchRequest.update({
+        where: { id: requestId },
+        data: {
+            status,
+            resultJson: extra?.resultJson ?? undefined,
+            error: extra?.error ?? undefined,
+        },
+    });
+}
+export async function getAccountLinkedChannelIds(accountId) {
+    const links = await db.accountChannelMap.findMany({
+        where: { accountId },
+        select: { channel: { select: { telegramId: true } } },
+    });
+    return new Set(links.map((l) => l.channel.telegramId.toString()));
+}
+export async function getExistingChannelsByTelegramId() {
+    const channels = await db.telegramChannel.findMany({
+        select: { id: true, telegramId: true },
+    });
+    const map = new Map();
+    for (const ch of channels) {
+        map.set(ch.telegramId.toString(), ch.id);
+    }
+    return map;
+}
+export async function getAccountById(accountId) {
+    return db.telegramAccount.findUnique({ where: { id: accountId } });
+}
+//# sourceMappingURL=queries.js.map

worker/dist/fetch-listener.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Start listening for pg_notify signals from the web app.
+ *
+ * Channels:
+ *   - `channel_fetch` — payload = requestId → fetch channels for an account
+ *   - `generate_invite` — payload = channelId → generate invite link for destination
+ *   - `create_destination` — payload = JSON { requestId, title } → create supergroup via TDLib
+ *   - `ingestion_trigger` — trigger an immediate ingestion cycle
+ */
+export declare function startFetchListener(): Promise<void>;
+export declare function stopFetchListener(): void;

worker/dist/fetch-listener.js

@@ -0,0 +1,195 @@
+import { pool } from "./db/client.js";
+import { childLogger } from "./util/logger.js";
+import { withTdlibMutex } from "./util/mutex.js";
+import { processFetchRequest } from "./worker.js";
+import { generateInviteLink, createSupergroup } from "./tdlib/chats.js";
+import { createTdlibClient, closeTdlibClient } from "./tdlib/client.js";
+import { triggerImmediateCycle } from "./scheduler.js";
+import { getGlobalDestinationChannel, setGlobalSetting, getActiveAccounts, upsertChannel, ensureAccountChannelLink, } from "./db/queries.js";
+const log = childLogger("fetch-listener");
+let pgClient = null;
+/**
+ * Start listening for pg_notify signals from the web app.
+ *
+ * Channels:
+ *   - `channel_fetch` — payload = requestId → fetch channels for an account
+ *   - `generate_invite` — payload = channelId → generate invite link for destination
+ *   - `create_destination` — payload = JSON { requestId, title } → create supergroup via TDLib
+ *   - `ingestion_trigger` — trigger an immediate ingestion cycle
+ */
+export async function startFetchListener() {
+    pgClient = await pool.connect();
+    await pgClient.query("LISTEN channel_fetch");
+    await pgClient.query("LISTEN generate_invite");
+    await pgClient.query("LISTEN create_destination");
+    await pgClient.query("LISTEN ingestion_trigger");
+    pgClient.on("notification", (msg) => {
+        if (msg.channel === "channel_fetch" && msg.payload) {
+            handleChannelFetch(msg.payload);
+        }
+        else if (msg.channel === "generate_invite" && msg.payload) {
+            handleGenerateInvite(msg.payload);
+        }
+        else if (msg.channel === "create_destination" && msg.payload) {
+            handleCreateDestination(msg.payload);
+        }
+        else if (msg.channel === "ingestion_trigger") {
+            handleIngestionTrigger();
+        }
+    });
+    log.info("Fetch listener started (channel_fetch, generate_invite, create_destination, ingestion_trigger)");
+}
+export function stopFetchListener() {
+    if (pgClient) {
+        pgClient.release();
+        pgClient = null;
+    }
+    log.info("Fetch listener stopped");
+}
+// ── Channel fetch handler ──
+// Chain promises to ensure sequential execution
+let fetchQueue = Promise.resolve();
+function handleChannelFetch(requestId) {
+    fetchQueue = fetchQueue.then(async () => {
+        try {
+            await withTdlibMutex("fetch-channels", () => processFetchRequest(requestId));
+        }
+        catch (err) {
+            log.error({ err, requestId }, "Failed to process fetch request");
+        }
+    });
+}
+// ── Invite link generation handler ──
+function handleGenerateInvite(channelId) {
+    fetchQueue = fetchQueue.then(async () => {
+        try {
+            await withTdlibMutex("generate-invite", async () => {
+                const destChannel = await getGlobalDestinationChannel();
+                if (!destChannel || destChannel.id !== channelId) {
+                    log.warn({ channelId }, "Destination channel mismatch, skipping invite generation");
+                    return;
+                }
+                // Use the first available authenticated account to generate the link
+                const accounts = await getActiveAccounts();
+                if (accounts.length === 0) {
+                    log.warn("No authenticated accounts to generate invite link");
+                    return;
+                }
+                const account = accounts[0];
+                const client = await createTdlibClient({ id: account.id, phone: account.phone });
+                try {
+                    const link = await generateInviteLink(client, destChannel.telegramId);
+                    await setGlobalSetting("destination_invite_link", link);
+                    log.info({ link }, "Invite link generated and saved");
+                }
+                finally {
+                    await closeTdlibClient(client);
+                }
+            });
+        }
+        catch (err) {
+            log.error({ err, channelId }, "Failed to generate invite link");
+        }
+    });
+}
+// ── Create destination supergroup handler ──
+function handleCreateDestination(payload) {
+    fetchQueue = fetchQueue.then(async () => {
+        let requestId;
+        try {
+            const parsed = JSON.parse(payload);
+            requestId = parsed.requestId;
+            await withTdlibMutex("create-destination", async () => {
+                const { db } = await import("./db/client.js");
+                // Mark the request as in-progress
+                await db.channelFetchRequest.update({
+                    where: { id: parsed.requestId },
+                    data: { status: "IN_PROGRESS" },
+                });
+                // Use the first available authenticated account
+                const accounts = await getActiveAccounts();
+                if (accounts.length === 0) {
+                    throw new Error("No authenticated accounts available to create the group");
+                }
+                const account = accounts[0];
+                const client = await createTdlibClient({ id: account.id, phone: account.phone });
+                try {
+                    // Create the supergroup via TDLib
+                    const result = await createSupergroup(client, parsed.title);
+                    log.info({ chatId: result.chatId.toString(), title: result.title }, "Supergroup created");
+                    // Upsert it as a DESTINATION channel in the DB (active by default)
+                    const channel = await upsertChannel({
+                        telegramId: result.chatId,
+                        title: result.title,
+                        type: "DESTINATION",
+                        isForum: false,
+                        isActive: true,
+                    });
+                    // Set as global destination
+                    await setGlobalSetting("destination_channel_id", channel.id);
+                    // Generate an invite link
+                    const link = await generateInviteLink(client, result.chatId);
+                    await setGlobalSetting("destination_invite_link", link);
+                    log.info({ link }, "Invite link generated for new destination");
+                    // Link all authenticated accounts as WRITER
+                    for (const acc of accounts) {
+                        try {
+                            await ensureAccountChannelLink(acc.id, channel.id, "WRITER");
+                        }
+                        catch {
+                            // Already linked
+                        }
+                    }
+                    // Mark fetch request as completed with the channel info
+                    await db.channelFetchRequest.update({
+                        where: { id: parsed.requestId },
+                        data: {
+                            status: "COMPLETED",
+                            resultJson: JSON.stringify({
+                                channelId: channel.id,
+                                telegramId: result.chatId.toString(),
+                                title: result.title,
+                                inviteLink: link,
+                            }),
+                        },
+                    });
+                    log.info({ channelId: channel.id, telegramId: result.chatId.toString() }, "Destination channel created and configured");
+                }
+                finally {
+                    await closeTdlibClient(client);
+                }
+            });
+        }
+        catch (err) {
+            log.error({ err, payload }, "Failed to create destination channel");
+            if (requestId) {
+                try {
+                    const { db } = await import("./db/client.js");
+                    await db.channelFetchRequest.update({
+                        where: { id: requestId },
+                        data: {
+                            status: "FAILED",
+                            error: err instanceof Error ? err.message : String(err),
+                        },
+                    });
+                }
+                catch {
+                    // Best-effort
+                }
+            }
+        }
+    });
+}
+// ── Ingestion trigger handler ──
+function handleIngestionTrigger() {
+    fetchQueue = fetchQueue.then(async () => {
+        try {
+            log.info("Ingestion trigger received from UI");
+            await triggerImmediateCycle();
+        }
+        catch (err) {
+            log.error({ err }, "Failed to trigger immediate ingestion cycle");
+        }
+    });
+}
+//# sourceMappingURL=fetch-listener.js.map

worker/dist/index.d.ts

@@ -0,0 +1 @@
+export {};

worker/dist/index.js

@@ -0,0 +1,50 @@
+import { mkdir } from "fs/promises";
+import { config } from "./util/config.js";
+import { logger } from "./util/logger.js";
+import { markStaleRunsAsFailed } from "./db/queries.js";
+import { cleanupTempDir } from "./worker.js";
+import { startScheduler, stopScheduler } from "./scheduler.js";
+import { startFetchListener, stopFetchListener } from "./fetch-listener.js";
+import { db, pool } from "./db/client.js";
+const log = logger.child({ module: "main" });
+async function main() {
+    log.info("DragonsStash Telegram Worker starting");
+    log.info({ config: { ...config, databaseUrl: "***" } }, "Configuration loaded");
+    if (!config.telegramApiId || !config.telegramApiHash) {
+        log.fatal("TELEGRAM_API_ID and TELEGRAM_API_HASH are both required");
+        process.exit(1);
+    }
+    // Ensure temp directory exists
+    await mkdir(config.tempDir, { recursive: true });
+    await mkdir(config.tdlibStateDir, { recursive: true });
+    // Clean up stale state
+    await cleanupTempDir();
+    await markStaleRunsAsFailed();
+    // Start the fetch listener (pg_notify for on-demand channel fetching)
+    await startFetchListener();
+    // Start the scheduler
+    await startScheduler();
+}
+// Graceful shutdown
+function shutdown(signal) {
+    log.info({ signal }, "Shutdown signal received");
+    stopScheduler();
+    stopFetchListener();
+    // Close DB connections
+    Promise.all([db.$disconnect(), pool.end()])
+        .then(() => {
+        log.info("Shutdown complete");
+        process.exit(0);
+    })
+        .catch((err) => {
+        log.error({ err }, "Error during shutdown");
+        process.exit(1);
+    });
+}
+process.on("SIGTERM", () => shutdown("SIGTERM"));
+process.on("SIGINT", () => shutdown("SIGINT"));
+main().catch((err) => {
+    log.fatal({ err }, "Worker failed to start");
+    process.exit(1);
+});
+//# sourceMappingURL=index.js.map

worker/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,OAAO,EAAE,qBAAqB,EAAE,MAAM,iBAAiB,CAAC;AACxD,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/D,OAAO,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAC5E,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAC;AAE1C,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;AAE7C,KAAK,UAAU,IAAI;IACjB,GAAG,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;IAClD,GAAG,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,GAAG,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,EAAE,EAAE,sBAAsB,CAAC,CAAC;IAEhF,IAAI,CAAC,MAAM,CAAC,aAAa,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC;QACrD,GAAG,CAAC,KAAK,CAAC,yDAAyD,CAAC,CAAC;QACrE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,+BAA+B;IAC/B,MAAM,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACjD,MAAM,KAAK,CAAC,MAAM,CAAC,aAAa,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAEvD,uBAAuB;IACvB,MAAM,cAAc,EAAE,CAAC;IACvB,MAAM,qBAAqB,EAAE,CAAC;IAE9B,sEAAsE;IACtE,MAAM,kBAAkB,EAAE,CAAC;IAE3B,sBAAsB;IACtB,MAAM,cAAc,EAAE,CAAC;AACzB,CAAC;AAED,oBAAoB;AACpB,SAAS,QAAQ,CAAC,MAAc;IAC9B,GAAG,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,0BAA0B,CAAC,CAAC;IACjD,aAAa,EAAE,CAAC;IAChB,iBAAiB,EAAE,CAAC;IAEpB,uBAAuB;IACvB,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,WAAW,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;SACxC,IAAI,CAAC,GAAG,EAAE;QACT,GAAG,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;QAC9B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC;SACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;QACb,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,EAAE,uBAAuB,CAAC,CAAC;QAC5C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;AACP,CAAC;AAED,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;AACjD,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;AAE/C,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IACnB,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,EAAE,wBAAwB,CAAC,CAAC;IAC7C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}

worker/dist/preview/match.d.ts

@@ -0,0 +1,22 @@
+export interface TelegramPhoto {
+    id: bigint;
+    date: Date;
+    /** Caption text on the photo message (if any). */
+    caption: string;
+    /** The smallest photo size available — used as thumbnail. */
+    fileId: string;
+    fileSize: number;
+}
+export interface ArchiveRef {
+    baseName: string;
+    firstMessageId: bigint;
+    firstMessageDate: Date;
+}
+/**
+ * Try to match a photo message to an archive by:
+ * 1. Caption contains the archive baseName (without extension)
+ * 2. Photo was posted within ±10 messages (time-window: ±6 hours)
+ *
+ * Returns the best match (closest in time), or null.
+ */
+export declare function matchPreviewToArchive(photos: TelegramPhoto[], archives: ArchiveRef[]): Map<string, TelegramPhoto>;

worker/dist/preview/match.js

@@ -0,0 +1,53 @@
+import { childLogger } from "../util/logger.js";
+const log = childLogger("preview-match");
+/**
+ * Try to match a photo message to an archive by:
+ * 1. Caption contains the archive baseName (without extension)
+ * 2. Photo was posted within ±10 messages (time-window: ±6 hours)
+ *
+ * Returns the best match (closest in time), or null.
+ */
+export function matchPreviewToArchive(photos, archives) {
+    const results = new Map();
+    const TIME_WINDOW_MS = 6 * 60 * 60 * 1000; // 6 hours
+    for (const archive of archives) {
+        // Normalize the archive base name for matching
+        const normalizedBase = normalizeForMatch(archive.baseName);
+        if (!normalizedBase)
+            continue;
+        let bestMatch = null;
+        let bestTimeDiff = Infinity;
+        for (const photo of photos) {
+            const timeDiff = Math.abs(photo.date.getTime() - archive.firstMessageDate.getTime());
+            // Must be within time window
+            if (timeDiff > TIME_WINDOW_MS)
+                continue;
+            // Check if the photo caption contains the archive base name
+            const normalizedCaption = normalizeForMatch(photo.caption);
+            if (!normalizedCaption)
+                continue;
+            const matches = normalizedCaption.includes(normalizedBase) ||
+                normalizedBase.includes(normalizedCaption);
+            if (matches && timeDiff < bestTimeDiff) {
+                bestMatch = photo;
+                bestTimeDiff = timeDiff;
+            }
+        }
+        if (bestMatch) {
+            log.debug({ baseName: archive.baseName, photoId: bestMatch.id.toString() }, "Matched preview photo to archive");
+            results.set(archive.baseName, bestMatch);
+        }
+    }
+    return results;
+}
+/**
+ * Strip extension, punctuation, and normalize for fuzzy matching.
+ */
+function normalizeForMatch(input) {
+    return input
+        .toLowerCase()
+        .replace(/\.[a-z0-9]{1,5}$/i, "") // strip extension
+        .replace(/[_\-.\s]+/g, " ") // normalize separators
+        .trim();
+}
+//# sourceMappingURL=match.js.map

worker/dist/preview/match.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"match.js","sourceRoot":"","sources":["../../src/preview/match.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,MAAM,GAAG,GAAG,WAAW,CAAC,eAAe,CAAC,CAAC;AAkBzC;;;;;;GAMG;AACH,MAAM,UAAU,qBAAqB,CACnC,MAAuB,EACvB,QAAsB;IAEtB,MAAM,OAAO,GAAG,IAAI,GAAG,EAAyB,CAAC;IACjD,MAAM,cAAc,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,UAAU;IAErD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,+CAA+C;QAC/C,MAAM,cAAc,GAAG,iBAAiB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC3D,IAAI,CAAC,cAAc;YAAE,SAAS;QAE9B,IAAI,SAAS,GAAyB,IAAI,CAAC;QAC3C,IAAI,YAAY,GAAG,QAAQ,CAAC;QAE5B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CACvB,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,OAAO,CAAC,gBAAgB,CAAC,OAAO,EAAE,CAC1D,CAAC;YAEF,6BAA6B;YAC7B,IAAI,QAAQ,GAAG,cAAc;gBAAE,SAAS;YAExC,4DAA4D;YAC5D,MAAM,iBAAiB,GAAG,iBAAiB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAC3D,IAAI,CAAC,iBAAiB;gBAAE,SAAS;YAEjC,MAAM,OAAO,GACX,iBAAiB,CAAC,QAAQ,CAAC,cAAc,CAAC;gBAC1C,cAAc,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC;YAE7C,IAAI,OAAO,IAAI,QAAQ,GAAG,YAAY,EAAE,CAAC;gBACvC,SAAS,GAAG,KAAK,CAAC;gBAClB,YAAY,GAAG,QAAQ,CAAC;YAC1B,CAAC;QACH,CAAC;QAED,IAAI,SAAS,EAAE,CAAC;YACd,GAAG,CAAC,KAAK,CACP,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,OAAO,EAAE,SAAS,CAAC,EAAE,CAAC,QAAQ,EAAE,EAAE,EAChE,kCAAkC,CACnC,CAAC;YACF,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;QAC3C,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB,CAAC,KAAa;IACtC,OAAO,KAAK;SACT,WAAW,EAAE;SACb,OAAO,CAAC,mBAAmB,EAAE,EAAE,CAAC,CAAC,kBAAkB;SACnD,OAAO,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC,uBAAuB;SAClD,IAAI,EAAE,CAAC;AACZ,CAAC"}

worker/dist/scheduler.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Start the scheduler. Runs an immediate first cycle, then schedules subsequent ones.
+ */
+export declare function startScheduler(): Promise<void>;
+/**
+ * Trigger an immediate ingestion cycle (e.g. from the admin UI).
+ * If a cycle is already running, this is a no-op.
+ */
+export declare function triggerImmediateCycle(): Promise<void>;
+/**
+ * Stop the scheduler gracefully.
+ */
+export declare function stopScheduler(): void;

worker/dist/scheduler.js

@@ -0,0 +1,121 @@
+import { config } from "./util/config.js";
+import { childLogger } from "./util/logger.js";
+import { withTdlibMutex } from "./util/mutex.js";
+import { getActiveAccounts, getPendingAccounts } from "./db/queries.js";
+import { runWorkerForAccount, authenticateAccount } from "./worker.js";
+const log = childLogger("scheduler");
+let running = false;
+let timer = null;
+let cycleCount = 0;
+/**
+ * Maximum time for a single ingestion cycle (ms).
+ * After this, new accounts won't be started (in-progress work finishes).
+ * Default: 4 hours. Configurable via WORKER_CYCLE_TIMEOUT_MINUTES.
+ */
+const CYCLE_TIMEOUT_MS = (parseInt(process.env.WORKER_CYCLE_TIMEOUT_MINUTES ?? "240", 10)) * 60 * 1000;
+/**
+ * Run one ingestion cycle:
+ * 1. Authenticate any PENDING accounts (triggers SMS code flow + auto-fetch channels)
+ * 2. Process all active AUTHENTICATED accounts for ingestion
+ *
+ * All TDLib operations are wrapped in the mutex to ensure only one client
+ * runs at a time (also shared with the fetch listener for on-demand requests).
+ *
+ * The cycle has a configurable timeout (WORKER_CYCLE_TIMEOUT_MINUTES, default 4h).
+ * Once the timeout elapses, no new accounts will be started but any in-progress
+ * account processing is allowed to finish its current archive set.
+ */
+async function runCycle() {
+    if (running) {
+        log.warn("Previous cycle still running, skipping");
+        return;
+    }
+    running = true;
+    cycleCount++;
+    const cycleStart = Date.now();
+    log.info({ cycle: cycleCount, timeoutMinutes: CYCLE_TIMEOUT_MS / 60_000 }, "Starting ingestion cycle");
+    try {
+        // ── Phase 1: Authenticate pending accounts ──
+        const pendingAccounts = await getPendingAccounts();
+        if (pendingAccounts.length > 0) {
+            log.info({ count: pendingAccounts.length }, "Found pending accounts, starting authentication");
+            for (const account of pendingAccounts) {
+                if (Date.now() - cycleStart > CYCLE_TIMEOUT_MS) {
+                    log.warn("Cycle timeout reached during authentication phase, stopping");
+                    break;
+                }
+                await withTdlibMutex(`auth:${account.phone}`, () => authenticateAccount(account));
+            }
+        }
+        // ── Phase 2: Ingest for authenticated accounts ──
+        const accounts = await getActiveAccounts();
+        if (accounts.length === 0) {
+            log.info("No active authenticated accounts, nothing to ingest");
+            return;
+        }
+        log.info({ accountCount: accounts.length }, "Processing accounts");
+        for (const account of accounts) {
+            if (Date.now() - cycleStart > CYCLE_TIMEOUT_MS) {
+                log.warn({ elapsed: Math.round((Date.now() - cycleStart) / 60_000), timeoutMinutes: CYCLE_TIMEOUT_MS / 60_000 }, "Cycle timeout reached, skipping remaining accounts");
+                break;
+            }
+            await withTdlibMutex(`ingest:${account.phone}`, () => runWorkerForAccount(account));
+        }
+        log.info({ elapsed: Math.round((Date.now() - cycleStart) / 1000) }, "Ingestion cycle complete");
+    }
+    catch (err) {
+        log.error({ err }, "Ingestion cycle failed");
+    }
+    finally {
+        running = false;
+    }
+}
+/**
+ * Schedule the next cycle with jitter.
+ */
+function scheduleNext() {
+    const intervalMs = config.workerIntervalMinutes * 60 * 1000;
+    const jitterMs = Math.random() * config.jitterMinutes * 60 * 1000;
+    const delay = intervalMs + jitterMs;
+    log.info({ nextRunInMinutes: Math.round(delay / 60000) }, "Next cycle scheduled");
+    timer = setTimeout(async () => {
+        await runCycle();
+        scheduleNext();
+    }, delay);
+}
+/**
+ * Start the scheduler. Runs an immediate first cycle, then schedules subsequent ones.
+ */
+export async function startScheduler() {
+    log.info({
+        intervalMinutes: config.workerIntervalMinutes,
+        jitterMinutes: config.jitterMinutes,
+    }, "Scheduler starting");
+    // Run immediately on start
+    await runCycle();
+    // Then schedule recurring cycles
+    scheduleNext();
+}
+/**
+ * Trigger an immediate ingestion cycle (e.g. from the admin UI).
+ * If a cycle is already running, this is a no-op.
+ */
+export async function triggerImmediateCycle() {
+    if (running) {
+        log.info("Cycle already running, ignoring trigger");
+        return;
+    }
+    log.info("Immediate cycle triggered via UI");
+    await runCycle();
+}
+/**
+ * Stop the scheduler gracefully.
+ */
+export function stopScheduler() {
+    if (timer) {
+        clearTimeout(timer);
+        timer = null;
+    }
+    log.info("Scheduler stopped");
+}
+//# sourceMappingURL=scheduler.js.map

worker/dist/scheduler.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scheduler.js","sourceRoot":"","sources":["../src/scheduler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AACxE,OAAO,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAEvE,MAAM,GAAG,GAAG,WAAW,CAAC,WAAW,CAAC,CAAC;AAErC,IAAI,OAAO,GAAG,KAAK,CAAC;AACpB,IAAI,KAAK,GAAyC,IAAI,CAAC;AACvD,IAAI,UAAU,GAAG,CAAC,CAAC;AAEnB;;;;GAIG;AACH,MAAM,gBAAgB,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,4BAA4B,IAAI,KAAK,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;AAEvG;;;;;;;;;;;GAWG;AACH,KAAK,UAAU,QAAQ;IACrB,IAAI,OAAO,EAAE,CAAC;QACZ,GAAG,CAAC,IAAI,CAAC,wCAAwC,CAAC,CAAC;QACnD,OAAO;IACT,CAAC;IAED,OAAO,GAAG,IAAI,CAAC;IACf,UAAU,EAAE,CAAC;IACb,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC9B,GAAG,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,cAAc,EAAE,gBAAgB,GAAG,MAAM,EAAE,EAAE,0BAA0B,CAAC,CAAC;IAEvG,IAAI,CAAC;QACH,+CAA+C;QAC/C,MAAM,eAAe,GAAG,MAAM,kBAAkB,EAAE,CAAC;QACnD,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/B,GAAG,CAAC,IAAI,CACN,EAAE,KAAK,EAAE,eAAe,CAAC,MAAM,EAAE,EACjC,iDAAiD,CAClD,CAAC;YACF,KAAK,MAAM,OAAO,IAAI,eAAe,EAAE,CAAC;gBACtC,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,GAAG,gBAAgB,EAAE,CAAC;oBAC/C,GAAG,CAAC,IAAI,CAAC,6DAA6D,CAAC,CAAC;oBACxE,MAAM;gBACR,CAAC;gBACD,MAAM,cAAc,CAAC,QAAQ,OAAO,CAAC,KAAK,EAAE,EAAE,GAAG,EAAE,CACjD,mBAAmB,CAAC,OAAO,CAAC,CAC7B,CAAC;YACJ,CAAC;QACH,CAAC;QAED,mDAAmD;QACnD,MAAM,QAAQ,GAAG,MAAM,iBAAiB,EAAE,CAAC;QAE3C,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,GAAG,CAAC,IAAI,CAAC,qDAAqD,CAAC,CAAC;YAChE,OAAO;QACT,CAAC;QAED,GAAG,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,QAAQ,CAAC,MAAM,EAAE,EAAE,qBAAqB,CAAC,CAAC;QAEnE,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,GAAG,gBAAgB,EAAE,CAAC;gBAC/C,GAAG,CAAC,IAAI,CACN,EAAE,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC,GAAG,MAAM,CAAC,EAAE,cAAc,EAAE,gBAAgB,GAAG,MAAM,EAAE,EACtG,oDAAoD,CACrD,CAAC;gBACF,MAAM;YACR,CAAC;YACD,MAAM,cAAc,CAAC,UAAU,OAAO,CAAC,KAAK,EAAE,EAAE,GAAG,EAAE,CACnD,mBAAmB,CAAC,OAAO,CAAC,CAC7B,CAAC;QACJ,CAAC;QAED,GAAG,CAAC,IAAI,CACN,EAAE,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC,GAAG,IAAI,CAAC,EAAE,EACzD,0BAA0B,CAC3B,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,EAAE,wBAAwB,CAAC,CAAC;IAC/C,CAAC;YAAS,CAAC;QACT,OAAO,GAAG,KAAK,CAAC;IAClB,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,YAAY;IACnB,MAAM,UAAU,GAAG,MAAM,CAAC,qBAAqB,GAAG,EAAE,GAAG,IAAI,CAAC;IAC5D,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC,aAAa,GAAG,EAAE,GAAG,IAAI,CAAC;IAClE,MAAM,KAAK,GAAG,UAAU,GAAG,QAAQ,CAAC;IAEpC,GAAG,CAAC,IAAI,CACN,EAAE,gBAAgB,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,EAAE,EAC/C,sBAAsB,CACvB,CAAC;IAEF,KAAK,GAAG,UAAU,CAAC,KAAK,IAAI,EAAE;QAC5B,MAAM,QAAQ,EAAE,CAAC;QACjB,YAAY,EAAE,CAAC;IACjB,CAAC,EAAE,KAAK,CAAC,CAAC;AACZ,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc;IAClC,GAAG,CAAC,IAAI,CACN;QACE,eAAe,EAAE,MAAM,CAAC,qBAAqB;QAC7C,aAAa,EAAE,MAAM,CAAC,aAAa;KACpC,EACD,oBAAoB,CACrB,CAAC;IAEF,2BAA2B;IAC3B,MAAM,QAAQ,EAAE,CAAC;IAEjB,iCAAiC;IACjC,YAAY,EAAE,CAAC;AACjB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB;IACzC,IAAI,OAAO,EAAE,CAAC;QACZ,GAAG,CAAC,IAAI,CAAC,yCAAyC,CAAC,CAAC;QACpD,OAAO;IACT,CAAC;IACD,GAAG,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;IAC7C,MAAM,QAAQ,EAAE,CAAC;AACnB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,aAAa;IAC3B,IAAI,KAAK,EAAE,CAAC;QACV,YAAY,CAAC,KAAK,CAAC,CAAC;QACpB,KAAK,GAAG,IAAI,CAAC;IACf,CAAC;IACD,GAAG,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;AAChC,CAAC"}

worker/dist/tdlib/chats.d.ts

@@ -0,0 +1,31 @@
+import type { Client } from "tdl";
+export interface TelegramChatInfo {
+    chatId: bigint;
+    title: string;
+    type: "channel" | "supergroup" | "group" | "private" | "other";
+    isForum: boolean;
+    memberCount?: number;
+}
+/**
+ * Fetch all chats the account is a member of.
+ * Uses TDLib's getChats to load the chat list, then getChat for details.
+ * Filters to channels and supergroups only (groups/privates are not useful for ingestion).
+ */
+export declare function getAccountChats(client: Client): Promise<TelegramChatInfo[]>;
+/**
+ * Generate an invite link for a chat. The account must be an admin or have
+ * invite link permissions.
+ */
+export declare function generateInviteLink(client: Client, chatId: bigint): Promise<string>;
+/**
+ * Create a new supergroup (private group) via TDLib.
+ * Returns the chat ID and title.
+ */
+export declare function createSupergroup(client: Client, title: string): Promise<{
+    chatId: bigint;
+    title: string;
+}>;
+/**
+ * Join a chat using an invite link.
+ */
+export declare function joinChatByInviteLink(client: Client, inviteLink: string): Promise<void>;

worker/dist/tdlib/chats.js

@@ -0,0 +1,124 @@
+import { childLogger } from "../util/logger.js";
+import { config } from "../util/config.js";
+const log = childLogger("chats");
+/**
+ * Fetch all chats the account is a member of.
+ * Uses TDLib's getChats to load the chat list, then getChat for details.
+ * Filters to channels and supergroups only (groups/privates are not useful for ingestion).
+ */
+export async function getAccountChats(client) {
+    const chats = [];
+    // Load main chat list — TDLib loads in batches
+    let offsetOrder = "9223372036854775807"; // max int64 as string
+    let offsetChatId = 0;
+    let hasMore = true;
+    while (hasMore) {
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const result = (await client.invoke({
+            _: "getChats",
+            chat_list: { _: "chatListMain" },
+            limit: 100,
+        }));
+        if (!result.chat_ids || result.chat_ids.length === 0) {
+            break;
+        }
+        for (const chatId of result.chat_ids) {
+            try {
+                // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                const chat = (await client.invoke({
+                    _: "getChat",
+                    chat_id: chatId,
+                }));
+                const chatType = chat.type?._;
+                let type = "other";
+                let isForum = false;
+                if (chatType === "chatTypeSupergroup") {
+                    // Get supergroup details to check if it's a channel or group
+                    try {
+                        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                        const sg = (await client.invoke({
+                            _: "getSupergroup",
+                            supergroup_id: chat.type.supergroup_id,
+                        }));
+                        type = sg.is_channel ? "channel" : "supergroup";
+                        isForum = sg.is_forum ?? false;
+                    }
+                    catch {
+                        type = "supergroup";
+                    }
+                }
+                else if (chatType === "chatTypeBasicGroup") {
+                    type = "group";
+                }
+                else if (chatType === "chatTypePrivate" || chatType === "chatTypeSecret") {
+                    type = "private";
+                }
+                // Only include channels and supergroups
+                if (type === "channel" || type === "supergroup") {
+                    chats.push({
+                        chatId: BigInt(chatId),
+                        title: chat.title ?? `Chat ${chatId}`,
+                        type,
+                        isForum,
+                    });
+                }
+            }
+            catch (err) {
+                log.warn({ chatId, err }, "Failed to get chat details, skipping");
+            }
+        }
+        // getChats with chatListMain returns all chats at once in newer TDLib versions
+        // So we break after the first batch
+        hasMore = false;
+        await sleep(config.apiDelayMs);
+    }
+    log.info({ total: chats.length }, "Fetched channels/supergroups from Telegram");
+    return chats;
+}
+/**
+ * Generate an invite link for a chat. The account must be an admin or have
+ * invite link permissions.
+ */
+export async function generateInviteLink(client, chatId) {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const result = (await client.invoke({
+        _: "createChatInviteLink",
+        chat_id: Number(chatId),
+        name: "DragonsStash Auto-Join",
+        creates_join_request: false,
+    }));
+    const link = result.invite_link;
+    log.info({ chatId: chatId.toString(), link }, "Generated invite link");
+    return link;
+}
+/**
+ * Create a new supergroup (private group) via TDLib.
+ * Returns the chat ID and title.
+ */
+export async function createSupergroup(client, title) {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const result = (await client.invoke({
+        _: "createNewSupergroupChat",
+        title,
+        is_forum: false,
+        is_channel: false,
+        description: "DragonsStash archive destination — all accounts write here",
+    }));
+    const chatId = BigInt(result.id);
+    log.info({ chatId: chatId.toString(), title }, "Created new supergroup");
+    return { chatId, title: result.title ?? title };
+}
+/**
+ * Join a chat using an invite link.
+ */
+export async function joinChatByInviteLink(client, inviteLink) {
+    await client.invoke({
+        _: "joinChatByInviteLink",
+        invite_link: inviteLink,
+    });
+    log.info({ inviteLink }, "Joined chat by invite link");
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+//# sourceMappingURL=chats.js.map

worker/dist/tdlib/chats.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chats.js","sourceRoot":"","sources":["../../src/tdlib/chats.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,MAAM,GAAG,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;AAUjC;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,MAAc;IAEd,MAAM,KAAK,GAAuB,EAAE,CAAC;IAErC,+CAA+C;IAC/C,IAAI,WAAW,GAAG,qBAAqB,CAAC,CAAC,sBAAsB;IAC/D,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,OAAO,GAAG,IAAI,CAAC;IAEnB,OAAO,OAAO,EAAE,CAAC;QACf,8DAA8D;QAC9D,MAAM,MAAM,GAAG,CAAC,MAAM,MAAM,CAAC,MAAM,CAAC;YAClC,CAAC,EAAE,UAAU;YACb,SAAS,EAAE,EAAE,CAAC,EAAE,cAAc,EAAE;YAChC,KAAK,EAAE,GAAG;SACX,CAAC,CAA2B,CAAC;QAE9B,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrD,MAAM;QACR,CAAC;QAED,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YACrC,IAAI,CAAC;gBACH,8DAA8D;gBAC9D,MAAM,IAAI,GAAG,CAAC,MAAM,MAAM,CAAC,MAAM,CAAC;oBAChC,CAAC,EAAE,SAAS;oBACZ,OAAO,EAAE,MAAM;iBAChB,CAAC,CAAQ,CAAC;gBAEX,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC9B,IAAI,IAAI,GAA6B,OAAO,CAAC;gBAC7C,IAAI,OAAO,GAAG,KAAK,CAAC;gBAEpB,IAAI,QAAQ,KAAK,oBAAoB,EAAE,CAAC;oBACtC,6DAA6D;oBAC7D,IAAI,CAAC;wBACH,8DAA8D;wBAC9D,MAAM,EAAE,GAAG,CAAC,MAAM,MAAM,CAAC,MAAM,CAAC;4BAC9B,CAAC,EAAE,eAAe;4BAClB,aAAa,EAAE,IAAI,CAAC,IAAI,CAAC,aAAa;yBACvC,CAAC,CAAQ,CAAC;wBAEX,IAAI,GAAG,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC;wBAChD,OAAO,GAAG,EAAE,CAAC,QAAQ,IAAI,KAAK,CAAC;oBACjC,CAAC;oBAAC,MAAM,CAAC;wBACP,IAAI,GAAG,YAAY,CAAC;oBACtB,CAAC;gBACH,CAAC;qBAAM,IAAI,QAAQ,KAAK,oBAAoB,EAAE,CAAC;oBAC7C,IAAI,GAAG,OAAO,CAAC;gBACjB,CAAC;qBAAM,IAAI,QAAQ,KAAK,iBAAiB,IAAI,QAAQ,KAAK,gBAAgB,EAAE,CAAC;oBAC3E,IAAI,GAAG,SAAS,CAAC;gBACnB,CAAC;gBAED,wCAAwC;gBACxC,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,YAAY,EAAE,CAAC;oBAChD,KAAK,CAAC,IAAI,CAAC;wBACT,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC;wBACtB,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,QAAQ,MAAM,EAAE;wBACrC,IAAI;wBACJ,OAAO;qBACR,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,GAAG,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,sCAAsC,CAAC,CAAC;YACpE,CAAC;QACH,CAAC;QAED,+EAA+E;QAC/E,oCAAoC;QACpC,OAAO,GAAG,KAAK,CAAC;QAEhB,MAAM,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IACjC,CAAC;IAED,GAAG,CAAC,IAAI,CACN,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,EACvB,4CAA4C,CAC7C,CAAC;IAEF,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,MAAc,EACd,MAAc;IAEd,8DAA8D;IAC9D,MAAM,MAAM,GAAG,CAAC,MAAM,MAAM,CAAC,MAAM,CAAC;QAClC,CAAC,EAAE,sBAAsB;QACzB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC;QACvB,IAAI,EAAE,wBAAwB;QAC9B,oBAAoB,EAAE,KAAK;KAC5B,CAAC,CAAQ,CAAC;IAEX,MAAM,IAAI,GAAG,MAAM,CAAC,WAAqB,CAAC;IAC1C,GAAG,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,EAAE,uBAAuB,CAAC,CAAC;IACvE,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,MAAc,EACd,KAAa;IAEb,8DAA8D;IAC9D,MAAM,MAAM,GAAG,CAAC,MAAM,MAAM,CAAC,MAAM,CAAC;QAClC,CAAC,EAAE,yBAAyB;QAC5B,KAAK;QACL,QAAQ,EAAE,KAAK;QACf,UAAU,EAAE,KAAK;QACjB,WAAW,EAAE,4DAA4D;KAC1E,CAAC,CAAQ,CAAC;IAEX,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACjC,GAAG,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,EAAE,wBAAwB,CAAC,CAAC;IACzE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,KAAK,EAAE,CAAC;AAClD,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,MAAc,EACd,UAAkB;IAElB,MAAM,MAAM,CAAC,MAAM,CAAC;QAClB,CAAC,EAAE,sBAAsB;QACzB,WAAW,EAAE,UAAU;KACxB,CAAC,CAAC;IACH,GAAG,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,EAAE,4BAA4B,CAAC,CAAC;AACzD,CAAC;AAED,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC"}

worker/dist/tdlib/client.d.ts

@@ -0,0 +1,18 @@
+import { type Client } from "tdl";
+interface AccountConfig {
+    id: string;
+    phone: string;
+}
+/**
+ * Create and authenticate a TDLib client for a Telegram account.
+ * Authentication flow communicates with the admin UI via the database:
+ * - Worker sets authState to AWAITING_CODE when TDLib asks for phone code
+ * - Admin enters the code via UI, which writes it to authCode field
+ * - Worker polls DB for the code and feeds it to TDLib
+ */
+export declare function createTdlibClient(account: AccountConfig): Promise<Client>;
+/**
+ * Close a TDLib client gracefully.
+ */
+export declare function closeTdlibClient(client: Client): Promise<void>;
+export {};

worker/dist/tdlib/client.js

@@ -0,0 +1,96 @@
+import tdl, { createClient } from "tdl";
+import { getTdjson } from "prebuilt-tdlib";
+import path from "path";
+import { config } from "../util/config.js";
+import { childLogger } from "../util/logger.js";
+import { updateAccountAuthState, getAccountAuthCode, } from "../db/queries.js";
+const log = childLogger("tdlib-client");
+// Configure tdl to use the prebuilt tdjson shared library
+tdl.configure({ tdjson: getTdjson() });
+/**
+ * Create and authenticate a TDLib client for a Telegram account.
+ * Authentication flow communicates with the admin UI via the database:
+ * - Worker sets authState to AWAITING_CODE when TDLib asks for phone code
+ * - Admin enters the code via UI, which writes it to authCode field
+ * - Worker polls DB for the code and feeds it to TDLib
+ */
+export async function createTdlibClient(account) {
+    const dbPath = path.join(config.tdlibStateDir, account.id);
+    const client = createClient({
+        apiId: config.telegramApiId,
+        apiHash: config.telegramApiHash,
+        databaseDirectory: dbPath,
+        filesDirectory: path.join(dbPath, "files"),
+    });
+    client.on("error", (err) => {
+        log.error({ err, accountId: account.id }, "TDLib client error");
+    });
+    try {
+        await client.login(() => ({
+            getPhoneNumber: async () => {
+                log.info({ accountId: account.id }, "TDLib requesting phone number");
+                return account.phone;
+            },
+            getAuthCode: async () => {
+                log.info({ accountId: account.id }, "TDLib requesting auth code");
+                await updateAccountAuthState(account.id, "AWAITING_CODE");
+                // Poll database for the code entered via admin UI
+                const code = await pollForAuthCode(account.id);
+                if (!code) {
+                    throw new Error("Auth code not provided within timeout");
+                }
+                // Clear the code after reading
+                await updateAccountAuthState(account.id, "AUTHENTICATED", null);
+                return code;
+            },
+            getPassword: async () => {
+                log.info({ accountId: account.id }, "TDLib requesting 2FA password");
+                await updateAccountAuthState(account.id, "AWAITING_PASSWORD");
+                // Poll database for the password entered via admin UI
+                const code = await pollForAuthCode(account.id);
+                if (!code) {
+                    throw new Error("2FA password not provided within timeout");
+                }
+                await updateAccountAuthState(account.id, "AUTHENTICATED", null);
+                return code;
+            },
+        }));
+        await updateAccountAuthState(account.id, "AUTHENTICATED");
+        log.info({ accountId: account.id }, "TDLib client authenticated");
+        return client;
+    }
+    catch (err) {
+        log.error({ err, accountId: account.id }, "TDLib authentication failed");
+        await updateAccountAuthState(account.id, "EXPIRED");
+        throw err;
+    }
+}
+/**
+ * Poll the database every 5 seconds for an auth code, up to 5 minutes.
+ */
+async function pollForAuthCode(accountId, timeoutMs = 300_000) {
+    const start = Date.now();
+    while (Date.now() - start < timeoutMs) {
+        const result = await getAccountAuthCode(accountId);
+        if (result?.authCode) {
+            return result.authCode;
+        }
+        await sleep(5000);
+    }
+    return null;
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+/**
+ * Close a TDLib client gracefully.
+ */
+export async function closeTdlibClient(client) {
+    try {
+        await client.close();
+    }
+    catch (err) {
+        log.warn({ err }, "Error closing TDLib client");
+    }
+}
+//# sourceMappingURL=client.js.map

worker/dist/tdlib/client.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"client.js","sourceRoot":"","sources":["../../src/tdlib/client.ts"],"names":[],"mappings":"AAAA,OAAO,GAAG,EAAE,EAAE,YAAY,EAAe,MAAM,KAAK,CAAC;AACrD,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EACL,sBAAsB,EACtB,kBAAkB,GACnB,MAAM,kBAAkB,CAAC;AAE1B,MAAM,GAAG,GAAG,WAAW,CAAC,cAAc,CAAC,CAAC;AAExC,0DAA0D;AAC1D,GAAG,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;AAOvC;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,OAAsB;IAEtB,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC;IAE3D,MAAM,MAAM,GAAG,YAAY,CAAC;QAC1B,KAAK,EAAE,MAAM,CAAC,aAAa;QAC3B,OAAO,EAAE,MAAM,CAAC,eAAe;QAC/B,iBAAiB,EAAE,MAAM;QACzB,cAAc,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC;KAC3C,CAAC,CAAC;IAEH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;QACzB,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,EAAE,oBAAoB,CAAC,CAAC;IAClE,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC;YACxB,cAAc,EAAE,KAAK,IAAI,EAAE;gBACzB,GAAG,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,EAAE,+BAA+B,CAAC,CAAC;gBACrE,OAAO,OAAO,CAAC,KAAK,CAAC;YACvB,CAAC;YACD,WAAW,EAAE,KAAK,IAAI,EAAE;gBACtB,GAAG,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,EAAE,4BAA4B,CAAC,CAAC;gBAClE,MAAM,sBAAsB,CAAC,OAAO,CAAC,EAAE,EAAE,eAAe,CAAC,CAAC;gBAE1D,kDAAkD;gBAClD,MAAM,IAAI,GAAG,MAAM,eAAe,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;gBAC/C,IAAI,CAAC,IAAI,EAAE,CAAC;oBACV,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;gBAC3D,CAAC;gBAED,+BAA+B;gBAC/B,MAAM,sBAAsB,CAAC,OAAO,CAAC,EAAE,EAAE,eAAe,EAAE,IAAI,CAAC,CAAC;gBAChE,OAAO,IAAI,CAAC;YACd,CAAC;YACD,WAAW,EAAE,KAAK,IAAI,EAAE;gBACtB,GAAG,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,EAAE,+BAA+B,CAAC,CAAC;gBACrE,MAAM,sBAAsB,CAAC,OAAO,CAAC,EAAE,EAAE,mBAAmB,CAAC,CAAC;gBAE9D,sDAAsD;gBACtD,MAAM,IAAI,GAAG,MAAM,eAAe,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;gBAC/C,IAAI,CAAC,IAAI,EAAE,CAAC;oBACV,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;gBAC9D,CAAC;gBAED,MAAM,sBAAsB,CAAC,OAAO,CAAC,EAAE,EAAE,eAAe,EAAE,IAAI,CAAC,CAAC;gBAChE,OAAO,IAAI,CAAC;YACd,CAAC;SACF,CAAC,CAAC,CAAC;QAEJ,MAAM,sBAAsB,CAAC,OAAO,CAAC,EAAE,EAAE,eAAe,CAAC,CAAC;QAC1D,GAAG,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,EAAE,4BAA4B,CAAC,CAAC;QAClE,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,EAAE,6BAA6B,CAAC,CAAC;QACzE,MAAM,sBAAsB,CAAC,OAAO,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;QACpD,MAAM,GAAG,CAAC;IACZ,CAAC;AACH,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,eAAe,CAC5B,SAAiB,EACjB,SAAS,GAAG,OAAO;IAEnB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,GAAG,SAAS,EAAE,CAAC;QACtC,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,SAAS,CAAC,CAAC;QACnD,IAAI,MAAM,EAAE,QAAQ,EAAE,CAAC;YACrB,OAAO,MAAM,CAAC,QAAQ,CAAC;QACzB,CAAC;QACD,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,MAAc;IACnD,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,EAAE,4BAA4B,CAAC,CAAC;IAClD,CAAC;AACH,CAAC"}

worker/dist/tdlib/download.d.ts

@@ -0,0 +1,67 @@
+import type { Client } from "tdl";
+import type { TelegramMessage } from "../archive/multipart.js";
+import type { TelegramPhoto } from "../preview/match.js";
+/** Maximum number of pages to scan per channel/topic to prevent infinite loops */
+export declare const MAX_SCAN_PAGES = 5000;
+/** Timeout for a single TDLib API call (ms) */
+export declare const INVOKE_TIMEOUT_MS = 120000;
+export interface ChannelScanResult {
+    archives: TelegramMessage[];
+    photos: TelegramPhoto[];
+    totalScanned: number;
+}
+export type ScanProgressCallback = (messagesScanned: number) => void;
+/**
+ * Invoke a TDLib method with a timeout to prevent indefinite hangs.
+ * If TDLib does not respond within the timeout, the promise rejects.
+ */
+export declare function invokeWithTimeout<T>(client: Client, request: Record<string, any>, timeoutMs?: number): Promise<T>;
+/**
+ * Fetch messages from a channel, stopping once we've scanned past the
+ * last-processed boundary (with one page of lookback for multipart safety).
+ * Collects both archive attachments AND photo messages (for preview matching).
+ * Returns messages in chronological order (oldest first).
+ *
+ * When `lastProcessedMessageId` is null (first run), scans everything.
+ * The worker applies a post-grouping filter to skip fully-processed sets,
+ * and keeps `packageExistsBySourceMessage` as a safety net.
+ *
+ * Safety features:
+ *  - Max page limit to prevent infinite loops
+ *  - Stuck detection: breaks if from_message_id stops advancing
+ *  - Timeout on each TDLib API call
+ */
+export declare function getChannelMessages(client: Client, chatId: bigint, lastProcessedMessageId?: bigint | null, limit?: number, onProgress?: ScanProgressCallback): Promise<ChannelScanResult>;
+/**
+ * Download a photo thumbnail from Telegram and return its raw bytes.
+ * Uses synchronous download (photos are small, typically < 100KB).
+ * Returns null if download fails (non-critical).
+ */
+export declare function downloadPhotoThumbnail(client: Client, fileId: string): Promise<Buffer | null>;
+export interface DownloadProgress {
+    fileId: string;
+    fileName: string;
+    downloadedBytes: number;
+    totalBytes: number;
+    percent: number;
+    isComplete: boolean;
+}
+export type ProgressCallback = (progress: DownloadProgress) => void;
+/**
+ * Download a file from Telegram to a local path with progress tracking
+ * and integrity verification.
+ *
+ * Progress flow:
+ *  1. Starts async download via TDLib
+ *  2. Listens for `updateFile` events to track download progress
+ *  3. Logs progress at every 10% increment
+ *  4. Once complete, verifies the local file size matches the expected size
+ *  5. Moves the file from TDLib's cache to the destination path
+ *
+ * Verification:
+ *  - Compares actual file size on disk to the expected size from Telegram
+ *  - Throws on mismatch (partial/corrupt download)
+ *  - Throws on timeout (configurable, scales with file size)
+ *  - Throws if download stops without completing (network error, etc.)
+ */
+export declare function downloadFile(client: Client, fileId: string, destPath: string, expectedSize: bigint, fileName: string, onProgress?: ProgressCallback): Promise<void>;

worker/dist/tdlib/download.js

@@ -0,0 +1,307 @@
+import { readFile, rename, copyFile, unlink, stat } from "fs/promises";
+import { config } from "../util/config.js";
+import { childLogger } from "../util/logger.js";
+import { isArchiveAttachment } from "../archive/detect.js";
+const log = childLogger("download");
+/** Maximum number of pages to scan per channel/topic to prevent infinite loops */
+export const MAX_SCAN_PAGES = 5000;
+/** Timeout for a single TDLib API call (ms) */
+export const INVOKE_TIMEOUT_MS = 120_000; // 2 minutes
+/**
+ * Invoke a TDLib method with a timeout to prevent indefinite hangs.
+ * If TDLib does not respond within the timeout, the promise rejects.
+ */
+export async function invokeWithTimeout(client, 
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+request, timeoutMs = INVOKE_TIMEOUT_MS) {
+    return new Promise((resolve, reject) => {
+        const timer = setTimeout(() => {
+            reject(new Error(`TDLib invoke timed out after ${timeoutMs}ms for ${request._}`));
+        }, timeoutMs);
+        client.invoke(request)
+            .then((result) => {
+            clearTimeout(timer);
+            resolve(result);
+        })
+            .catch((err) => {
+            clearTimeout(timer);
+            reject(err);
+        });
+    });
+}
+/**
+ * Fetch messages from a channel, stopping once we've scanned past the
+ * last-processed boundary (with one page of lookback for multipart safety).
+ * Collects both archive attachments AND photo messages (for preview matching).
+ * Returns messages in chronological order (oldest first).
+ *
+ * When `lastProcessedMessageId` is null (first run), scans everything.
+ * The worker applies a post-grouping filter to skip fully-processed sets,
+ * and keeps `packageExistsBySourceMessage` as a safety net.
+ *
+ * Safety features:
+ *  - Max page limit to prevent infinite loops
+ *  - Stuck detection: breaks if from_message_id stops advancing
+ *  - Timeout on each TDLib API call
+ */
+export async function getChannelMessages(client, chatId, lastProcessedMessageId, limit = 100, onProgress) {
+    const archives = [];
+    const photos = [];
+    const boundary = lastProcessedMessageId ? Number(lastProcessedMessageId) : null;
+    let currentFromId = 0;
+    let totalScanned = 0;
+    let pageCount = 0;
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+        if (pageCount >= MAX_SCAN_PAGES) {
+            log.warn({ chatId: chatId.toString(), pageCount, totalScanned }, "Hit max page limit for channel scan, stopping");
+            break;
+        }
+        pageCount++;
+        const previousFromId = currentFromId;
+        const result = await invokeWithTimeout(client, {
+            _: "getChatHistory",
+            chat_id: Number(chatId),
+            from_message_id: currentFromId,
+            offset: 0,
+            limit: Math.min(limit, 100),
+            only_local: false,
+        });
+        if (!result.messages || result.messages.length === 0)
+            break;
+        totalScanned += result.messages.length;
+        for (const msg of result.messages) {
+            // Check for archive documents
+            const doc = msg.content?.document;
+            if (doc?.file_name && doc.document && isArchiveAttachment(doc.file_name)) {
+                archives.push({
+                    id: BigInt(msg.id),
+                    fileName: doc.file_name,
+                    fileId: String(doc.document.id),
+                    fileSize: BigInt(doc.document.size),
+                    date: new Date(msg.date * 1000),
+                });
+                continue;
+            }
+            // Check for photo messages (potential previews)
+            const photo = msg.content?.photo;
+            const caption = msg.content?.caption?.text ?? "";
+            if (photo?.sizes && photo.sizes.length > 0) {
+                const smallest = photo.sizes[0];
+                photos.push({
+                    id: BigInt(msg.id),
+                    date: new Date(msg.date * 1000),
+                    caption,
+                    fileId: String(smallest.photo.id),
+                    fileSize: smallest.photo.size || smallest.photo.expected_size,
+                });
+            }
+        }
+        // Report scanning progress after each page
+        onProgress?.(totalScanned);
+        currentFromId = result.messages[result.messages.length - 1].id;
+        // Stuck detection: if from_message_id didn't advance, break to prevent infinite loop
+        if (currentFromId === previousFromId) {
+            log.warn({ chatId: chatId.toString(), currentFromId, totalScanned }, "Pagination stuck (from_message_id not advancing), breaking");
+            break;
+        }
+        // Stop scanning once we've gone past the boundary (this page is the lookback)
+        if (boundary && currentFromId < boundary)
+            break;
+        if (result.messages.length < Math.min(limit, 100))
+            break;
+        // Rate limit delay
+        await sleep(config.apiDelayMs);
+    }
+    log.info({ chatId: chatId.toString(), archives: archives.length, photos: photos.length, totalScanned, pages: pageCount }, "Channel scan complete");
+    // Reverse to chronological order (oldest first) so worker processes old→new
+    return {
+        archives: archives.reverse(),
+        photos: photos.reverse(),
+        totalScanned,
+    };
+}
+/**
+ * Download a photo thumbnail from Telegram and return its raw bytes.
+ * Uses synchronous download (photos are small, typically < 100KB).
+ * Returns null if download fails (non-critical).
+ */
+export async function downloadPhotoThumbnail(client, fileId) {
+    const numericId = parseInt(fileId, 10);
+    try {
+        const result = (await client.invoke({
+            _: "downloadFile",
+            file_id: numericId,
+            priority: 1, // Low priority — thumbnails are nice-to-have
+            offset: 0,
+            limit: 0,
+            synchronous: true, // Small file — wait for it
+        }));
+        if (result?.local?.is_downloading_completed && result.local.path) {
+            const data = await readFile(result.local.path);
+            log.debug({ fileId, bytes: data.length }, "Downloaded photo thumbnail");
+            return data;
+        }
+    }
+    catch (err) {
+        log.warn({ fileId, err }, "Failed to download photo thumbnail");
+    }
+    return null;
+}
+/**
+ * Download a file from Telegram to a local path with progress tracking
+ * and integrity verification.
+ *
+ * Progress flow:
+ *  1. Starts async download via TDLib
+ *  2. Listens for `updateFile` events to track download progress
+ *  3. Logs progress at every 10% increment
+ *  4. Once complete, verifies the local file size matches the expected size
+ *  5. Moves the file from TDLib's cache to the destination path
+ *
+ * Verification:
+ *  - Compares actual file size on disk to the expected size from Telegram
+ *  - Throws on mismatch (partial/corrupt download)
+ *  - Throws on timeout (configurable, scales with file size)
+ *  - Throws if download stops without completing (network error, etc.)
+ */
+export async function downloadFile(client, fileId, destPath, expectedSize, fileName, onProgress) {
+    const numericId = parseInt(fileId, 10);
+    const totalBytes = Number(expectedSize);
+    log.info({ fileId, fileName, destPath, totalBytes }, "Starting file download");
+    // Report initial progress
+    onProgress?.({
+        fileId,
+        fileName,
+        downloadedBytes: 0,
+        totalBytes,
+        percent: 0,
+        isComplete: false,
+    });
+    return new Promise((resolve, reject) => {
+        let lastLoggedPercent = 0;
+        let settled = false;
+        // Timeout: 10 minutes per GB, minimum 5 minutes
+        const timeoutMs = Math.max(5 * 60_000, (totalBytes / (1024 * 1024 * 1024)) * 10 * 60_000);
+        const timer = setTimeout(() => {
+            if (!settled) {
+                settled = true;
+                cleanup();
+                reject(new Error(`Download timed out after ${Math.round(timeoutMs / 60_000)}min for ${fileName}`));
+            }
+        }, timeoutMs);
+        // Listen for file update events to track progress
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const handleUpdate = (update) => {
+            if (update?._ !== "updateFile")
+                return;
+            const file = update.file;
+            if (!file || file.id !== numericId)
+                return;
+            const downloaded = file.local.downloaded_size;
+            const percent = totalBytes > 0 ? Math.round((downloaded / totalBytes) * 100) : 0;
+            // Log at every 10% increment
+            if (percent >= lastLoggedPercent + 10) {
+                lastLoggedPercent = percent - (percent % 10);
+                log.info({ fileId, fileName, downloaded, totalBytes, percent: `${percent}%` }, "Download progress");
+            }
+            // Report to callback
+            onProgress?.({
+                fileId,
+                fileName,
+                downloadedBytes: downloaded,
+                totalBytes,
+                percent,
+                isComplete: file.local.is_downloading_completed,
+            });
+            // Download finished
+            if (file.local.is_downloading_completed) {
+                if (!settled) {
+                    settled = true;
+                    cleanup();
+                    verifyAndMove(file.local.path, destPath, totalBytes, fileName, fileId)
+                        .then(resolve)
+                        .catch(reject);
+                }
+            }
+            // Download stopped without completing (network error, cancelled, etc.)
+            if (!file.local.is_downloading_active &&
+                !file.local.is_downloading_completed) {
+                if (!settled) {
+                    settled = true;
+                    cleanup();
+                    reject(new Error(`Download stopped unexpectedly for ${fileName} ` +
+                        `(${downloaded}/${totalBytes} bytes, ${percent}%)`));
+                }
+            }
+        };
+        const cleanup = () => {
+            clearTimeout(timer);
+            client.off("update", handleUpdate);
+        };
+        // Subscribe to updates BEFORE starting download
+        client.on("update", handleUpdate);
+        // Start async download (non-blocking — progress via updateFile events)
+        client
+            .invoke({
+            _: "downloadFile",
+            file_id: numericId,
+            priority: 32,
+            offset: 0,
+            limit: 0,
+            synchronous: false,
+        })
+            .then((result) => {
+            // If the file was already cached locally, invoke returns immediately
+            const file = result;
+            if (file?.local?.is_downloading_completed && !settled) {
+                settled = true;
+                cleanup();
+                verifyAndMove(file.local.path, destPath, totalBytes, fileName, fileId)
+                    .then(resolve)
+                    .catch(reject);
+            }
+        })
+            .catch((err) => {
+            if (!settled) {
+                settled = true;
+                cleanup();
+                reject(err);
+            }
+        });
+    });
+}
+/**
+ * Verify the downloaded file's size matches the expected size,
+ * then move it to the destination path.
+ */
+async function verifyAndMove(localPath, destPath, expectedBytes, fileName, fileId) {
+    const stats = await stat(localPath);
+    const actualBytes = stats.size;
+    if (expectedBytes > 0 && actualBytes !== expectedBytes) {
+        log.error({ fileId, fileName, expectedBytes, actualBytes }, "Download size mismatch — file is incomplete or corrupted");
+        throw new Error(`Download verification failed for ${fileName}: ` +
+            `expected ${expectedBytes} bytes, got ${actualBytes} bytes`);
+    }
+    log.info({ fileId, fileName, bytes: actualBytes, destPath }, "File verified and complete");
+    // Move from TDLib's cache to our temp directory.
+    // Use rename first (fast, same filesystem), fall back to copy+delete
+    // when source and destination are on different filesystems (EXDEV).
+    try {
+        await rename(localPath, destPath);
+    }
+    catch (err) {
+        if (err.code === "EXDEV") {
+            log.debug({ fileId, fileName }, "Cross-device rename — falling back to copy + unlink");
+            await copyFile(localPath, destPath);
+            await unlink(localPath);
+        }
+        else {
+            throw err;
+        }
+    }
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+//# sourceMappingURL=download.js.map

worker/dist/tdlib/topics.d.ts

@@ -0,0 +1,32 @@
+import type { Client } from "tdl";
+import type { ChannelScanResult, ScanProgressCallback } from "./download.js";
+export interface ForumTopic {
+    topicId: bigint;
+    name: string;
+}
+/**
+ * Check if a chat is a forum supergroup (topics enabled).
+ */
+export declare function isChatForum(client: Client, chatId: bigint): Promise<boolean>;
+/**
+ * Get all forum topics in a supergroup.
+ * Includes stuck detection and timeout protection on API calls.
+ */
+export declare function getForumTopicList(client: Client, chatId: bigint): Promise<ForumTopic[]>;
+/**
+ * Fetch messages from a specific forum topic (thread), stopping once
+ * we've scanned past the last-processed boundary (with one page of lookback).
+ * Uses searchChatMessages with message_thread_id to scan within a topic.
+ *
+ * Returns messages in chronological order (oldest first).
+ *
+ * When `lastProcessedMessageId` is null (first run), scans everything.
+ * The worker applies a post-grouping filter to skip fully-processed sets,
+ * and keeps `packageExistsBySourceMessage` as a safety net.
+ *
+ * Safety features:
+ *  - Max page limit to prevent infinite loops
+ *  - Stuck detection: breaks if from_message_id stops advancing
+ *  - Timeout on each TDLib API call
+ */
+export declare function getTopicMessages(client: Client, chatId: bigint, topicId: bigint, lastProcessedMessageId?: bigint | null, limit?: number, onProgress?: ScanProgressCallback): Promise<ChannelScanResult>;

worker/dist/tdlib/topics.js

@@ -0,0 +1,196 @@
+import { config } from "../util/config.js";
+import { childLogger } from "../util/logger.js";
+import { isArchiveAttachment } from "../archive/detect.js";
+import { invokeWithTimeout, MAX_SCAN_PAGES } from "./download.js";
+const log = childLogger("topics");
+/**
+ * Check if a chat is a forum supergroup (topics enabled).
+ */
+export async function isChatForum(client, chatId) {
+    try {
+        const chat = await invokeWithTimeout(client, {
+            _: "getChat",
+            chat_id: Number(chatId),
+        });
+        if (chat.type?._ === "chatTypeSupergroup" && chat.type.is_forum) {
+            return true;
+        }
+        // Also check via getSupergroup for older TDLib versions
+        if (chat.type?._ === "chatTypeSupergroup" && chat.type.supergroup_id) {
+            const sg = await invokeWithTimeout(client, {
+                _: "getSupergroup",
+                supergroup_id: chat.type.supergroup_id,
+            });
+            return sg.is_forum === true;
+        }
+        return false;
+    }
+    catch (err) {
+        log.warn({ err, chatId: chatId.toString() }, "Failed to check if chat is forum");
+        return false;
+    }
+}
+/**
+ * Get all forum topics in a supergroup.
+ * Includes stuck detection and timeout protection on API calls.
+ */
+export async function getForumTopicList(client, chatId) {
+    const topics = [];
+    let offsetDate = 0;
+    let offsetMessageId = 0;
+    let offsetMessageThreadId = 0;
+    let pageCount = 0;
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+        if (pageCount >= MAX_SCAN_PAGES) {
+            log.warn({ chatId: chatId.toString(), pageCount, topicCount: topics.length }, "Hit max page limit for topic enumeration, stopping");
+            break;
+        }
+        pageCount++;
+        const prevOffsetDate = offsetDate;
+        const prevOffsetMessageId = offsetMessageId;
+        const prevOffsetMessageThreadId = offsetMessageThreadId;
+        const result = await invokeWithTimeout(client, {
+            _: "getForumTopics",
+            chat_id: Number(chatId),
+            query: "",
+            offset_date: offsetDate,
+            offset_message_id: offsetMessageId,
+            offset_message_thread_id: offsetMessageThreadId,
+            limit: 100,
+        });
+        if (!result.topics || result.topics.length === 0)
+            break;
+        for (const t of result.topics) {
+            if (!t.info?.message_thread_id)
+                continue;
+            // Skip the "General" topic — it's not creator-specific
+            if (t.info.is_general)
+                continue;
+            topics.push({
+                topicId: BigInt(t.info.message_thread_id),
+                name: t.info.name ?? "Unnamed",
+            });
+        }
+        // Check if there are more pages
+        if (!result.next_offset_date &&
+            !result.next_offset_message_id &&
+            !result.next_offset_message_thread_id) {
+            break;
+        }
+        offsetDate = result.next_offset_date ?? 0;
+        offsetMessageId = result.next_offset_message_id ?? 0;
+        offsetMessageThreadId = result.next_offset_message_thread_id ?? 0;
+        // Stuck detection: if offsets didn't advance, break
+        if (offsetDate === prevOffsetDate &&
+            offsetMessageId === prevOffsetMessageId &&
+            offsetMessageThreadId === prevOffsetMessageThreadId) {
+            log.warn({ chatId: chatId.toString(), topicCount: topics.length }, "Topic pagination stuck (offsets not advancing), breaking");
+            break;
+        }
+        await sleep(config.apiDelayMs);
+    }
+    log.info({ chatId: chatId.toString(), topicCount: topics.length }, "Enumerated forum topics");
+    return topics;
+}
+/**
+ * Fetch messages from a specific forum topic (thread), stopping once
+ * we've scanned past the last-processed boundary (with one page of lookback).
+ * Uses searchChatMessages with message_thread_id to scan within a topic.
+ *
+ * Returns messages in chronological order (oldest first).
+ *
+ * When `lastProcessedMessageId` is null (first run), scans everything.
+ * The worker applies a post-grouping filter to skip fully-processed sets,
+ * and keeps `packageExistsBySourceMessage` as a safety net.
+ *
+ * Safety features:
+ *  - Max page limit to prevent infinite loops
+ *  - Stuck detection: breaks if from_message_id stops advancing
+ *  - Timeout on each TDLib API call
+ */
+export async function getTopicMessages(client, chatId, topicId, lastProcessedMessageId, limit = 100, onProgress) {
+    const archives = [];
+    const photos = [];
+    const boundary = lastProcessedMessageId ? Number(lastProcessedMessageId) : null;
+    let currentFromId = 0;
+    let totalScanned = 0;
+    let pageCount = 0;
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+        if (pageCount >= MAX_SCAN_PAGES) {
+            log.warn({ chatId: chatId.toString(), topicId: topicId.toString(), pageCount, totalScanned }, "Hit max page limit for topic scan, stopping");
+            break;
+        }
+        pageCount++;
+        const previousFromId = currentFromId;
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const result = await invokeWithTimeout(client, {
+            _: "searchChatMessages",
+            chat_id: Number(chatId),
+            query: "",
+            message_thread_id: Number(topicId),
+            from_message_id: currentFromId,
+            offset: 0,
+            limit: Math.min(limit, 100),
+            filter: null,
+            sender_id: null,
+            saved_messages_topic_id: 0,
+        });
+        if (!result.messages || result.messages.length === 0)
+            break;
+        totalScanned += result.messages.length;
+        for (const msg of result.messages) {
+            // Check for archive documents
+            const doc = msg.content?.document;
+            if (doc?.file_name && doc.document && isArchiveAttachment(doc.file_name)) {
+                archives.push({
+                    id: BigInt(msg.id),
+                    fileName: doc.file_name,
+                    fileId: String(doc.document.id),
+                    fileSize: BigInt(doc.document.size),
+                    date: new Date(msg.date * 1000),
+                });
+                continue;
+            }
+            // Check for photo messages (potential previews)
+            const photo = msg.content?.photo;
+            const caption = msg.content?.caption?.text ?? "";
+            if (photo?.sizes && photo.sizes.length > 0) {
+                const smallest = photo.sizes[0];
+                photos.push({
+                    id: BigInt(msg.id),
+                    date: new Date(msg.date * 1000),
+                    caption,
+                    fileId: String(smallest.photo.id),
+                    fileSize: smallest.photo.size || smallest.photo.expected_size,
+                });
+            }
+        }
+        // Report scanning progress after each page
+        onProgress?.(totalScanned);
+        currentFromId = result.messages[result.messages.length - 1].id;
+        // Stuck detection: if from_message_id didn't advance, break to prevent infinite loop
+        if (currentFromId === previousFromId) {
+            log.warn({ chatId: chatId.toString(), topicId: topicId.toString(), currentFromId, totalScanned }, "Topic pagination stuck (from_message_id not advancing), breaking");
+            break;
+        }
+        // Stop scanning once we've gone past the boundary (this page is the lookback)
+        if (boundary && currentFromId < boundary)
+            break;
+        if (result.messages.length < Math.min(limit, 100))
+            break;
+        await sleep(config.apiDelayMs);
+    }
+    log.info({ chatId: chatId.toString(), topicId: topicId.toString(), archives: archives.length, photos: photos.length, totalScanned, pages: pageCount }, "Topic scan complete");
+    // Reverse to chronological order (oldest first) so worker processes old→new
+    return {
+        archives: archives.reverse(),
+        photos: photos.reverse(),
+        totalScanned,
+    };
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+//# sourceMappingURL=topics.js.map

worker/dist/upload/channel.d.ts

@@ -0,0 +1,16 @@
+import type { Client } from "tdl";
+export interface UploadResult {
+    messageId: bigint;
+}
+/**
+ * Upload one or more files to a destination Telegram channel.
+ * For multipart archives, each file is sent as a separate message.
+ * Returns the **final** (server-assigned) message ID of the first uploaded message.
+ *
+ * IMPORTANT: `sendMessage` returns a *temporary* message immediately.
+ * The actual file upload happens asynchronously in TDLib. We listen for
+ * `updateMessageSendSucceeded` to get the real server-side message ID and
+ * to make sure the upload is fully committed before we clean up temp files
+ * or close the TDLib client (which would cancel pending uploads).
+ */
+export declare function uploadToChannel(client: Client, chatId: bigint, filePaths: string[], caption?: string): Promise<UploadResult>;

worker/dist/upload/channel.js

@@ -0,0 +1,137 @@
+import path from "path";
+import { stat } from "fs/promises";
+import { config } from "../util/config.js";
+import { childLogger } from "../util/logger.js";
+const log = childLogger("upload");
+/**
+ * Upload one or more files to a destination Telegram channel.
+ * For multipart archives, each file is sent as a separate message.
+ * Returns the **final** (server-assigned) message ID of the first uploaded message.
+ *
+ * IMPORTANT: `sendMessage` returns a *temporary* message immediately.
+ * The actual file upload happens asynchronously in TDLib. We listen for
+ * `updateMessageSendSucceeded` to get the real server-side message ID and
+ * to make sure the upload is fully committed before we clean up temp files
+ * or close the TDLib client (which would cancel pending uploads).
+ */
+export async function uploadToChannel(client, chatId, filePaths, caption) {
+    let firstMessageId = null;
+    for (let i = 0; i < filePaths.length; i++) {
+        const filePath = filePaths[i];
+        const fileCaption = i === 0 && caption ? caption : undefined;
+        const fileName = path.basename(filePath);
+        let fileSizeMB = 0;
+        try {
+            const s = await stat(filePath);
+            fileSizeMB = Math.round(s.size / (1024 * 1024));
+        }
+        catch {
+            // Non-critical
+        }
+        log.info({ chatId: Number(chatId), fileName, sizeMB: fileSizeMB, part: i + 1, total: filePaths.length }, "Uploading file to channel");
+        const serverMsgId = await sendAndWaitForUpload(client, chatId, filePath, fileCaption, fileName, fileSizeMB);
+        if (i === 0) {
+            firstMessageId = serverMsgId;
+        }
+        // Rate limit delay between uploads
+        if (i < filePaths.length - 1) {
+            await sleep(config.apiDelayMs);
+        }
+    }
+    if (firstMessageId === null) {
+        throw new Error("Upload failed: no messages sent");
+    }
+    log.info({ chatId: Number(chatId), messageId: Number(firstMessageId), files: filePaths.length }, "All uploads confirmed by Telegram");
+    return { messageId: firstMessageId };
+}
+/**
+ * Send a single file message and wait for Telegram to confirm the upload.
+ * Returns the final server-assigned message ID.
+ */
+async function sendAndWaitForUpload(client, chatId, filePath, caption, fileName, fileSizeMB) {
+    // Send the message — this returns a temporary message immediately
+    const tempMsg = (await client.invoke({
+        _: "sendMessage",
+        chat_id: Number(chatId),
+        input_message_content: {
+            _: "inputMessageDocument",
+            document: {
+                _: "inputFileLocal",
+                path: filePath,
+            },
+            caption: caption
+                ? {
+                    _: "formattedText",
+                    text: caption,
+                }
+                : undefined,
+        },
+    }));
+    const tempMsgId = tempMsg.id;
+    log.debug({ fileName, tempMsgId }, "Message queued, waiting for upload confirmation");
+    // Wait for the actual upload to complete
+    return new Promise((resolve, reject) => {
+        let settled = false;
+        let lastLoggedPercent = 0;
+        // Timeout: 10 minutes per GB, minimum 10 minutes
+        const timeoutMs = Math.max(10 * 60_000, (fileSizeMB / 1024) * 10 * 60_000);
+        const timer = setTimeout(() => {
+            if (!settled) {
+                settled = true;
+                cleanup();
+                reject(new Error(`Upload timed out after ${Math.round(timeoutMs / 60_000)}min for ${fileName}`));
+            }
+        }, timeoutMs);
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const handleUpdate = (update) => {
+            // Track upload progress via updateFile events
+            if (update?._ === "updateFile") {
+                const file = update.file;
+                if (file?.remote?.is_uploading_active && file.expected_size > 0) {
+                    const uploaded = file.remote.uploaded_size ?? 0;
+                    const total = file.expected_size;
+                    const percent = Math.round((uploaded / total) * 100);
+                    if (percent >= lastLoggedPercent + 20) {
+                        lastLoggedPercent = percent - (percent % 20);
+                        log.info({ fileName, uploaded, total, percent: `${percent}%` }, "Upload progress");
+                    }
+                }
+            }
+            // The money event: upload succeeded, we get the final server message ID
+            if (update?._ === "updateMessageSendSucceeded") {
+                const msg = update.message;
+                const oldMsgId = update.old_message_id;
+                if (oldMsgId === tempMsgId) {
+                    if (!settled) {
+                        settled = true;
+                        cleanup();
+                        const finalId = BigInt(msg.id);
+                        log.info({ fileName, tempMsgId, finalMsgId: Number(finalId) }, "Upload confirmed by Telegram");
+                        resolve(finalId);
+                    }
+                }
+            }
+            // Upload failed
+            if (update?._ === "updateMessageSendFailed") {
+                const oldMsgId = update.old_message_id;
+                if (oldMsgId === tempMsgId) {
+                    if (!settled) {
+                        settled = true;
+                        cleanup();
+                        const errorMsg = update.error?.message ?? "Unknown upload error";
+                        reject(new Error(`Upload failed for ${fileName}: ${errorMsg}`));
+                    }
+                }
+            }
+        };
+        const cleanup = () => {
+            clearTimeout(timer);
+            client.off("update", handleUpdate);
+        };
+        client.on("update", handleUpdate);
+    });
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+//# sourceMappingURL=channel.js.map

worker/dist/upload/channel.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"channel.js","sourceRoot":"","sources":["../../src/upload/channel.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AAEnC,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,MAAM,GAAG,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC;AAMlC;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,MAAc,EACd,MAAc,EACd,SAAmB,EACnB,OAAgB;IAEhB,IAAI,cAAc,GAAkB,IAAI,CAAC;IAEzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;QAC9B,MAAM,WAAW,GACf,CAAC,KAAK,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;QAE3C,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACzC,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,CAAC;YACH,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC/B,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC;QAClD,CAAC;QAAC,MAAM,CAAC;YACP,eAAe;QACjB,CAAC;QAED,GAAG,CAAC,IAAI,CACN,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,SAAS,CAAC,MAAM,EAAE,EAC9F,2BAA2B,CAC5B,CAAC;QAEF,MAAM,WAAW,GAAG,MAAM,oBAAoB,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;QAE5G,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YACZ,cAAc,GAAG,WAAW,CAAC;QAC/B,CAAC;QAED,mCAAmC;QACnC,IAAI,CAAC,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7B,MAAM,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;IAED,IAAI,cAAc,KAAK,IAAI,EAAE,CAAC;QAC5B,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;IACrD,CAAC;IAED,GAAG,CAAC,IAAI,CACN,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,cAAc,CAAC,EAAE,KAAK,EAAE,SAAS,CAAC,MAAM,EAAE,EACtF,mCAAmC,CACpC,CAAC;IAEF,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,CAAC;AACvC,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,oBAAoB,CACjC,MAAc,EACd,MAAc,EACd,QAAgB,EAChB,OAA2B,EAC3B,QAAgB,EAChB,UAAkB;IAElB,kEAAkE;IAClE,MAAM,OAAO,GAAG,CAAC,MAAM,MAAM,CAAC,MAAM,CAAC;QACnC,CAAC,EAAE,aAAa;QAChB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC;QACvB,qBAAqB,EAAE;YACrB,CAAC,EAAE,sBAAsB;YACzB,QAAQ,EAAE;gBACR,CAAC,EAAE,gBAAgB;gBACnB,IAAI,EAAE,QAAQ;aACf;YACD,OAAO,EAAE,OAAO;gBACd,CAAC,CAAC;oBACE,CAAC,EAAE,eAAe;oBAClB,IAAI,EAAE,OAAO;iBACd;gBACH,CAAC,CAAC,SAAS;SACd;KACF,CAAC,CAAmB,CAAC;IAEtB,MAAM,SAAS,GAAG,OAAO,CAAC,EAAE,CAAC;IAE7B,GAAG,CAAC,KAAK,CACP,EAAE,QAAQ,EAAE,SAAS,EAAE,EACvB,iDAAiD,CAClD,CAAC;IAEF,yCAAyC;IACzC,OAAO,IAAI,OAAO,CAAS,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC7C,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,IAAI,iBAAiB,GAAG,CAAC,CAAC;QAE1B,iDAAiD;QACjD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CACxB,EAAE,GAAG,MAAM,EACX,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAClC,CAAC;QAEF,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,OAAO,GAAG,IAAI,CAAC;gBACf,OAAO,EAAE,CAAC;gBACV,MAAM,CACJ,IAAI,KAAK,CACP,0BAA0B,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,MAAM,CAAC,WAAW,QAAQ,EAAE,CAC9E,CACF,CAAC;YACJ,CAAC;QACH,CAAC,EAAE,SAAS,CAAC,CAAC;QAEd,8DAA8D;QAC9D,MAAM,YAAY,GAAG,CAAC,MAAW,EAAE,EAAE;YACnC,8CAA8C;YAC9C,IAAI,MAAM,EAAE,CAAC,KAAK,YAAY,EAAE,CAAC;gBAC/B,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;gBACzB,IAAI,IAAI,EAAE,MAAM,EAAE,mBAAmB,IAAI,IAAI,CAAC,aAAa,GAAG,CAAC,EAAE,CAAC;oBAChE,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa,IAAI,CAAC,CAAC;oBAChD,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC;oBACjC,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC;oBACrD,IAAI,OAAO,IAAI,iBAAiB,GAAG,EAAE,EAAE,CAAC;wBACtC,iBAAiB,GAAG,OAAO,GAAG,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC;wBAC7C,GAAG,CAAC,IAAI,CACN,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,OAAO,GAAG,EAAE,EACrD,iBAAiB,CAClB,CAAC;oBACJ,CAAC;gBACH,CAAC;YACH,CAAC;YAED,wEAAwE;YACxE,IAAI,MAAM,EAAE,CAAC,KAAK,4BAA4B,EAAE,CAAC;gBAC/C,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC;gBAC3B,MAAM,QAAQ,GAAG,MAAM,CAAC,cAAc,CAAC;gBACvC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;oBAC3B,IAAI,CAAC,OAAO,EAAE,CAAC;wBACb,OAAO,GAAG,IAAI,CAAC;wBACf,OAAO,EAAE,CAAC;wBACV,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;wBAC/B,GAAG,CAAC,IAAI,CACN,EAAE,QAAQ,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,CAAC,OAAO,CAAC,EAAE,EACpD,8BAA8B,CAC/B,CAAC;wBACF,OAAO,CAAC,OAAO,CAAC,CAAC;oBACnB,CAAC;gBACH,CAAC;YACH,CAAC;YAED,gBAAgB;YAChB,IAAI,MAAM,EAAE,CAAC,KAAK,yBAAyB,EAAE,CAAC;gBAC5C,MAAM,QAAQ,GAAG,MAAM,CAAC,cAAc,CAAC;gBACvC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;oBAC3B,IAAI,CAAC,OAAO,EAAE,CAAC;wBACb,OAAO,GAAG,IAAI,CAAC;wBACf,OAAO,EAAE,CAAC;wBACV,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,EAAE,OAAO,IAAI,sBAAsB,CAAC;wBACjE,MAAM,CAAC,IAAI,KAAK,CAAC,qBAAqB,QAAQ,KAAK,QAAQ,EAAE,CAAC,CAAC,CAAC;oBAClE,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC,CAAC;QAEF,MAAM,OAAO,GAAG,GAAG,EAAE;YACnB,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;QACrC,CAAC,CAAC;QAEF,MAAM,CAAC,EAAE,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IACpC,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC"}

worker/dist/util/config.d.ts

@@ -0,0 +1,18 @@
+export declare const config: {
+    readonly databaseUrl: string;
+    readonly workerIntervalMinutes: number;
+    readonly tempDir: string;
+    readonly tdlibStateDir: string;
+    readonly maxZipSizeMB: number;
+    readonly logLevel: "debug" | "info" | "warn" | "error";
+    readonly telegramApiId: number;
+    readonly telegramApiHash: string;
+    /** Maximum jitter added to scheduler interval (in minutes) */
+    readonly jitterMinutes: 5;
+    /** Maximum time span for multipart archive parts (in hours). 0 = no limit. */
+    readonly multipartTimeoutHours: number;
+    /** Delay between Telegram API calls (in ms) to avoid rate limits */
+    readonly apiDelayMs: 1000;
+    /** Max retries for rate-limited requests */
+    readonly maxRetries: 5;
+};

worker/dist/util/config.js

@@ -0,0 +1,19 @@
+export const config = {
+    databaseUrl: process.env.DATABASE_URL ?? "",
+    workerIntervalMinutes: parseInt(process.env.WORKER_INTERVAL_MINUTES ?? "60", 10),
+    tempDir: process.env.WORKER_TEMP_DIR ?? "/tmp/zips",
+    tdlibStateDir: process.env.TDLIB_STATE_DIR ?? "/data/tdlib",
+    maxZipSizeMB: parseInt(process.env.WORKER_MAX_ZIP_SIZE_MB ?? "4096", 10),
+    logLevel: (process.env.LOG_LEVEL ?? "info"),
+    telegramApiId: parseInt(process.env.TELEGRAM_API_ID ?? "0", 10),
+    telegramApiHash: process.env.TELEGRAM_API_HASH ?? "",
+    /** Maximum jitter added to scheduler interval (in minutes) */
+    jitterMinutes: 5,
+    /** Maximum time span for multipart archive parts (in hours). 0 = no limit. */
+    multipartTimeoutHours: parseInt(process.env.MULTIPART_TIMEOUT_HOURS ?? "0", 10),
+    /** Delay between Telegram API calls (in ms) to avoid rate limits */
+    apiDelayMs: 1000,
+    /** Max retries for rate-limited requests */
+    maxRetries: 5,
+};
+//# sourceMappingURL=config.js.map

worker/dist/util/config.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/util/config.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,MAAM,GAAG;IACpB,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,EAAE;IAC3C,qBAAqB,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB,IAAI,IAAI,EAAE,EAAE,CAAC;IAChF,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,WAAW;IACnD,aAAa,EAAE,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,aAAa;IAC3D,YAAY,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,sBAAsB,IAAI,MAAM,EAAE,EAAE,CAAC;IACxE,QAAQ,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,MAAM,CAAwC;IAClF,aAAa,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,GAAG,EAAE,EAAE,CAAC;IAC/D,eAAe,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,EAAE;IACpD,8DAA8D;IAC9D,aAAa,EAAE,CAAC;IAChB,8EAA8E;IAC9E,qBAAqB,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB,IAAI,GAAG,EAAE,EAAE,CAAC;IAC/E,oEAAoE;IACpE,UAAU,EAAE,IAAI;IAChB,4CAA4C;IAC5C,UAAU,EAAE,CAAC;CACL,CAAC"}

worker/dist/util/logger.d.ts

@@ -0,0 +1,3 @@
+import pino from "pino";
+export declare const logger: pino.Logger<never, boolean>;
+export declare function childLogger(name: string, extra?: Record<string, unknown>): pino.Logger<never, boolean>;

worker/dist/util/logger.js

@@ -0,0 +1,12 @@
+import pino from "pino";
+import { config } from "./config.js";
+export const logger = pino({
+    level: config.logLevel,
+    transport: config.logLevel === "debug"
+        ? { target: "pino/file", options: { destination: 1 } }
+        : undefined,
+});
+export function childLogger(name, extra) {
+    return logger.child({ module: name, ...extra });
+}
+//# sourceMappingURL=logger.js.map

worker/dist/util/logger.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"logger.js","sourceRoot":"","sources":["../../src/util/logger.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC,MAAM,CAAC,MAAM,MAAM,GAAG,IAAI,CAAC;IACzB,KAAK,EAAE,MAAM,CAAC,QAAQ;IACtB,SAAS,EACP,MAAM,CAAC,QAAQ,KAAK,OAAO;QACzB,CAAC,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,EAAE;QACtD,CAAC,CAAC,SAAS;CAChB,CAAC,CAAC;AAEH,MAAM,UAAU,WAAW,CAAC,IAAY,EAAE,KAA+B;IACvE,OAAO,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,KAAK,EAAE,CAAC,CAAC;AAClD,CAAC"}

worker/dist/util/mutex.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Ensures only one TDLib client runs at a time across the entire worker process.
+ * Both the scheduler (auth, ingestion) and the fetch listener acquire this
+ * before creating any TDLib client.
+ *
+ * Includes a wait timeout to prevent indefinite blocking if the current holder hangs.
+ */
+export declare function withTdlibMutex<T>(label: string, fn: () => Promise<T>): Promise<T>;

worker/dist/util/mutex.js

@@ -0,0 +1,61 @@
+import { childLogger } from "./logger.js";
+const log = childLogger("mutex");
+let locked = false;
+let holder = "";
+const queue = [];
+/**
+ * Maximum time to wait for the TDLib mutex (ms).
+ * If the mutex is not available within this time, the operation is rejected.
+ * Default: 30 minutes (long enough for large downloads, short enough to detect hangs).
+ */
+const MUTEX_WAIT_TIMEOUT_MS = 30 * 60 * 1000;
+/**
+ * Ensures only one TDLib client runs at a time across the entire worker process.
+ * Both the scheduler (auth, ingestion) and the fetch listener acquire this
+ * before creating any TDLib client.
+ *
+ * Includes a wait timeout to prevent indefinite blocking if the current holder hangs.
+ */
+export async function withTdlibMutex(label, fn) {
+    if (locked) {
+        log.info({ waiting: label, holder }, "Waiting for TDLib mutex");
+        await new Promise((resolve, reject) => {
+            const entry = { resolve, reject, label };
+            queue.push(entry);
+            // Timeout: reject if we've been waiting too long
+            const timer = setTimeout(() => {
+                const idx = queue.indexOf(entry);
+                if (idx !== -1) {
+                    queue.splice(idx, 1);
+                    reject(new Error(`TDLib mutex wait timeout after ${MUTEX_WAIT_TIMEOUT_MS / 60_000}min ` +
+                        `(waiting: ${label}, holder: ${holder})`));
+                }
+            }, MUTEX_WAIT_TIMEOUT_MS);
+            // Wrap resolve to clear the timer
+            const origResolve = entry.resolve;
+            entry.resolve = () => {
+                clearTimeout(timer);
+                origResolve();
+            };
+        });
+    }
+    locked = true;
+    holder = label;
+    log.debug({ label }, "TDLib mutex acquired");
+    try {
+        return await fn();
+    }
+    finally {
+        locked = false;
+        holder = "";
+        const next = queue.shift();
+        if (next) {
+            log.debug({ next: next.label }, "TDLib mutex releasing to next waiter");
+            next.resolve();
+        }
+        else {
+            log.debug({ label }, "TDLib mutex released");
+        }
+    }
+}
+//# sourceMappingURL=mutex.js.map

worker/dist/util/mutex.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mutex.js","sourceRoot":"","sources":["../../src/util/mutex.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAE1C,MAAM,GAAG,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;AAEjC,IAAI,MAAM,GAAG,KAAK,CAAC;AACnB,IAAI,MAAM,GAAG,EAAE,CAAC;AAChB,MAAM,KAAK,GAAgF,EAAE,CAAC;AAE9F;;;;GAIG;AACH,MAAM,qBAAqB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAE7C;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,KAAa,EACb,EAAoB;IAEpB,IAAI,MAAM,EAAE,CAAC;QACX,GAAG,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,yBAAyB,CAAC,CAAC;QAChE,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC1C,MAAM,KAAK,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;YACzC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAElB,iDAAiD;YACjD,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;gBAC5B,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;gBACjC,IAAI,GAAG,KAAK,CAAC,CAAC,EAAE,CAAC;oBACf,KAAK,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;oBACrB,MAAM,CAAC,IAAI,KAAK,CACd,kCAAkC,qBAAqB,GAAG,MAAM,MAAM;wBACtE,aAAa,KAAK,aAAa,MAAM,GAAG,CACzC,CAAC,CAAC;gBACL,CAAC;YACH,CAAC,EAAE,qBAAqB,CAAC,CAAC;YAE1B,kCAAkC;YAClC,MAAM,WAAW,GAAG,KAAK,CAAC,OAAO,CAAC;YAClC,KAAK,CAAC,OAAO,GAAG,GAAG,EAAE;gBACnB,YAAY,CAAC,KAAK,CAAC,CAAC;gBACpB,WAAW,EAAE,CAAC;YAChB,CAAC,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IAED,MAAM,GAAG,IAAI,CAAC;IACd,MAAM,GAAG,KAAK,CAAC;IACf,GAAG,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,EAAE,sBAAsB,CAAC,CAAC;IAE7C,IAAI,CAAC;QACH,OAAO,MAAM,EAAE,EAAE,CAAC;IACpB,CAAC;YAAS,CAAC;QACT,MAAM,GAAG,KAAK,CAAC;QACf,MAAM,GAAG,EAAE,CAAC;QACZ,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;QAC3B,IAAI,IAAI,EAAE,CAAC;YACT,GAAG,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,KAAK,EAAE,EAAE,sCAAsC,CAAC,CAAC;YACxE,IAAI,CAAC,OAAO,EAAE,CAAC;QACjB,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,EAAE,sBAAsB,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;AACH,CAAC"}

worker/dist/worker.d.ts

@@ -0,0 +1,28 @@
+import type { TelegramAccount } from "@prisma/client";
+/**
+ * Authenticate a PENDING account by creating a TDLib client.
+ * TDLib will send an SMS code to the phone number, and the client.login()
+ * callbacks set the authState to AWAITING_CODE. Once the admin enters the
+ * code via the UI, pollForAuthCode picks it up and completes the login.
+ *
+ * After successful auth:
+ * 1. Fetches channels from Telegram and writes as a ChannelFetchRequest
+ *    (so the admin can select sources in the UI)
+ * 2. Auto-joins the destination group if an invite link is configured
+ */
+export declare function authenticateAccount(account: TelegramAccount): Promise<void>;
+/**
+ * Process a ChannelFetchRequest: fetch channels from Telegram,
+ * enrich with DB state, and write the result JSON.
+ * Called by the fetch listener (pg_notify) and by authenticateAccount.
+ */
+export declare function processFetchRequest(requestId: string): Promise<void>;
+/**
+ * Run a full ingestion cycle for a single Telegram account.
+ * Every step writes live activity to the DB so the admin UI can display it.
+ */
+export declare function runWorkerForAccount(account: TelegramAccount): Promise<void>;
+/**
+ * Clean up any leftover temp files/directories from previous runs.
+ */
+export declare function cleanupTempDir(): Promise<void>;

worker/dist/worker.js

@@ -0,0 +1,745 @@
+import path from "path";
+import { unlink, readdir, mkdir, rm } from "fs/promises";
+import { config } from "./util/config.js";
+import { childLogger } from "./util/logger.js";
+import { tryAcquireLock, releaseLock } from "./db/locks.js";
+import { getSourceChannelMappings, getGlobalDestinationChannel, packageExistsByHash, packageExistsBySourceMessage, createPackageWithFiles, createIngestionRun, completeIngestionRun, failIngestionRun, updateLastProcessedMessage, updateRunActivity, setChannelForum, getTopicProgress, upsertTopicProgress, upsertChannel, ensureAccountChannelLink, getGlobalSetting, getChannelFetchRequest, updateFetchRequestStatus, getAccountLinkedChannelIds, getExistingChannelsByTelegramId, deleteOrphanedPackageByHash, } from "./db/queries.js";
+import { createTdlibClient, closeTdlibClient } from "./tdlib/client.js";
+import { getAccountChats, joinChatByInviteLink } from "./tdlib/chats.js";
+import { getChannelMessages, downloadFile, downloadPhotoThumbnail } from "./tdlib/download.js";
+import { isChatForum, getForumTopicList, getTopicMessages } from "./tdlib/topics.js";
+import { matchPreviewToArchive } from "./preview/match.js";
+import { groupArchiveSets } from "./archive/multipart.js";
+import { extractCreatorFromFileName } from "./archive/creator.js";
+import { hashParts } from "./archive/hash.js";
+import { readZipCentralDirectory } from "./archive/zip-reader.js";
+import { readRarContents } from "./archive/rar-reader.js";
+import { byteLevelSplit, concatenateFiles } from "./archive/split.js";
+import { uploadToChannel } from "./upload/channel.js";
+const log = childLogger("worker");
+/**
+ * Authenticate a PENDING account by creating a TDLib client.
+ * TDLib will send an SMS code to the phone number, and the client.login()
+ * callbacks set the authState to AWAITING_CODE. Once the admin enters the
+ * code via the UI, pollForAuthCode picks it up and completes the login.
+ *
+ * After successful auth:
+ * 1. Fetches channels from Telegram and writes as a ChannelFetchRequest
+ *    (so the admin can select sources in the UI)
+ * 2. Auto-joins the destination group if an invite link is configured
+ */
+export async function authenticateAccount(account) {
+    const aLog = childLogger("auth", { accountId: account.id, phone: account.phone });
+    aLog.info("Starting authentication flow");
+    let client;
+    try {
+        client = await createTdlibClient({
+            id: account.id,
+            phone: account.phone,
+        });
+        aLog.info("Authentication successful");
+        // Auto-fetch channels and create a fetch request result
+        aLog.info("Fetching channels from Telegram...");
+        await createAutoFetchRequest(client, account.id, aLog);
+        // Auto-join the destination group if an invite link exists
+        const inviteLink = await getGlobalSetting("destination_invite_link");
+        if (inviteLink) {
+            aLog.info("Attempting to join destination group via invite link...");
+            try {
+                await joinChatByInviteLink(client, inviteLink);
+                // Link this account as WRITER to the destination channel
+                const destChannel = await getGlobalDestinationChannel();
+                if (destChannel) {
+                    await ensureAccountChannelLink(account.id, destChannel.id, "WRITER");
+                    aLog.info({ destChannel: destChannel.title }, "Joined destination group and linked as WRITER");
+                }
+            }
+            catch (err) {
+                // May already be a member — that's fine
+                aLog.warn({ err }, "Could not join destination group (may already be a member)");
+                // Still try to link as WRITER
+                const destChannel = await getGlobalDestinationChannel();
+                if (destChannel) {
+                    await ensureAccountChannelLink(account.id, destChannel.id, "WRITER");
+                }
+            }
+        }
+    }
+    catch (err) {
+        aLog.error({ err }, "Authentication failed");
+    }
+    finally {
+        if (client) {
+            await closeTdlibClient(client);
+        }
+    }
+}
+/**
+ * Process a ChannelFetchRequest: fetch channels from Telegram,
+ * enrich with DB state, and write the result JSON.
+ * Called by the fetch listener (pg_notify) and by authenticateAccount.
+ */
+export async function processFetchRequest(requestId) {
+    const aLog = childLogger("fetch-request", { requestId });
+    const request = await getChannelFetchRequest(requestId);
+    if (!request || request.status !== "PENDING") {
+        aLog.warn("Fetch request not found or not pending, skipping");
+        return;
+    }
+    await updateFetchRequestStatus(requestId, "IN_PROGRESS");
+    aLog.info({ accountId: request.accountId }, "Processing fetch request");
+    const client = await createTdlibClient({
+        id: request.account.id,
+        phone: request.account.phone,
+    });
+    try {
+        const chats = await getAccountChats(client);
+        // Enrich with DB state
+        const linkedTelegramIds = await getAccountLinkedChannelIds(request.accountId);
+        const existingChannels = await getExistingChannelsByTelegramId();
+        const enrichedChats = chats.map((chat) => {
+            const telegramIdStr = chat.chatId.toString();
+            return {
+                chatId: telegramIdStr,
+                title: chat.title,
+                type: chat.type,
+                isForum: chat.isForum,
+                memberCount: chat.memberCount ?? null,
+                alreadyLinked: linkedTelegramIds.has(telegramIdStr),
+                existingChannelId: existingChannels.get(telegramIdStr) ?? null,
+            };
+        });
+        // Also upsert channel metadata while we have the data
+        for (const chat of chats) {
+            try {
+                await upsertChannel({
+                    telegramId: chat.chatId,
+                    title: chat.title,
+                    type: "SOURCE",
+                    isForum: chat.isForum,
+                });
+            }
+            catch {
+                // Non-critical — metadata sync can fail silently
+            }
+        }
+        await updateFetchRequestStatus(requestId, "COMPLETED", {
+            resultJson: JSON.stringify(enrichedChats),
+        });
+        aLog.info({ total: chats.length, linked: [...linkedTelegramIds].length }, "Fetch request completed");
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        aLog.error({ err }, "Fetch request failed");
+        await updateFetchRequestStatus(requestId, "FAILED", { error: message });
+    }
+    finally {
+        await closeTdlibClient(client);
+    }
+}
+/**
+ * Internal helper called after authentication to auto-create a fetch request
+ * with the channel list (so the UI can show the picker immediately).
+ */
+async function createAutoFetchRequest(client, accountId, aLog) {
+    const chats = await getAccountChats(client);
+    const linkedTelegramIds = await getAccountLinkedChannelIds(accountId);
+    const existingChannels = await getExistingChannelsByTelegramId();
+    const enrichedChats = chats.map((chat) => {
+        const telegramIdStr = chat.chatId.toString();
+        return {
+            chatId: telegramIdStr,
+            title: chat.title,
+            type: chat.type,
+            isForum: chat.isForum,
+            memberCount: chat.memberCount ?? null,
+            alreadyLinked: linkedTelegramIds.has(telegramIdStr),
+            existingChannelId: existingChannels.get(telegramIdStr) ?? null,
+        };
+    });
+    // Upsert channel metadata
+    for (const chat of chats) {
+        try {
+            await upsertChannel({
+                telegramId: chat.chatId,
+                title: chat.title,
+                type: "SOURCE",
+                isForum: chat.isForum,
+            });
+        }
+        catch {
+            // Non-critical
+        }
+    }
+    // Create the fetch request record with the result already filled in
+    const { db } = await import("./db/client.js");
+    await db.channelFetchRequest.create({
+        data: {
+            accountId,
+            status: "COMPLETED",
+            resultJson: JSON.stringify(enrichedChats),
+        },
+    });
+    aLog.info({ total: chats.length }, "Auto-fetch request created with channel list");
+}
+/**
+ * Throttle DB writes for download progress to avoid hammering the DB.
+ * Only writes if at least 2 seconds have passed since the last write.
+ */
+function createThrottledActivityUpdater(runId, minIntervalMs = 2000) {
+    let lastWriteTime = 0;
+    let pendingUpdate = null;
+    let flushTimer = null;
+    const flush = async () => {
+        if (pendingUpdate) {
+            const update = pendingUpdate;
+            pendingUpdate = null;
+            lastWriteTime = Date.now();
+            await updateRunActivity(runId, update).catch(() => { });
+        }
+    };
+    return {
+        update: (activity) => {
+            pendingUpdate = activity;
+            const elapsed = Date.now() - lastWriteTime;
+            if (elapsed >= minIntervalMs) {
+                if (flushTimer)
+                    clearTimeout(flushTimer);
+                flush();
+            }
+            else if (!flushTimer) {
+                flushTimer = setTimeout(() => {
+                    flushTimer = null;
+                    flush();
+                }, minIntervalMs - elapsed);
+            }
+        },
+        flush,
+    };
+}
+/**
+ * Run a full ingestion cycle for a single Telegram account.
+ * Every step writes live activity to the DB so the admin UI can display it.
+ */
+export async function runWorkerForAccount(account) {
+    const accountLog = childLogger("worker", { accountId: account.id, phone: account.phone });
+    // 1. Acquire advisory lock
+    const acquired = await tryAcquireLock(account.id);
+    if (!acquired) {
+        accountLog.info("Account already locked, skipping");
+        return;
+    }
+    let runId;
+    try {
+        // 2. Create ingestion run
+        const run = await createIngestionRun(account.id);
+        runId = run.id;
+        const activeRunId = runId;
+        accountLog.info({ runId }, "Ingestion run started");
+        const throttled = createThrottledActivityUpdater(activeRunId);
+        // 3. Initialize TDLib client
+        await updateRunActivity(activeRunId, {
+            currentActivity: "Connecting to Telegram",
+            currentStep: "connecting",
+        });
+        const client = await createTdlibClient({
+            id: account.id,
+            phone: account.phone,
+        });
+        const counters = {
+            messagesScanned: 0,
+            zipsFound: 0,
+            zipsDuplicate: 0,
+            zipsIngested: 0,
+        };
+        try {
+            // 4. Get assigned source channels and global destination
+            const channelMappings = await getSourceChannelMappings(account.id);
+            const destChannel = await getGlobalDestinationChannel();
+            if (!destChannel) {
+                throw new Error("No global destination channel configured — set one in the admin UI");
+            }
+            const totalChannels = channelMappings.length;
+            for (let chIdx = 0; chIdx < channelMappings.length; chIdx++) {
+                const mapping = channelMappings[chIdx];
+                const channel = mapping.channel;
+                const channelLabel = totalChannels > 1
+                    ? `[${chIdx + 1}/${totalChannels}] ${channel.title}`
+                    : channel.title;
+                try {
+                    // ── Check if channel is a forum ──
+                    const forum = await isChatForum(client, channel.telegramId);
+                    if (forum !== channel.isForum) {
+                        await setChannelForum(channel.id, forum);
+                        accountLog.info({ channelId: channel.id, title: channel.title, isForum: forum }, "Updated channel forum status");
+                    }
+                    const pipelineCtx = {
+                        client,
+                        runId: activeRunId,
+                        channelTitle: channel.title,
+                        channel,
+                        destChannelTelegramId: destChannel.telegramId,
+                        destChannelId: destChannel.id,
+                        throttled,
+                        counters,
+                        topicCreator: null,
+                        sourceTopicId: null,
+                        accountLog,
+                    };
+                    if (forum) {
+                        // ── Forum channel: scan per-topic ──
+                        await updateRunActivity(activeRunId, {
+                            currentActivity: `Enumerating topics in "${channelLabel}"`,
+                            currentStep: "scanning",
+                            currentChannel: channelLabel,
+                            currentFile: null,
+                            currentFileNum: null,
+                            totalFiles: null,
+                            downloadedBytes: null,
+                            totalBytes: null,
+                            downloadPercent: null,
+                            messagesScanned: counters.messagesScanned,
+                        });
+                        const topics = await getForumTopicList(client, channel.telegramId);
+                        const topicProgressList = await getTopicProgress(mapping.id);
+                        accountLog.info({ channelId: channel.id, title: channel.title, topicCount: topics.length }, "Scanning forum channel by topic");
+                        for (let tIdx = 0; tIdx < topics.length; tIdx++) {
+                            const topic = topics[tIdx];
+                            try {
+                                const progress = topicProgressList.find((tp) => tp.topicId === topic.topicId);
+                                const topicLabel = `${channel.title} › ${topic.name}`;
+                                const topicProgress = topics.length > 1
+                                    ? ` (topic ${tIdx + 1}/${topics.length})`
+                                    : "";
+                                await updateRunActivity(activeRunId, {
+                                    currentActivity: `Scanning "${topicLabel}"${topicProgress}`,
+                                    currentStep: "scanning",
+                                    currentChannel: channelLabel,
+                                    currentFile: null,
+                                    currentFileNum: null,
+                                    totalFiles: null,
+                                    downloadedBytes: null,
+                                    totalBytes: null,
+                                    downloadPercent: null,
+                                    messagesScanned: counters.messagesScanned,
+                                });
+                                const scanResult = await getTopicMessages(client, channel.telegramId, topic.topicId, progress?.lastProcessedMessageId, 100, (scanned) => {
+                                    throttled.update({
+                                        currentActivity: `Scanning "${topicLabel}"${topicProgress} — ${scanned} messages scanned`,
+                                        currentStep: "scanning",
+                                        currentChannel: channelLabel,
+                                        messagesScanned: counters.messagesScanned + scanned,
+                                    });
+                                });
+                                // Add scanned messages to global counter
+                                counters.messagesScanned += scanResult.totalScanned;
+                                if (scanResult.archives.length === 0) {
+                                    accountLog.debug({ channelId: channel.id, topic: topic.name }, "No new archives in topic");
+                                    continue;
+                                }
+                                accountLog.info({ topic: topic.name, archives: scanResult.archives.length, photos: scanResult.photos.length }, "Found messages in topic");
+                                // Process archives with topic creator
+                                pipelineCtx.topicCreator = topic.name;
+                                pipelineCtx.sourceTopicId = topic.topicId;
+                                pipelineCtx.channelTitle = `${channel.title} › ${topic.name}`;
+                                const maxProcessedId = await processArchiveSets(pipelineCtx, scanResult, run.id, progress?.lastProcessedMessageId);
+                                // Only advance progress to the highest successfully processed message
+                                if (maxProcessedId) {
+                                    await upsertTopicProgress(mapping.id, topic.topicId, topic.name, maxProcessedId);
+                                }
+                            }
+                            catch (topicErr) {
+                                accountLog.warn({ err: topicErr, channelId: channel.id, topic: topic.name, topicId: topic.topicId.toString() }, "Failed to process topic, skipping");
+                            }
+                        }
+                    }
+                    else {
+                        // ── Non-forum channel: flat scan (existing behavior) ──
+                        await updateRunActivity(activeRunId, {
+                            currentActivity: `Scanning "${channelLabel}" for new archives`,
+                            currentStep: "scanning",
+                            currentChannel: channelLabel,
+                            currentFile: null,
+                            currentFileNum: null,
+                            totalFiles: null,
+                            downloadedBytes: null,
+                            totalBytes: null,
+                            downloadPercent: null,
+                            messagesScanned: counters.messagesScanned,
+                        });
+                        accountLog.info({ channelId: channel.id, title: channel.title }, "Processing source channel");
+                        const scanResult = await getChannelMessages(client, channel.telegramId, mapping.lastProcessedMessageId, 100, (scanned) => {
+                            throttled.update({
+                                currentActivity: `Scanning "${channelLabel}" — ${scanned} messages scanned`,
+                                currentStep: "scanning",
+                                currentChannel: channelLabel,
+                                messagesScanned: counters.messagesScanned + scanned,
+                            });
+                        });
+                        // Add scanned messages to global counter
+                        counters.messagesScanned += scanResult.totalScanned;
+                        if (scanResult.archives.length === 0) {
+                            accountLog.debug({ channelId: channel.id }, "No new archives");
+                            continue;
+                        }
+                        accountLog.info({ archives: scanResult.archives.length, photos: scanResult.photos.length }, "Found messages in channel");
+                        // For non-forum, creator comes from filename (set to null, resolved per-archive)
+                        pipelineCtx.topicCreator = null;
+                        pipelineCtx.sourceTopicId = null;
+                        pipelineCtx.channelTitle = channel.title;
+                        const maxProcessedId = await processArchiveSets(pipelineCtx, scanResult, run.id, mapping.lastProcessedMessageId);
+                        // Only advance progress to the highest successfully processed message
+                        if (maxProcessedId) {
+                            await updateLastProcessedMessage(mapping.id, maxProcessedId);
+                        }
+                    }
+                }
+                catch (channelErr) {
+                    accountLog.warn({ err: channelErr, channelId: channel.id, title: channel.title }, "Failed to process channel, skipping to next");
+                }
+            }
+            // ── Done ──
+            await completeIngestionRun(activeRunId, counters);
+            accountLog.info({ counters }, "Ingestion run completed");
+        }
+        finally {
+            await closeTdlibClient(client);
+        }
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        accountLog.error({ err }, "Ingestion run failed");
+        if (runId) {
+            await failIngestionRun(runId, message).catch((e) => accountLog.error({ e }, "Failed to mark run as failed"));
+        }
+    }
+    finally {
+        await releaseLock(account.id);
+    }
+}
+/**
+ * Process a scan result through the archive pipeline:
+ * group → download → hash → dedup → metadata → split → upload → preview → index.
+ *
+ * Returns the highest message ID that was successfully processed (ingested or
+ * confirmed duplicate). The caller should only advance the progress boundary
+ * to this value — never to the max of all scanned messages.
+ */
+async function processArchiveSets(ctx, scanResult, ingestionRunId, lastProcessedMessageId) {
+    const { client, runId, channelTitle, channel, throttled, counters, accountLog } = ctx;
+    // Group into archive sets
+    let archiveSets = groupArchiveSets(scanResult.archives);
+    // Filter out sets where ALL parts are at or below the boundary (already processed)
+    if (lastProcessedMessageId) {
+        const totalBefore = archiveSets.length;
+        archiveSets = archiveSets.filter((set) => set.parts.some((p) => p.id > lastProcessedMessageId));
+        const filtered = totalBefore - archiveSets.length;
+        if (filtered > 0) {
+            accountLog.info({ filtered, remaining: archiveSets.length }, "Filtered out already-processed archive sets");
+        }
+    }
+    counters.zipsFound += archiveSets.length;
+    // Match preview photos to archive sets
+    const previewMatches = matchPreviewToArchive(scanResult.photos, archiveSets.map((s) => ({
+        baseName: s.baseName,
+        firstMessageId: s.parts[0].id,
+        firstMessageDate: s.parts[0].date,
+    })));
+    if (previewMatches.size > 0) {
+        accountLog.info({ matched: previewMatches.size, total: archiveSets.length }, "Matched preview photos to archives");
+    }
+    await updateRunActivity(runId, {
+        currentActivity: `Found ${archiveSets.length} archive(s) in "${channelTitle}"`,
+        currentStep: "scanning",
+        currentChannel: channelTitle,
+        totalFiles: archiveSets.length,
+        zipsFound: counters.zipsFound,
+        messagesScanned: counters.messagesScanned,
+    });
+    // Track the highest message ID that was successfully processed
+    let maxProcessedId = null;
+    for (let setIdx = 0; setIdx < archiveSets.length; setIdx++) {
+        try {
+            await processOneArchiveSet(ctx, archiveSets[setIdx], setIdx, archiveSets.length, previewMatches, ingestionRunId);
+            // Set completed (ingested or confirmed duplicate) — advance watermark
+            const setMaxId = archiveSets[setIdx].parts.reduce((max, p) => (p.id > max ? p.id : max), 0n);
+            if (setMaxId > (maxProcessedId ?? 0n)) {
+                maxProcessedId = setMaxId;
+            }
+        }
+        catch (setErr) {
+            // If a set fails, do NOT advance the watermark past it
+            accountLog.warn({ err: setErr, baseName: archiveSets[setIdx].baseName }, "Archive set failed, watermark will not advance past this set");
+        }
+    }
+    return maxProcessedId;
+}
+/**
+ * Process a single archive set through the full pipeline.
+ */
+async function processOneArchiveSet(ctx, archiveSet, setIdx, totalSets, previewMatches, ingestionRunId) {
+    const { client, runId, channelTitle, channel, destChannelTelegramId, destChannelId, throttled, counters, topicCreator, sourceTopicId, accountLog, } = ctx;
+    const archiveName = archiveSet.parts[0].fileName;
+    // ── Early skip: check if this archive set was already ingested ──
+    // This avoids re-downloading large archives that were processed in a prior run.
+    const alreadyIngested = await packageExistsBySourceMessage(channel.id, archiveSet.parts[0].id);
+    if (alreadyIngested) {
+        counters.zipsDuplicate++;
+        accountLog.debug({ fileName: archiveName, sourceMessageId: Number(archiveSet.parts[0].id) }, "Archive already ingested (by source message), skipping");
+        await updateRunActivity(runId, {
+            currentActivity: `Skipped ${archiveName} (already ingested)`,
+            currentStep: "deduplicating",
+            currentChannel: channelTitle,
+            currentFile: archiveName,
+            currentFileNum: setIdx + 1,
+            totalFiles: totalSets,
+            zipsDuplicate: counters.zipsDuplicate,
+        });
+        return;
+    }
+    const tempPaths = [];
+    let splitPaths = [];
+    // Per-set subdirectory so uploaded files keep their original filenames
+    const setDir = path.join(config.tempDir, `${ingestionRunId}_${archiveSet.parts[0].id}`);
+    await mkdir(setDir, { recursive: true });
+    try {
+        // ── Downloading ──
+        for (let partIdx = 0; partIdx < archiveSet.parts.length; partIdx++) {
+            const part = archiveSet.parts[partIdx];
+            const tempPath = path.join(setDir, part.fileName);
+            const partLabel = archiveSet.parts.length > 1
+                ? ` (part ${partIdx + 1}/${archiveSet.parts.length})`
+                : "";
+            await updateRunActivity(runId, {
+                currentActivity: `Downloading ${part.fileName}${partLabel}`,
+                currentStep: "downloading",
+                currentChannel: channelTitle,
+                currentFile: part.fileName,
+                currentFileNum: setIdx + 1,
+                totalFiles: totalSets,
+                downloadedBytes: 0n,
+                totalBytes: part.fileSize,
+                downloadPercent: 0,
+                messagesScanned: counters.messagesScanned,
+            });
+            accountLog.info({
+                fileName: part.fileName,
+                fileSize: Number(part.fileSize),
+                part: partIdx + 1,
+                totalParts: archiveSet.parts.length,
+            }, "Downloading archive part");
+            await downloadFile(client, part.fileId, tempPath, part.fileSize, part.fileName, (progress) => {
+                throttled.update({
+                    currentActivity: `Downloading ${part.fileName}${partLabel} — ${progress.percent}%`,
+                    currentStep: "downloading",
+                    currentChannel: channelTitle,
+                    currentFile: part.fileName,
+                    currentFileNum: setIdx + 1,
+                    totalFiles: totalSets,
+                    downloadedBytes: BigInt(progress.downloadedBytes),
+                    totalBytes: BigInt(progress.totalBytes),
+                    downloadPercent: progress.percent,
+                });
+            });
+            await throttled.flush();
+            tempPaths.push(tempPath);
+        }
+        // ── Hashing ──
+        await updateRunActivity(runId, {
+            currentActivity: `Computing hash for ${archiveName}`,
+            currentStep: "hashing",
+            currentChannel: channelTitle,
+            currentFile: archiveName,
+            currentFileNum: setIdx + 1,
+            totalFiles: totalSets,
+            downloadedBytes: null,
+            totalBytes: null,
+            downloadPercent: null,
+        });
+        const contentHash = await hashParts(tempPaths);
+        // ── Deduplicating ──
+        await updateRunActivity(runId, {
+            currentActivity: `Checking if ${archiveName} is a duplicate`,
+            currentStep: "deduplicating",
+            currentChannel: channelTitle,
+            currentFile: archiveName,
+            currentFileNum: setIdx + 1,
+            totalFiles: totalSets,
+        });
+        const exists = await packageExistsByHash(contentHash);
+        if (exists) {
+            counters.zipsDuplicate++;
+            accountLog.debug({ contentHash }, "Duplicate archive, skipping");
+            await updateRunActivity(runId, {
+                currentActivity: `Skipped ${archiveName} (duplicate)`,
+                currentStep: "deduplicating",
+                currentChannel: channelTitle,
+                currentFile: archiveName,
+                currentFileNum: setIdx + 1,
+                totalFiles: totalSets,
+                zipsDuplicate: counters.zipsDuplicate,
+            });
+            return;
+        }
+        // ── Reading metadata ──
+        await updateRunActivity(runId, {
+            currentActivity: `Reading file list from ${archiveName}`,
+            currentStep: "reading_metadata",
+            currentChannel: channelTitle,
+            currentFile: archiveName,
+            currentFileNum: setIdx + 1,
+            totalFiles: totalSets,
+        });
+        let entries = [];
+        try {
+            if (archiveSet.type === "ZIP") {
+                entries = await readZipCentralDirectory(tempPaths);
+            }
+            else {
+                entries = await readRarContents(tempPaths[0]);
+            }
+        }
+        catch (err) {
+            accountLog.warn({ err, baseName: archiveSet.baseName }, "Failed to read archive metadata, ingesting without file list");
+        }
+        // ── Splitting / Repacking (if needed) ──
+        let uploadPaths = [...tempPaths];
+        const totalSize = archiveSet.parts.reduce((sum, p) => sum + p.fileSize, 0n);
+        const MAX_UPLOAD_SIZE = 2n * 1024n * 1024n * 1024n;
+        const hasOversizedPart = archiveSet.parts.some((p) => p.fileSize > MAX_UPLOAD_SIZE);
+        if (hasOversizedPart) {
+            // Full repack: concatenate all parts → single file → re-split into uniform 2GB chunks
+            await updateRunActivity(runId, {
+                currentActivity: `Repacking ${archiveName} (parts >2GB, concatenating + re-splitting)`,
+                currentStep: "splitting",
+                currentChannel: channelTitle,
+                currentFile: archiveName,
+                currentFileNum: setIdx + 1,
+                totalFiles: totalSets,
+            });
+            const concatPath = path.join(setDir, `${archiveSet.baseName}.concat`);
+            await concatenateFiles(tempPaths, concatPath);
+            splitPaths = await byteLevelSplit(concatPath);
+            uploadPaths = splitPaths;
+            // Clean up the concat intermediate file
+            await unlink(concatPath).catch(() => { });
+        }
+        else if (!archiveSet.isMultipart && totalSize > MAX_UPLOAD_SIZE) {
+            // Single file >2GB: split directly
+            await updateRunActivity(runId, {
+                currentActivity: `Splitting ${archiveName} for upload (>2GB)`,
+                currentStep: "splitting",
+                currentChannel: channelTitle,
+                currentFile: archiveName,
+                currentFileNum: setIdx + 1,
+                totalFiles: totalSets,
+            });
+            splitPaths = await byteLevelSplit(tempPaths[0]);
+            uploadPaths = splitPaths;
+        }
+        // ── Uploading ──
+        const uploadLabel = uploadPaths.length > 1
+            ? ` (${uploadPaths.length} parts)`
+            : "";
+        await updateRunActivity(runId, {
+            currentActivity: `Uploading ${archiveName} to archive channel${uploadLabel}`,
+            currentStep: "uploading",
+            currentChannel: channelTitle,
+            currentFile: archiveName,
+            currentFileNum: setIdx + 1,
+            totalFiles: totalSets,
+        });
+        const destResult = await uploadToChannel(client, destChannelTelegramId, uploadPaths);
+        // ── Preview thumbnail ──
+        let previewData = null;
+        let previewMsgId = null;
+        const matchedPhoto = previewMatches.get(archiveSet.baseName);
+        if (matchedPhoto) {
+            await updateRunActivity(runId, {
+                currentActivity: `Downloading preview image for ${archiveName}`,
+                currentStep: "preview",
+                currentChannel: channelTitle,
+                currentFile: archiveName,
+                currentFileNum: setIdx + 1,
+                totalFiles: totalSets,
+            });
+            previewData = await downloadPhotoThumbnail(client, matchedPhoto.fileId);
+            previewMsgId = matchedPhoto.id;
+        }
+        // ── Resolve creator: topic name > filename extraction > null ──
+        const creator = topicCreator ?? extractCreatorFromFileName(archiveName) ?? null;
+        // ── Indexing ──
+        await updateRunActivity(runId, {
+            currentActivity: `Saving metadata for ${archiveName} (${entries.length} files)`,
+            currentStep: "indexing",
+            currentChannel: channelTitle,
+            currentFile: archiveName,
+            currentFileNum: setIdx + 1,
+            totalFiles: totalSets,
+        });
+        // Clean up any orphaned record (same hash but no dest upload) before creating
+        await deleteOrphanedPackageByHash(contentHash);
+        await createPackageWithFiles({
+            contentHash,
+            fileName: archiveName,
+            fileSize: totalSize,
+            archiveType: archiveSet.type,
+            sourceChannelId: channel.id,
+            sourceMessageId: archiveSet.parts[0].id,
+            sourceTopicId,
+            destChannelId,
+            destMessageId: destResult.messageId,
+            isMultipart: archiveSet.parts.length > 1 || uploadPaths.length > 1,
+            partCount: uploadPaths.length,
+            ingestionRunId,
+            creator,
+            previewData,
+            previewMsgId,
+            files: entries,
+        });
+        counters.zipsIngested++;
+        await updateRunActivity(runId, {
+            currentActivity: `Ingested ${archiveName} (${entries.length} files indexed)`,
+            currentStep: "complete",
+            currentChannel: channelTitle,
+            currentFile: archiveName,
+            currentFileNum: setIdx + 1,
+            totalFiles: totalSets,
+            zipsIngested: counters.zipsIngested,
+        });
+        accountLog.info({ fileName: archiveName, contentHash, fileCount: entries.length, creator }, "Archive ingested");
+    }
+    finally {
+        // ALWAYS delete temp files and the set directory
+        await deleteFiles([...tempPaths, ...splitPaths]);
+        await rm(setDir, { recursive: true, force: true }).catch(() => { });
+    }
+}
+async function deleteFiles(paths) {
+    for (const p of paths) {
+        try {
+            await unlink(p);
+        }
+        catch {
+            // File may already be deleted or never created
+        }
+    }
+}
+/**
+ * Clean up any leftover temp files/directories from previous runs.
+ */
+export async function cleanupTempDir() {
+    try {
+        const entries = await readdir(config.tempDir);
+        for (const entry of entries) {
+            await rm(path.join(config.tempDir, entry), { recursive: true, force: true }).catch(() => { });
+        }
+        if (entries.length > 0) {
+            log.info({ count: entries.length }, "Cleaned up stale temp files");
+        }
+    }
+    catch {
+        // Directory might not exist yet
+    }
+}
+//# sourceMappingURL=worker.js.map

worker/src/scheduler.ts

@@ -10,6 +10,13 @@ let running = false;
 let timer: ReturnType<typeof setTimeout> | null = null;
 let cycleCount = 0;
 
+/**
+ * Maximum time for a single ingestion cycle (ms).
+ * After this, new accounts won't be started (in-progress work finishes).
+ * Default: 4 hours. Configurable via WORKER_CYCLE_TIMEOUT_MINUTES.
+ */
+const CYCLE_TIMEOUT_MS = (parseInt(process.env.WORKER_CYCLE_TIMEOUT_MINUTES ?? "240", 10)) * 60 * 1000;
+
 /**
  * Run one ingestion cycle:
  * 1. Authenticate any PENDING accounts (triggers SMS code flow + auto-fetch channels)
@@ -17,6 +24,10 @@ let cycleCount = 0;
  *
  * All TDLib operations are wrapped in the mutex to ensure only one client
  * runs at a time (also shared with the fetch listener for on-demand requests).
+ *
+ * The cycle has a configurable timeout (WORKER_CYCLE_TIMEOUT_MINUTES, default 4h).
+ * Once the timeout elapses, no new accounts will be started but any in-progress
+ * account processing is allowed to finish its current archive set.
  */
 async function runCycle(): Promise<void> {
   if (running) {
@@ -26,7 +37,8 @@ async function runCycle(): Promise<void> {
 
   running = true;
   cycleCount++;
-  log.info({ cycle: cycleCount }, "Starting ingestion cycle");
+  const cycleStart = Date.now();
+  log.info({ cycle: cycleCount, timeoutMinutes: CYCLE_TIMEOUT_MS / 60_000 }, "Starting ingestion cycle");
 
   try {
     // ── Phase 1: Authenticate pending accounts ──
@@ -37,6 +49,10 @@ async function runCycle(): Promise<void> {
         "Found pending accounts, starting authentication"
       );
       for (const account of pendingAccounts) {
+        if (Date.now() - cycleStart > CYCLE_TIMEOUT_MS) {
+          log.warn("Cycle timeout reached during authentication phase, stopping");
+          break;
+        }
         await withTdlibMutex(`auth:${account.phone}`, () =>
           authenticateAccount(account)
         );
@@ -54,12 +70,22 @@ async function runCycle(): Promise<void> {
     log.info({ accountCount: accounts.length }, "Processing accounts");
 
     for (const account of accounts) {
+      if (Date.now() - cycleStart > CYCLE_TIMEOUT_MS) {
+        log.warn(
+          { elapsed: Math.round((Date.now() - cycleStart) / 60_000), timeoutMinutes: CYCLE_TIMEOUT_MS / 60_000 },
+          "Cycle timeout reached, skipping remaining accounts"
+        );
+        break;
+      }
       await withTdlibMutex(`ingest:${account.phone}`, () =>
         runWorkerForAccount(account)
       );
     }
 
-    log.info("Ingestion cycle complete");
+    log.info(
+      { elapsed: Math.round((Date.now() - cycleStart) / 1000) },
+      "Ingestion cycle complete"
+    );
   } catch (err) {
     log.error({ err }, "Ingestion cycle failed");
   } finally {

worker/src/tdlib/download.ts

@@ -8,6 +8,12 @@ import type { TelegramPhoto } from "../preview/match.js";
 
 const log = childLogger("download");
 
+/** Maximum number of pages to scan per channel/topic to prevent infinite loops */
+export const MAX_SCAN_PAGES = 5000;
+
+/** Timeout for a single TDLib API call (ms) */
+export const INVOKE_TIMEOUT_MS = 120_000; // 2 minutes
+
 interface TdPhotoSize {
   type: string;
   photo: {
@@ -71,6 +77,33 @@ export interface ChannelScanResult {
 
 export type ScanProgressCallback = (messagesScanned: number) => void;
 
+/**
+ * Invoke a TDLib method with a timeout to prevent indefinite hangs.
+ * If TDLib does not respond within the timeout, the promise rejects.
+ */
+export async function invokeWithTimeout<T>(
+  client: Client,
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  request: Record<string, any>,
+  timeoutMs = INVOKE_TIMEOUT_MS
+): Promise<T> {
+  return new Promise<T>((resolve, reject) => {
+    const timer = setTimeout(() => {
+      reject(new Error(`TDLib invoke timed out after ${timeoutMs}ms for ${request._}`));
+    }, timeoutMs);
+
+    (client.invoke(request) as Promise<T>)
+      .then((result) => {
+        clearTimeout(timer);
+        resolve(result);
+      })
+      .catch((err) => {
+        clearTimeout(timer);
+        reject(err);
+      });
+  });
+}
+
 /**
  * Fetch messages from a channel, stopping once we've scanned past the
  * last-processed boundary (with one page of lookback for multipart safety).
@@ -80,6 +113,11 @@ export type ScanProgressCallback = (messagesScanned: number) => void;
  * When `lastProcessedMessageId` is null (first run), scans everything.
  * The worker applies a post-grouping filter to skip fully-processed sets,
  * and keeps `packageExistsBySourceMessage` as a safety net.
+ *
+ * Safety features:
+ *  - Max page limit to prevent infinite loops
+ *  - Stuck detection: breaks if from_message_id stops advancing
+ *  - Timeout on each TDLib API call
  */
 export async function getChannelMessages(
   client: Client,
@@ -94,17 +132,29 @@ export async function getChannelMessages(
 
   let currentFromId = 0;
   let totalScanned = 0;
+  let pageCount = 0;
 
   // eslint-disable-next-line no-constant-condition
   while (true) {
-    const result = (await client.invoke({
+    if (pageCount >= MAX_SCAN_PAGES) {
+      log.warn(
+        { chatId: chatId.toString(), pageCount, totalScanned },
+        "Hit max page limit for channel scan, stopping"
+      );
+      break;
+    }
+    pageCount++;
+
+    const previousFromId = currentFromId;
+
+    const result = await invokeWithTimeout<{ messages: TdMessage[] }>(client, {
       _: "getChatHistory",
       chat_id: Number(chatId),
       from_message_id: currentFromId,
       offset: 0,
       limit: Math.min(limit, 100),
       only_local: false,
-    })) as { messages: TdMessage[] };
+    });
 
     if (!result.messages || result.messages.length === 0) break;
 
@@ -144,17 +194,26 @@ export async function getChannelMessages(
 
     currentFromId = result.messages[result.messages.length - 1].id;
 
+    // Stuck detection: if from_message_id didn't advance, break to prevent infinite loop
+    if (currentFromId === previousFromId) {
+      log.warn(
+        { chatId: chatId.toString(), currentFromId, totalScanned },
+        "Pagination stuck (from_message_id not advancing), breaking"
+      );
+      break;
+    }
+
     // Stop scanning once we've gone past the boundary (this page is the lookback)
     if (boundary && currentFromId < boundary) break;
 
-    if (result.messages.length < 100) break;
+    if (result.messages.length < Math.min(limit, 100)) break;
 
     // Rate limit delay
     await sleep(config.apiDelayMs);
   }
 
   log.info(
-    { chatId: chatId.toString(), archives: archives.length, photos: photos.length, totalScanned },
+    { chatId: chatId.toString(), archives: archives.length, photos: photos.length, totalScanned, pages: pageCount },
     "Channel scan complete"
   );
 

worker/src/tdlib/topics.ts

@@ -5,6 +5,7 @@ import { isArchiveAttachment } from "../archive/detect.js";
 import type { TelegramMessage } from "../archive/multipart.js";
 import type { TelegramPhoto } from "../preview/match.js";
 import type { ChannelScanResult, ScanProgressCallback } from "./download.js";
+import { invokeWithTimeout, MAX_SCAN_PAGES, INVOKE_TIMEOUT_MS } from "./download.js";
 
 const log = childLogger("topics");
 
@@ -21,16 +22,16 @@ export async function isChatForum(
   chatId: bigint
 ): Promise<boolean> {
   try {
-    const chat = (await client.invoke({
+    const chat = await invokeWithTimeout<{
-      _: "getChat",
-      chat_id: Number(chatId),
-    })) as {
       type?: {
         _: string;
         supergroup_id?: number;
         is_forum?: boolean;
       };
-    };
+    }>(client, {
+      _: "getChat",
+      chat_id: Number(chatId),
+    });
 
     if (chat.type?._ === "chatTypeSupergroup" && chat.type.is_forum) {
       return true;
@@ -38,10 +39,10 @@ export async function isChatForum(
 
     // Also check via getSupergroup for older TDLib versions
     if (chat.type?._ === "chatTypeSupergroup" && chat.type.supergroup_id) {
-      const sg = (await client.invoke({
+      const sg = await invokeWithTimeout<{ is_forum?: boolean }>(client, {
         _: "getSupergroup",
         supergroup_id: chat.type.supergroup_id,
-      })) as { is_forum?: boolean };
+      });
       return sg.is_forum === true;
     }
 
@@ -54,6 +55,7 @@ export async function isChatForum(
 
 /**
  * Get all forum topics in a supergroup.
+ * Includes stuck detection and timeout protection on API calls.
  */
 export async function getForumTopicList(
   client: Client,
@@ -63,18 +65,24 @@ export async function getForumTopicList(
   let offsetDate = 0;
   let offsetMessageId = 0;
   let offsetMessageThreadId = 0;
+  let pageCount = 0;
 
   // eslint-disable-next-line no-constant-condition
   while (true) {
-    const result = (await client.invoke({
+    if (pageCount >= MAX_SCAN_PAGES) {
-      _: "getForumTopics",
+      log.warn(
-      chat_id: Number(chatId),
+        { chatId: chatId.toString(), pageCount, topicCount: topics.length },
-      query: "",
+        "Hit max page limit for topic enumeration, stopping"
-      offset_date: offsetDate,
+      );
-      offset_message_id: offsetMessageId,
+      break;
-      offset_message_thread_id: offsetMessageThreadId,
+    }
-      limit: 100,
+    pageCount++;
-    })) as {
+
+    const prevOffsetDate = offsetDate;
+    const prevOffsetMessageId = offsetMessageId;
+    const prevOffsetMessageThreadId = offsetMessageThreadId;
+
+    const result = await invokeWithTimeout<{
       topics?: {
         info?: {
           message_thread_id?: number;
@@ -85,7 +93,15 @@ export async function getForumTopicList(
       next_offset_date?: number;
       next_offset_message_id?: number;
       next_offset_message_thread_id?: number;
-    };
+    }>(client, {
+      _: "getForumTopics",
+      chat_id: Number(chatId),
+      query: "",
+      offset_date: offsetDate,
+      offset_message_id: offsetMessageId,
+      offset_message_thread_id: offsetMessageThreadId,
+      limit: 100,
+    });
 
     if (!result.topics || result.topics.length === 0) break;
 
@@ -113,6 +129,19 @@ export async function getForumTopicList(
     offsetMessageId = result.next_offset_message_id ?? 0;
     offsetMessageThreadId = result.next_offset_message_thread_id ?? 0;
 
+    // Stuck detection: if offsets didn't advance, break
+    if (
+      offsetDate === prevOffsetDate &&
+      offsetMessageId === prevOffsetMessageId &&
+      offsetMessageThreadId === prevOffsetMessageThreadId
+    ) {
+      log.warn(
+        { chatId: chatId.toString(), topicCount: topics.length },
+        "Topic pagination stuck (offsets not advancing), breaking"
+      );
+      break;
+    }
+
     await sleep(config.apiDelayMs);
   }
 
@@ -134,6 +163,11 @@ export async function getForumTopicList(
  * When `lastProcessedMessageId` is null (first run), scans everything.
  * The worker applies a post-grouping filter to skip fully-processed sets,
  * and keeps `packageExistsBySourceMessage` as a safety net.
+ *
+ * Safety features:
+ *  - Max page limit to prevent infinite loops
+ *  - Stuck detection: breaks if from_message_id stops advancing
+ *  - Timeout on each TDLib API call
  */
 export async function getTopicMessages(
   client: Client,
@@ -149,22 +183,23 @@ export async function getTopicMessages(
 
   let currentFromId = 0;
   let totalScanned = 0;
+  let pageCount = 0;
 
   // eslint-disable-next-line no-constant-condition
   while (true) {
+    if (pageCount >= MAX_SCAN_PAGES) {
+      log.warn(
+        { chatId: chatId.toString(), topicId: topicId.toString(), pageCount, totalScanned },
+        "Hit max page limit for topic scan, stopping"
+      );
+      break;
+    }
+    pageCount++;
+
+    const previousFromId = currentFromId;
+
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    const result = (await client.invoke({
+    const result = await invokeWithTimeout<{
-      _: "searchChatMessages",
-      chat_id: Number(chatId),
-      query: "",
-      message_thread_id: Number(topicId),
-      from_message_id: currentFromId,
-      offset: 0,
-      limit: Math.min(limit, 100),
-      filter: null,
-      sender_id: null,
-      saved_messages_topic_id: 0,
-    })) as {
       messages?: {
         id: number;
         date: number;
@@ -188,7 +223,18 @@ export async function getTopicMessages(
           caption?: { text?: string };
         };
       }[];
-    };
+    }>(client, {
+      _: "searchChatMessages",
+      chat_id: Number(chatId),
+      query: "",
+      message_thread_id: Number(topicId),
+      from_message_id: currentFromId,
+      offset: 0,
+      limit: Math.min(limit, 100),
+      filter: null,
+      sender_id: null,
+      saved_messages_topic_id: 0,
+    });
 
     if (!result.messages || result.messages.length === 0) break;
 
@@ -228,16 +274,25 @@ export async function getTopicMessages(
 
     currentFromId = result.messages[result.messages.length - 1].id;
 
+    // Stuck detection: if from_message_id didn't advance, break to prevent infinite loop
+    if (currentFromId === previousFromId) {
+      log.warn(
+        { chatId: chatId.toString(), topicId: topicId.toString(), currentFromId, totalScanned },
+        "Topic pagination stuck (from_message_id not advancing), breaking"
+      );
+      break;
+    }
+
     // Stop scanning once we've gone past the boundary (this page is the lookback)
     if (boundary && currentFromId < boundary) break;
 
-    if (result.messages.length < 100) break;
+    if (result.messages.length < Math.min(limit, 100)) break;
 
     await sleep(config.apiDelayMs);
   }
 
   log.info(
-    { chatId: chatId.toString(), topicId: topicId.toString(), archives: archives.length, photos: photos.length, totalScanned },
+    { chatId: chatId.toString(), topicId: topicId.toString(), archives: archives.length, photos: photos.length, totalScanned, pages: pageCount },
     "Topic scan complete"
   );
 

worker/src/util/mutex.ts

@@ -4,12 +4,21 @@ const log = childLogger("mutex");
 
 let locked = false;
 let holder = "";
-const queue: Array<{ resolve: () => void; label: string }> = [];
+const queue: Array<{ resolve: () => void; reject: (err: Error) => void; label: string }> = [];
+
+/**
+ * Maximum time to wait for the TDLib mutex (ms).
+ * If the mutex is not available within this time, the operation is rejected.
+ * Default: 30 minutes (long enough for large downloads, short enough to detect hangs).
+ */
+const MUTEX_WAIT_TIMEOUT_MS = 30 * 60 * 1000;
 
 /**
  * Ensures only one TDLib client runs at a time across the entire worker process.
  * Both the scheduler (auth, ingestion) and the fetch listener acquire this
  * before creating any TDLib client.
+ *
+ * Includes a wait timeout to prevent indefinite blocking if the current holder hangs.
  */
 export async function withTdlibMutex<T>(
   label: string,
@@ -17,7 +26,29 @@ export async function withTdlibMutex<T>(
 ): Promise<T> {
   if (locked) {
     log.info({ waiting: label, holder }, "Waiting for TDLib mutex");
-    await new Promise<void>((resolve) => queue.push({ resolve, label }));
+    await new Promise<void>((resolve, reject) => {
+      const entry = { resolve, reject, label };
+      queue.push(entry);
+
+      // Timeout: reject if we've been waiting too long
+      const timer = setTimeout(() => {
+        const idx = queue.indexOf(entry);
+        if (idx !== -1) {
+          queue.splice(idx, 1);
+          reject(new Error(
+            `TDLib mutex wait timeout after ${MUTEX_WAIT_TIMEOUT_MS / 60_000}min ` +
+            `(waiting: ${label}, holder: ${holder})`
+          ));
+        }
+      }, MUTEX_WAIT_TIMEOUT_MS);
+
+      // Wrap resolve to clear the timer
+      const origResolve = entry.resolve;
+      entry.resolve = () => {
+        clearTimeout(timer);
+        origResolve();
+      };
+    });
   }
 
   locked = true;