From ad7134646805dc32a975f75e7d4fa011d48f4d6e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 5 Mar 2026 13:02:41 +0000 Subject: [PATCH 1/4] Initial plan From 9adbdb2a77bb968a044907e27d516f2ccf1d960a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 5 Mar 2026 13:14:53 +0000 Subject: [PATCH 2/4] Fix worker getting stuck during sync: add timeouts, stuck detection, and safety limits - Add invokeWithTimeout wrapper for TDLib API calls (2min timeout per call) - Add stuck detection to getChannelMessages: break if from_message_id doesn't advance - Add stuck detection to getTopicMessages: same protection for topic scanning - Add stuck detection to getForumTopicList: break if pagination offsets don't advance - Add max page limit (5000) to all scanning loops to prevent infinite pagination - Add mutex wait timeout (30min) to prevent indefinite blocking when holder hangs - Add cycle timeout (4h default, configurable via WORKER_CYCLE_TIMEOUT_MINUTES) - Fix end-of-page detection to use actual limit value instead of hardcoded 100 Co-authored-by: xCyanGrizzly <53275238+xCyanGrizzly@users.noreply.github.com> --- worker/dist/archive/creator.d.ts | 12 + worker/dist/archive/creator.js | 21 + worker/dist/archive/creator.js.map | 1 + worker/dist/archive/detect.d.ts | 15 + worker/dist/archive/detect.js | 77 +++ worker/dist/archive/detect.js.map | 1 + worker/dist/archive/hash.d.ts | 6 + worker/dist/archive/hash.js | 22 + worker/dist/archive/hash.js.map | 1 + worker/dist/archive/multipart.d.ts | 19 + worker/dist/archive/multipart.js | 74 +++ worker/dist/archive/multipart.js.map | 1 + worker/dist/archive/rar-reader.d.ts | 6 + worker/dist/archive/rar-reader.js | 77 +++ worker/dist/archive/rar-reader.js.map | 1 + worker/dist/archive/split.d.ts | 11 + worker/dist/archive/split.js | 55 ++ worker/dist/archive/split.js.map | 1 + worker/dist/archive/zip-reader.d.ts | 15 + worker/dist/archive/zip-reader.js | 161 ++++++ worker/dist/archive/zip-reader.js.map | 1 + worker/dist/db/client.d.ts | 7 + worker/dist/db/client.js | 12 + worker/dist/db/client.js.map | 1 + worker/dist/db/locks.d.ts | 9 + worker/dist/db/locks.js | 53 ++ worker/dist/db/locks.js.map | 1 + worker/dist/db/queries.d.ts | 356 ++++++++++++ worker/dist/db/queries.js | 319 +++++++++++ worker/dist/db/queries.js.map | 1 + worker/dist/fetch-listener.d.ts | 11 + worker/dist/fetch-listener.js | 195 +++++++ worker/dist/fetch-listener.js.map | 1 + worker/dist/index.d.ts | 1 + worker/dist/index.js | 50 ++ worker/dist/index.js.map | 1 + worker/dist/preview/match.d.ts | 22 + worker/dist/preview/match.js | 53 ++ worker/dist/preview/match.js.map | 1 + worker/dist/scheduler.d.ts | 13 + worker/dist/scheduler.js | 121 +++++ worker/dist/scheduler.js.map | 1 + worker/dist/tdlib/chats.d.ts | 31 ++ worker/dist/tdlib/chats.js | 124 +++++ worker/dist/tdlib/chats.js.map | 1 + worker/dist/tdlib/client.d.ts | 18 + worker/dist/tdlib/client.js | 96 ++++ worker/dist/tdlib/client.js.map | 1 + worker/dist/tdlib/download.d.ts | 67 +++ worker/dist/tdlib/download.js | 307 +++++++++++ worker/dist/tdlib/download.js.map | 1 + worker/dist/tdlib/topics.d.ts | 32 ++ worker/dist/tdlib/topics.js | 196 +++++++ worker/dist/tdlib/topics.js.map | 1 + worker/dist/upload/channel.d.ts | 16 + worker/dist/upload/channel.js | 137 +++++ worker/dist/upload/channel.js.map | 1 + worker/dist/util/config.d.ts | 18 + worker/dist/util/config.js | 19 + worker/dist/util/config.js.map | 1 + worker/dist/util/logger.d.ts | 3 + worker/dist/util/logger.js | 12 + worker/dist/util/logger.js.map | 1 + worker/dist/util/mutex.d.ts | 8 + worker/dist/util/mutex.js | 61 +++ worker/dist/util/mutex.js.map | 1 + worker/dist/worker.d.ts | 28 + worker/dist/worker.js | 745 ++++++++++++++++++++++++++ worker/dist/worker.js.map | 1 + worker/src/scheduler.ts | 30 +- worker/src/tdlib/download.ts | 67 ++- worker/src/tdlib/topics.ts | 119 ++-- worker/src/util/mutex.ts | 35 +- 73 files changed, 3945 insertions(+), 40 deletions(-) create mode 100644 worker/dist/archive/creator.d.ts create mode 100644 worker/dist/archive/creator.js create mode 100644 worker/dist/archive/creator.js.map create mode 100644 worker/dist/archive/detect.d.ts create mode 100644 worker/dist/archive/detect.js create mode 100644 worker/dist/archive/detect.js.map create mode 100644 worker/dist/archive/hash.d.ts create mode 100644 worker/dist/archive/hash.js create mode 100644 worker/dist/archive/hash.js.map create mode 100644 worker/dist/archive/multipart.d.ts create mode 100644 worker/dist/archive/multipart.js create mode 100644 worker/dist/archive/multipart.js.map create mode 100644 worker/dist/archive/rar-reader.d.ts create mode 100644 worker/dist/archive/rar-reader.js create mode 100644 worker/dist/archive/rar-reader.js.map create mode 100644 worker/dist/archive/split.d.ts create mode 100644 worker/dist/archive/split.js create mode 100644 worker/dist/archive/split.js.map create mode 100644 worker/dist/archive/zip-reader.d.ts create mode 100644 worker/dist/archive/zip-reader.js create mode 100644 worker/dist/archive/zip-reader.js.map create mode 100644 worker/dist/db/client.d.ts create mode 100644 worker/dist/db/client.js create mode 100644 worker/dist/db/client.js.map create mode 100644 worker/dist/db/locks.d.ts create mode 100644 worker/dist/db/locks.js create mode 100644 worker/dist/db/locks.js.map create mode 100644 worker/dist/db/queries.d.ts create mode 100644 worker/dist/db/queries.js create mode 100644 worker/dist/db/queries.js.map create mode 100644 worker/dist/fetch-listener.d.ts create mode 100644 worker/dist/fetch-listener.js create mode 100644 worker/dist/fetch-listener.js.map create mode 100644 worker/dist/index.d.ts create mode 100644 worker/dist/index.js create mode 100644 worker/dist/index.js.map create mode 100644 worker/dist/preview/match.d.ts create mode 100644 worker/dist/preview/match.js create mode 100644 worker/dist/preview/match.js.map create mode 100644 worker/dist/scheduler.d.ts create mode 100644 worker/dist/scheduler.js create mode 100644 worker/dist/scheduler.js.map create mode 100644 worker/dist/tdlib/chats.d.ts create mode 100644 worker/dist/tdlib/chats.js create mode 100644 worker/dist/tdlib/chats.js.map create mode 100644 worker/dist/tdlib/client.d.ts create mode 100644 worker/dist/tdlib/client.js create mode 100644 worker/dist/tdlib/client.js.map create mode 100644 worker/dist/tdlib/download.d.ts create mode 100644 worker/dist/tdlib/download.js create mode 100644 worker/dist/tdlib/download.js.map create mode 100644 worker/dist/tdlib/topics.d.ts create mode 100644 worker/dist/tdlib/topics.js create mode 100644 worker/dist/tdlib/topics.js.map create mode 100644 worker/dist/upload/channel.d.ts create mode 100644 worker/dist/upload/channel.js create mode 100644 worker/dist/upload/channel.js.map create mode 100644 worker/dist/util/config.d.ts create mode 100644 worker/dist/util/config.js create mode 100644 worker/dist/util/config.js.map create mode 100644 worker/dist/util/logger.d.ts create mode 100644 worker/dist/util/logger.js create mode 100644 worker/dist/util/logger.js.map create mode 100644 worker/dist/util/mutex.d.ts create mode 100644 worker/dist/util/mutex.js create mode 100644 worker/dist/util/mutex.js.map create mode 100644 worker/dist/worker.d.ts create mode 100644 worker/dist/worker.js create mode 100644 worker/dist/worker.js.map diff --git a/worker/dist/archive/creator.d.ts b/worker/dist/archive/creator.d.ts new file mode 100644 index 0000000..07eacd3 --- /dev/null +++ b/worker/dist/archive/creator.d.ts @@ -0,0 +1,12 @@ +/** + * Extract a creator name from common archive file naming patterns. + * + * Priority in the worker: topic name > filename extraction. + * This is the fallback when no forum topic name is available. + * + * Patterns handled (split on ` - `): + * "Mammoth Factory - 2026-01.zip" → "Mammoth Factory" + * "Artist Name - Pack Title.part01.rar" → "Artist Name" + * "some_random_file.zip" → null + */ +export declare function extractCreatorFromFileName(fileName: string): string | null; diff --git a/worker/dist/archive/creator.js b/worker/dist/archive/creator.js new file mode 100644 index 0000000..2ecf610 --- /dev/null +++ b/worker/dist/archive/creator.js @@ -0,0 +1,21 @@ +/** + * Extract a creator name from common archive file naming patterns. + * + * Priority in the worker: topic name > filename extraction. + * This is the fallback when no forum topic name is available. + * + * Patterns handled (split on ` - `): + * "Mammoth Factory - 2026-01.zip" → "Mammoth Factory" + * "Artist Name - Pack Title.part01.rar" → "Artist Name" + * "some_random_file.zip" → null + */ +export function extractCreatorFromFileName(fileName) { + // Strip archive extensions (.zip, .rar, .part01.rar, .z01, etc.) + const bare = fileName.replace(/(\.(part\d+\.rar|z\d{2}|zip|rar))+$/i, ""); + const idx = bare.indexOf(" - "); + if (idx <= 0) + return null; + const creator = bare.slice(0, idx).trim(); + return creator.length > 0 ? creator : null; +} +//# sourceMappingURL=creator.js.map \ No newline at end of file diff --git a/worker/dist/archive/creator.js.map b/worker/dist/archive/creator.js.map new file mode 100644 index 0000000..4c5cdab --- /dev/null +++ b/worker/dist/archive/creator.js.map @@ -0,0 +1 @@ +{"version":3,"file":"creator.js","sourceRoot":"","sources":["../../src/archive/creator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AACH,MAAM,UAAU,0BAA0B,CAAC,QAAgB;IACzD,iEAAiE;IACjE,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,sCAAsC,EAAE,EAAE,CAAC,CAAC;IAE1E,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAChC,IAAI,GAAG,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IAE1B,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IAC1C,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;AAC7C,CAAC"} \ No newline at end of file diff --git a/worker/dist/archive/detect.d.ts b/worker/dist/archive/detect.d.ts new file mode 100644 index 0000000..9f0c21f --- /dev/null +++ b/worker/dist/archive/detect.d.ts @@ -0,0 +1,15 @@ +export type ArchiveFormat = "ZIP" | "RAR"; +export interface MultipartInfo { + baseName: string; + partNumber: number; + format: ArchiveFormat; + pattern: "ZIP_NUMBERED" | "ZIP_LEGACY" | "RAR_PART" | "RAR_LEGACY" | "SINGLE"; +} +/** + * Detect if a filename is an archive and extract multipart info. + */ +export declare function detectArchive(fileName: string): MultipartInfo | null; +/** + * Check if a filename looks like any archive attachment we should process. + */ +export declare function isArchiveAttachment(fileName: string): boolean; diff --git a/worker/dist/archive/detect.js b/worker/dist/archive/detect.js new file mode 100644 index 0000000..ab22108 --- /dev/null +++ b/worker/dist/archive/detect.js @@ -0,0 +1,77 @@ +const patterns = [ + // pack.zip.001, pack.zip.002 + { + regex: /^(.+\.zip)\.(\d{3,})$/i, + format: "ZIP", + pattern: "ZIP_NUMBERED", + getBaseName: (m) => m[1], + getPartNumber: (m) => parseInt(m[2], 10), + }, + // pack.z01, pack.z02 (legacy split — final part is pack.zip) + { + regex: /^(.+)\.z(\d{2,})$/i, + format: "ZIP", + pattern: "ZIP_LEGACY", + getBaseName: (m) => m[1], + getPartNumber: (m) => parseInt(m[2], 10), + }, + // pack.part1.rar, pack.part2.rar + { + regex: /^(.+)\.part(\d+)\.rar$/i, + format: "RAR", + pattern: "RAR_PART", + getBaseName: (m) => m[1], + getPartNumber: (m) => parseInt(m[2], 10), + }, + // pack.r00, pack.r01 (legacy split — final part is pack.rar) + { + regex: /^(.+)\.r(\d{2,})$/i, + format: "RAR", + pattern: "RAR_LEGACY", + getBaseName: (m) => m[1], + getPartNumber: (m) => parseInt(m[2], 10), + }, +]; +/** + * Detect if a filename is an archive and extract multipart info. + */ +export function detectArchive(fileName) { + // Check multipart patterns first + for (const p of patterns) { + const match = fileName.match(p.regex); + if (match) { + return { + baseName: p.getBaseName(match), + partNumber: p.getPartNumber(match), + format: p.format, + pattern: p.pattern, + }; + } + } + // Single .zip file — could be a standalone or the final part of a ZIP_LEGACY set + if (/\.zip$/i.test(fileName)) { + return { + baseName: fileName.replace(/\.zip$/i, ""), + partNumber: -1, // -1 signals "could be single or final legacy part" + format: "ZIP", + pattern: "SINGLE", + }; + } + // Single .rar file — could be standalone or final part of RAR_LEGACY set + if (/\.rar$/i.test(fileName)) { + return { + baseName: fileName.replace(/\.rar$/i, ""), + partNumber: -1, + format: "RAR", + pattern: "SINGLE", + }; + } + return null; +} +/** + * Check if a filename looks like any archive attachment we should process. + */ +export function isArchiveAttachment(fileName) { + return detectArchive(fileName) !== null; +} +//# sourceMappingURL=detect.js.map \ No newline at end of file diff --git a/worker/dist/archive/detect.js.map b/worker/dist/archive/detect.js.map new file mode 100644 index 0000000..96036b7 --- /dev/null +++ b/worker/dist/archive/detect.js.map @@ -0,0 +1 @@ +{"version":3,"file":"detect.js","sourceRoot":"","sources":["../../src/archive/detect.ts"],"names":[],"mappings":"AASA,MAAM,QAAQ,GAMR;IACJ,6BAA6B;IAC7B;QACE,KAAK,EAAE,wBAAwB;QAC/B,MAAM,EAAE,KAAK;QACb,OAAO,EAAE,cAAc;QACvB,WAAW,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACxB,aAAa,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;KACzC;IACD,6DAA6D;IAC7D;QACE,KAAK,EAAE,oBAAoB;QAC3B,MAAM,EAAE,KAAK;QACb,OAAO,EAAE,YAAY;QACrB,WAAW,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACxB,aAAa,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;KACzC;IACD,iCAAiC;IACjC;QACE,KAAK,EAAE,yBAAyB;QAChC,MAAM,EAAE,KAAK;QACb,OAAO,EAAE,UAAU;QACnB,WAAW,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACxB,aAAa,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;KACzC;IACD,6DAA6D;IAC7D;QACE,KAAK,EAAE,oBAAoB;QAC3B,MAAM,EAAE,KAAK;QACb,OAAO,EAAE,YAAY;QACrB,WAAW,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACxB,aAAa,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;KACzC;CACF,CAAC;AAEF;;GAEG;AACH,MAAM,UAAU,aAAa,CAAC,QAAgB;IAC5C,iCAAiC;IACjC,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QACtC,IAAI,KAAK,EAAE,CAAC;YACV,OAAO;gBACL,QAAQ,EAAE,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC;gBAC9B,UAAU,EAAE,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC;gBAClC,MAAM,EAAE,CAAC,CAAC,MAAM;gBAChB,OAAO,EAAE,CAAC,CAAC,OAAO;aACnB,CAAC;QACJ,CAAC;IACH,CAAC;IAED,iFAAiF;IACjF,IAAI,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC7B,OAAO;YACL,QAAQ,EAAE,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC;YACzC,UAAU,EAAE,CAAC,CAAC,EAAE,oDAAoD;YACpE,MAAM,EAAE,KAAK;YACb,OAAO,EAAE,QAAQ;SAClB,CAAC;IACJ,CAAC;IAED,yEAAyE;IACzE,IAAI,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC7B,OAAO;YACL,QAAQ,EAAE,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC;YACzC,UAAU,EAAE,CAAC,CAAC;YACd,MAAM,EAAE,KAAK;YACb,OAAO,EAAE,QAAQ;SAClB,CAAC;IACJ,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,QAAgB;IAClD,OAAO,aAAa,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC;AAC1C,CAAC"} \ No newline at end of file diff --git a/worker/dist/archive/hash.d.ts b/worker/dist/archive/hash.d.ts new file mode 100644 index 0000000..0f02b3f --- /dev/null +++ b/worker/dist/archive/hash.d.ts @@ -0,0 +1,6 @@ +/** + * Compute SHA-256 hash of one or more files by streaming them in order. + * Memory usage: O(1) — reads in 64KB chunks regardless of total size. + * For multipart archives, pass all parts sorted by part number. + */ +export declare function hashParts(filePaths: string[]): Promise; diff --git a/worker/dist/archive/hash.js b/worker/dist/archive/hash.js new file mode 100644 index 0000000..c13759c --- /dev/null +++ b/worker/dist/archive/hash.js @@ -0,0 +1,22 @@ +import { createReadStream } from "fs"; +import { createHash } from "crypto"; +import { pipeline } from "stream/promises"; +import { PassThrough } from "stream"; +/** + * Compute SHA-256 hash of one or more files by streaming them in order. + * Memory usage: O(1) — reads in 64KB chunks regardless of total size. + * For multipart archives, pass all parts sorted by part number. + */ +export async function hashParts(filePaths) { + const hash = createHash("sha256"); + for (const filePath of filePaths) { + await pipeline(createReadStream(filePath), new PassThrough({ + transform(chunk, _encoding, callback) { + hash.update(chunk); + callback(); + }, + })); + } + return hash.digest("hex"); +} +//# sourceMappingURL=hash.js.map \ No newline at end of file diff --git a/worker/dist/archive/hash.js.map b/worker/dist/archive/hash.js.map new file mode 100644 index 0000000..52d1556 --- /dev/null +++ b/worker/dist/archive/hash.js.map @@ -0,0 +1 @@ +{"version":3,"file":"hash.js","sourceRoot":"","sources":["../../src/archive/hash.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,IAAI,CAAC;AACtC,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAC3C,OAAO,EAAE,WAAW,EAAE,MAAM,QAAQ,CAAC;AAErC;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,SAAmB;IACjD,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IAClC,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,MAAM,QAAQ,CACZ,gBAAgB,CAAC,QAAQ,CAAC,EAC1B,IAAI,WAAW,CAAC;YACd,SAAS,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ;gBAClC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACnB,QAAQ,EAAE,CAAC;YACb,CAAC;SACF,CAAC,CACH,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC5B,CAAC"} \ No newline at end of file diff --git a/worker/dist/archive/multipart.d.ts b/worker/dist/archive/multipart.d.ts new file mode 100644 index 0000000..2b46519 --- /dev/null +++ b/worker/dist/archive/multipart.d.ts @@ -0,0 +1,19 @@ +import { type ArchiveFormat } from "./detect.js"; +export interface TelegramMessage { + id: bigint; + fileName: string; + fileId: string; + fileSize: bigint; + date: Date; +} +export interface ArchiveSet { + type: ArchiveFormat; + baseName: string; + parts: TelegramMessage[]; + isMultipart: boolean; +} +/** + * Group messages into archive sets (single files + multipart groups). + * Messages should be pre-filtered to only include archive attachments. + */ +export declare function groupArchiveSets(messages: TelegramMessage[]): ArchiveSet[]; diff --git a/worker/dist/archive/multipart.js b/worker/dist/archive/multipart.js new file mode 100644 index 0000000..efecfd1 --- /dev/null +++ b/worker/dist/archive/multipart.js @@ -0,0 +1,74 @@ +import { detectArchive } from "./detect.js"; +import { config } from "../util/config.js"; +import { childLogger } from "../util/logger.js"; +const log = childLogger("multipart"); +/** + * Group messages into archive sets (single files + multipart groups). + * Messages should be pre-filtered to only include archive attachments. + */ +export function groupArchiveSets(messages) { + // Detect and annotate each message + const annotated = []; + for (const msg of messages) { + const info = detectArchive(msg.fileName); + if (info) { + annotated.push({ msg, info }); + } + } + // Group by baseName + format + const groups = new Map(); + for (const item of annotated) { + const key = `${item.info.format}:${item.info.baseName.toLowerCase()}`; + const group = groups.get(key) ?? []; + group.push(item); + groups.set(key, group); + } + const results = []; + for (const [, group] of groups) { + const format = group[0].info.format; + const baseName = group[0].info.baseName; + // Separate explicit multipart entries from potential singles + const multipartEntries = group.filter((g) => g.info.pattern !== "SINGLE"); + const singleEntries = group.filter((g) => g.info.pattern === "SINGLE"); + if (multipartEntries.length > 0) { + // This is a multipart set + // Check if any single entry is the "final part" of a legacy split + const allEntries = [...multipartEntries, ...singleEntries]; + // Check time span — skip if parts span too long (0 = no limit) + if (config.multipartTimeoutHours > 0) { + const dates = allEntries.map((e) => e.msg.date.getTime()); + const span = Math.max(...dates) - Math.min(...dates); + const maxSpanMs = config.multipartTimeoutHours * 60 * 60 * 1000; + if (span > maxSpanMs) { + log.warn({ baseName, format, span: span / 3600000 }, "Multipart set spans too long, skipping"); + continue; + } + } + // Sort by part number (singles get a very high number so they come last — they're the final part) + allEntries.sort((a, b) => { + const aNum = a.info.partNumber === -1 ? 999999 : a.info.partNumber; + const bNum = b.info.partNumber === -1 ? 999999 : b.info.partNumber; + return aNum - bNum; + }); + results.push({ + type: format, + baseName, + parts: allEntries.map((e) => e.msg), + isMultipart: true, + }); + } + else { + // All entries are singles — each is its own archive set + for (const entry of singleEntries) { + results.push({ + type: format, + baseName: entry.info.baseName, + parts: [entry.msg], + isMultipart: false, + }); + } + } + } + return results; +} +//# sourceMappingURL=multipart.js.map \ No newline at end of file diff --git a/worker/dist/archive/multipart.js.map b/worker/dist/archive/multipart.js.map new file mode 100644 index 0000000..4708bf7 --- /dev/null +++ b/worker/dist/archive/multipart.js.map @@ -0,0 +1 @@ +{"version":3,"file":"multipart.js","sourceRoot":"","sources":["../../src/archive/multipart.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAA0C,MAAM,aAAa,CAAC;AACpF,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,MAAM,GAAG,GAAG,WAAW,CAAC,WAAW,CAAC,CAAC;AAiBrC;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,QAA2B;IAC1D,mCAAmC;IACnC,MAAM,SAAS,GAAoD,EAAE,CAAC;IACtE,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACzC,IAAI,IAAI,EAAE,CAAC;YACT,SAAS,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;IAED,6BAA6B;IAC7B,MAAM,MAAM,GAAG,IAAI,GAAG,EAA2D,CAAC;IAClF,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,EAAE,CAAC;QACtE,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACpC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjB,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACzB,CAAC;IAED,MAAM,OAAO,GAAiB,EAAE,CAAC;IAEjC,KAAK,MAAM,CAAC,EAAE,KAAK,CAAC,IAAI,MAAM,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;QACpC,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC;QAExC,6DAA6D;QAC7D,MAAM,gBAAgB,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC;QAC1E,MAAM,aAAa,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC;QAEvE,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChC,0BAA0B;YAC1B,kEAAkE;YAClE,MAAM,UAAU,GAAG,CAAC,GAAG,gBAAgB,EAAE,GAAG,aAAa,CAAC,CAAC;YAE3D,+DAA+D;YAC/D,IAAI,MAAM,CAAC,qBAAqB,GAAG,CAAC,EAAE,CAAC;gBACrC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;gBAC1D,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC;gBACrD,MAAM,SAAS,GAAG,MAAM,CAAC,qBAAqB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;gBAEhE,IAAI,IAAI,GAAG,SAAS,EAAE,CAAC;oBACrB,GAAG,CAAC,IAAI,CACN,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,GAAG,OAAO,EAAE,EAC1C,wCAAwC,CACzC,CAAC;oBACF,SAAS;gBACX,CAAC;YACH,CAAC;YAED,kGAAkG;YAClG,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;gBACvB,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,UAAU,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC;gBACnE,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,UAAU,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC;gBACnE,OAAO,IAAI,GAAG,IAAI,CAAC;YACrB,CAAC,CAAC,CAAC;YAEH,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,MAAM;gBACZ,QAAQ;gBACR,KAAK,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;gBACnC,WAAW,EAAE,IAAI;aAClB,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,wDAAwD;YACxD,KAAK,MAAM,KAAK,IAAI,aAAa,EAAE,CAAC;gBAClC,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,MAAM;oBACZ,QAAQ,EAAE,KAAK,CAAC,IAAI,CAAC,QAAQ;oBAC7B,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC;oBAClB,WAAW,EAAE,KAAK;iBACnB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"} \ No newline at end of file diff --git a/worker/dist/archive/rar-reader.d.ts b/worker/dist/archive/rar-reader.d.ts new file mode 100644 index 0000000..3053cdb --- /dev/null +++ b/worker/dist/archive/rar-reader.d.ts @@ -0,0 +1,6 @@ +import type { FileEntry } from "./zip-reader.js"; +/** + * Parse output of `unrar l -v ` to extract file metadata. + * unrar automatically discovers sibling parts when they're co-located. + */ +export declare function readRarContents(firstPartPath: string): Promise; diff --git a/worker/dist/archive/rar-reader.js b/worker/dist/archive/rar-reader.js new file mode 100644 index 0000000..d541f5d --- /dev/null +++ b/worker/dist/archive/rar-reader.js @@ -0,0 +1,77 @@ +import { execFile } from "child_process"; +import { promisify } from "util"; +import path from "path"; +import { childLogger } from "../util/logger.js"; +const execFileAsync = promisify(execFile); +const log = childLogger("rar-reader"); +/** + * Parse output of `unrar l -v ` to extract file metadata. + * unrar automatically discovers sibling parts when they're co-located. + */ +export async function readRarContents(firstPartPath) { + try { + const { stdout } = await execFileAsync("unrar", ["l", "-v", firstPartPath], { + timeout: 30000, + maxBuffer: 10 * 1024 * 1024, // 10MB for very large archives + }); + return parseUnrarOutput(stdout); + } + catch (err) { + log.warn({ err, file: firstPartPath }, "Failed to read RAR contents"); + return []; // Fallback: return empty on error + } +} +/** + * Parse the tabular output of `unrar l -v`. + * + * Example output format: + * Archive: test.rar + * Details: RAR 5 + * + * Attributes Size Packed Ratio Date Time CRC-32 Name + * ----------- --------- --------- ----- -------- ----- -------- ---- + * ...A.... 12345 10234 83% 2024-01-15 10:30 DEADBEEF folder/file.stl + * ----------- --------- --------- ----- -------- ----- -------- ---- + */ +function parseUnrarOutput(output) { + const entries = []; + const lines = output.split("\n"); + let inFileList = false; + let separatorCount = 0; + for (const line of lines) { + const trimmed = line.trim(); + // Detect separator lines (------- pattern) + if (/^-{5,}/.test(trimmed)) { + separatorCount++; + if (separatorCount === 1) { + inFileList = true; + } + else if (separatorCount >= 2) { + inFileList = false; + } + continue; + } + if (!inFileList) + continue; + // Parse file entry line + // Format: Attributes Size Packed Ratio Date Time CRC Name + const match = trimmed.match(/^\S+\s+(\d+)\s+(\d+)\s+\d+%\s+\S+\s+\S+\s+([0-9A-Fa-f]+)\s+(.+)$/); + if (match) { + const [, uncompressedStr, compressedStr, crc32, filePath] = match; + // Skip directory entries (typically end with / or have size 0 with dir attributes) + if (filePath.endsWith("/") || filePath.endsWith("\\")) + continue; + const ext = path.extname(filePath).toLowerCase(); + entries.push({ + path: filePath, + fileName: path.basename(filePath), + extension: ext ? ext.slice(1) : null, + compressedSize: BigInt(compressedStr), + uncompressedSize: BigInt(uncompressedStr), + crc32: crc32.toLowerCase(), + }); + } + } + return entries; +} +//# sourceMappingURL=rar-reader.js.map \ No newline at end of file diff --git a/worker/dist/archive/rar-reader.js.map b/worker/dist/archive/rar-reader.js.map new file mode 100644 index 0000000..35d94a4 --- /dev/null +++ b/worker/dist/archive/rar-reader.js.map @@ -0,0 +1 @@ +{"version":3,"file":"rar-reader.js","sourceRoot":"","sources":["../../src/archive/rar-reader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,MAAM,CAAC;AACjC,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAGhD,MAAM,aAAa,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;AAC1C,MAAM,GAAG,GAAG,WAAW,CAAC,YAAY,CAAC,CAAC;AAEtC;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,aAAqB;IAErB,IAAI,CAAC;QACH,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,aAAa,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,IAAI,EAAE,aAAa,CAAC,EAAE;YAC1E,OAAO,EAAE,KAAK;YACd,SAAS,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI,EAAE,+BAA+B;SAC7D,CAAC,CAAC;QAEH,OAAO,gBAAgB,CAAC,MAAM,CAAC,CAAC;IAClC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,aAAa,EAAE,EAAE,6BAA6B,CAAC,CAAC;QACtE,OAAO,EAAE,CAAC,CAAC,kCAAkC;IAC/C,CAAC;AACH,CAAC;AAED;;;;;;;;;;;GAWG;AACH,SAAS,gBAAgB,CAAC,MAAc;IACtC,MAAM,OAAO,GAAgB,EAAE,CAAC;IAChC,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAEjC,IAAI,UAAU,GAAG,KAAK,CAAC;IACvB,IAAI,cAAc,GAAG,CAAC,CAAC;IAEvB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAE5B,2CAA2C;QAC3C,IAAI,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAC3B,cAAc,EAAE,CAAC;YACjB,IAAI,cAAc,KAAK,CAAC,EAAE,CAAC;gBACzB,UAAU,GAAG,IAAI,CAAC;YACpB,CAAC;iBAAM,IAAI,cAAc,IAAI,CAAC,EAAE,CAAC;gBAC/B,UAAU,GAAG,KAAK,CAAC;YACrB,CAAC;YACD,SAAS;QACX,CAAC;QAED,IAAI,CAAC,UAAU;YAAE,SAAS;QAE1B,wBAAwB;QACxB,0DAA0D;QAC1D,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CACzB,kEAAkE,CACnE,CAAC;QAEF,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,CAAC,EAAE,eAAe,EAAE,aAAa,EAAE,KAAK,EAAE,QAAQ,CAAC,GAAG,KAAK,CAAC;YAElE,mFAAmF;YACnF,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC;gBAAE,SAAS;YAEhE,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;YACjD,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,QAAQ;gBACd,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBACjC,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI;gBACpC,cAAc,EAAE,MAAM,CAAC,aAAa,CAAC;gBACrC,gBAAgB,EAAE,MAAM,CAAC,eAAe,CAAC;gBACzC,KAAK,EAAE,KAAK,CAAC,WAAW,EAAE;aAC3B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"} \ No newline at end of file diff --git a/worker/dist/archive/split.d.ts b/worker/dist/archive/split.d.ts new file mode 100644 index 0000000..67a31ae --- /dev/null +++ b/worker/dist/archive/split.d.ts @@ -0,0 +1,11 @@ +/** + * Split a file into ≤2GB parts using byte-level splitting. + * Returns paths to the split parts. If the file is already ≤2GB, returns the original path. + */ +export declare function byteLevelSplit(filePath: string): Promise; +/** + * Concatenate multiple files into a single output file by streaming + * each input sequentially. Used for repacking multipart archives + * that have oversized parts (>2GB) before re-splitting. + */ +export declare function concatenateFiles(inputPaths: string[], outputPath: string): Promise; diff --git a/worker/dist/archive/split.js b/worker/dist/archive/split.js new file mode 100644 index 0000000..ec67450 --- /dev/null +++ b/worker/dist/archive/split.js @@ -0,0 +1,55 @@ +import { createReadStream, createWriteStream } from "fs"; +import { stat } from "fs/promises"; +import path from "path"; +import { pipeline } from "stream/promises"; +import { childLogger } from "../util/logger.js"; +const log = childLogger("split"); +/** 2GB in bytes — Telegram's file size limit */ +const MAX_PART_SIZE = 2n * 1024n * 1024n * 1024n; +/** + * Split a file into ≤2GB parts using byte-level splitting. + * Returns paths to the split parts. If the file is already ≤2GB, returns the original path. + */ +export async function byteLevelSplit(filePath) { + const stats = await stat(filePath); + const fileSize = BigInt(stats.size); + if (fileSize <= MAX_PART_SIZE) { + return [filePath]; + } + const dir = path.dirname(filePath); + const baseName = path.basename(filePath); + const partSize = Number(MAX_PART_SIZE); + const totalParts = Math.ceil(Number(fileSize) / partSize); + const parts = []; + log.info({ filePath, fileSize: Number(fileSize), totalParts }, "Splitting file"); + for (let i = 0; i < totalParts; i++) { + const partNum = String(i + 1).padStart(3, "0"); + const partPath = path.join(dir, `${baseName}.${partNum}`); + const start = i * partSize; + const end = Math.min(start + partSize - 1, Number(fileSize) - 1); + await pipeline(createReadStream(filePath, { start, end }), createWriteStream(partPath)); + parts.push(partPath); + } + log.info({ filePath, parts: parts.length }, "File split complete"); + return parts; +} +/** + * Concatenate multiple files into a single output file by streaming + * each input sequentially. Used for repacking multipart archives + * that have oversized parts (>2GB) before re-splitting. + */ +export async function concatenateFiles(inputPaths, outputPath) { + const out = createWriteStream(outputPath); + for (let i = 0; i < inputPaths.length; i++) { + log.info({ part: i + 1, total: inputPaths.length, file: path.basename(inputPaths[i]) }, "Concatenating part"); + await pipeline(createReadStream(inputPaths[i]), out, { end: false }); + } + // Close the output stream + await new Promise((resolve, reject) => { + out.end(() => resolve()); + out.on("error", reject); + }); + const stats = await stat(outputPath); + log.info({ outputPath, totalBytes: stats.size, parts: inputPaths.length }, "Concatenation complete"); +} +//# sourceMappingURL=split.js.map \ No newline at end of file diff --git a/worker/dist/archive/split.js.map b/worker/dist/archive/split.js.map new file mode 100644 index 0000000..79031bb --- /dev/null +++ b/worker/dist/archive/split.js.map @@ -0,0 +1 @@ +{"version":3,"file":"split.js","sourceRoot":"","sources":["../../src/archive/split.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,IAAI,CAAC;AACzD,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AACnC,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAC3C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,MAAM,GAAG,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;AAEjC,gDAAgD;AAChD,MAAM,aAAa,GAAG,EAAE,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,CAAC;AAEjD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,QAAgB;IACnD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;IACnC,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAEpC,IAAI,QAAQ,IAAI,aAAa,EAAE,CAAC;QAC9B,OAAO,CAAC,QAAQ,CAAC,CAAC;IACpB,CAAC;IAED,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACzC,MAAM,QAAQ,GAAG,MAAM,CAAC,aAAa,CAAC,CAAC;IACvC,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,QAAQ,CAAC,CAAC;IAC1D,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,GAAG,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC,EAAE,UAAU,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAEjF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,MAAM,OAAO,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAC/C,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,IAAI,OAAO,EAAE,CAAC,CAAC;QAC1D,MAAM,KAAK,GAAG,CAAC,GAAG,QAAQ,CAAC;QAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,QAAQ,GAAG,CAAC,EAAE,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;QAEjE,MAAM,QAAQ,CACZ,gBAAgB,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,EAC1C,iBAAiB,CAAC,QAAQ,CAAC,CAC5B,CAAC;QAEF,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACvB,CAAC;IAED,GAAG,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,EAAE,qBAAqB,CAAC,CAAC;IACnE,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,UAAoB,EACpB,UAAkB;IAElB,MAAM,GAAG,GAAG,iBAAiB,CAAC,UAAU,CAAC,CAAC;IAE1C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3C,GAAG,CAAC,IAAI,CACN,EAAE,IAAI,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,UAAU,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,EAC7E,oBAAoB,CACrB,CAAC;QACF,MAAM,QAAQ,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC;IACvE,CAAC;IAED,0BAA0B;IAC1B,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC1C,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;QACzB,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAC1B,CAAC,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC;IACrC,GAAG,CAAC,IAAI,CACN,EAAE,UAAU,EAAE,UAAU,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,UAAU,CAAC,MAAM,EAAE,EAChE,wBAAwB,CACzB,CAAC;AACJ,CAAC"} \ No newline at end of file diff --git a/worker/dist/archive/zip-reader.d.ts b/worker/dist/archive/zip-reader.d.ts new file mode 100644 index 0000000..f721913 --- /dev/null +++ b/worker/dist/archive/zip-reader.d.ts @@ -0,0 +1,15 @@ +export interface FileEntry { + path: string; + fileName: string; + extension: string | null; + compressedSize: bigint; + uncompressedSize: bigint; + crc32: string | null; +} +/** + * Read the central directory of a ZIP file without extracting any contents. + * For multipart ZIPs (.zip.001, .zip.002 etc.), uses a custom random-access + * reader that spans all parts seamlessly so yauzl can find the central + * directory at the end of the combined data. + */ +export declare function readZipCentralDirectory(filePaths: string[]): Promise; diff --git a/worker/dist/archive/zip-reader.js b/worker/dist/archive/zip-reader.js new file mode 100644 index 0000000..6e70b3f --- /dev/null +++ b/worker/dist/archive/zip-reader.js @@ -0,0 +1,161 @@ +import yauzl from "yauzl"; +import { open as fsOpen, stat as fsStat } from "fs/promises"; +import path from "path"; +import { Readable } from "stream"; +import { childLogger } from "../util/logger.js"; +const log = childLogger("zip-reader"); +/** + * Read the central directory of a ZIP file without extracting any contents. + * For multipart ZIPs (.zip.001, .zip.002 etc.), uses a custom random-access + * reader that spans all parts seamlessly so yauzl can find the central + * directory at the end of the combined data. + */ +export async function readZipCentralDirectory(filePaths) { + if (filePaths.length === 1) { + return readSingleZip(filePaths[0]); + } + // Multipart: use a spanning random-access reader + return readMultipartZip(filePaths); +} +/** Read a single (non-split) ZIP file. */ +function readSingleZip(targetFile) { + return new Promise((resolve) => { + yauzl.open(targetFile, { lazyEntries: true, autoClose: true }, (err, zipFile) => { + if (err) { + log.warn({ err, file: targetFile }, "Failed to open ZIP for reading"); + resolve([]); + return; + } + const entries = []; + zipFile.readEntry(); + zipFile.on("entry", (entry) => { + if (!entry.fileName.endsWith("/")) { + const ext = path.extname(entry.fileName).toLowerCase(); + entries.push({ + path: entry.fileName, + fileName: path.basename(entry.fileName), + extension: ext ? ext.slice(1) : null, + compressedSize: BigInt(entry.compressedSize), + uncompressedSize: BigInt(entry.uncompressedSize), + crc32: entry.crc32 !== 0 ? entry.crc32.toString(16).padStart(8, "0") : null, + }); + } + zipFile.readEntry(); + }); + zipFile.on("end", () => resolve(entries)); + zipFile.on("error", (error) => { + log.warn({ error, file: targetFile }, "Error reading ZIP entries"); + resolve(entries); + }); + }); + }); +} +/** + * Read a multipart split ZIP using yauzl's RandomAccessReader API. + * This creates a virtual "file" that spans all parts so yauzl can + * seek freely across the entire archive to read the central directory. + */ +async function readMultipartZip(filePaths) { + // Get sizes of all parts + const partSizes = []; + for (const fp of filePaths) { + const s = await fsStat(fp); + partSizes.push(s.size); + } + const totalSize = partSizes.reduce((a, b) => a + b, 0); + log.debug({ parts: filePaths.length, totalSize }, "Reading multipart ZIP via spanning reader"); + return new Promise((resolve) => { + const reader = createMultiPartReader(filePaths, partSizes); + yauzl.fromRandomAccessReader(reader, totalSize, { lazyEntries: true, autoClose: true }, (err, zipFile) => { + if (err) { + log.warn({ err }, "Failed to open multipart ZIP for reading"); + reader.close(() => { }); + resolve([]); + return; + } + const entries = []; + zipFile.readEntry(); + zipFile.on("entry", (entry) => { + if (!entry.fileName.endsWith("/")) { + const ext = path.extname(entry.fileName).toLowerCase(); + entries.push({ + path: entry.fileName, + fileName: path.basename(entry.fileName), + extension: ext ? ext.slice(1) : null, + compressedSize: BigInt(entry.compressedSize), + uncompressedSize: BigInt(entry.uncompressedSize), + crc32: entry.crc32 !== 0 ? entry.crc32.toString(16).padStart(8, "0") : null, + }); + } + zipFile.readEntry(); + }); + zipFile.on("end", () => { + log.info({ entries: entries.length }, "Multipart ZIP entries read"); + resolve(entries); + }); + zipFile.on("error", (error) => { + log.warn({ error }, "Error reading multipart ZIP entries"); + resolve(entries); + }); + }); + }); +} +/** + * Create a yauzl RandomAccessReader that reads across multiple split part files. + * Maps a global offset to the correct part file and local offset. + * + * Uses Object.create to properly inherit from yauzl.RandomAccessReader + * (whose constructor + prototype is defined at runtime, not as a TS class). + */ +function createMultiPartReader(filePaths, partSizes) { + // Build cumulative offset table + const partOffsets = []; + let offset = 0; + for (const size of partSizes) { + partOffsets.push(offset); + offset += size; + } + // Create an instance by calling the parent constructor + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const reader = new yauzl.RandomAccessReader(); + // Override _readStreamForRange — yauzl calls this to read a range of bytes + // eslint-disable-next-line @typescript-eslint/no-explicit-any + reader._readStreamForRange = function (start, end) { + const readable = new Readable({ read() { } }); + readRange(start, end, readable).catch((err) => { + readable.destroy(err); + }); + return readable; + }; + async function readRange(start, end, readable) { + let remaining = end - start; + let globalOffset = start; + while (remaining > 0) { + // Find which part this offset falls in + let partIdx = partOffsets.length - 1; + for (let i = 0; i < partOffsets.length; i++) { + if (i + 1 < partOffsets.length && globalOffset < partOffsets[i + 1]) { + partIdx = i; + break; + } + } + const localOffset = globalOffset - partOffsets[partIdx]; + const partRemaining = partSizes[partIdx] - localOffset; + const toRead = Math.min(remaining, partRemaining); + const fh = await fsOpen(filePaths[partIdx], "r"); + try { + const buf = Buffer.alloc(toRead); + const { bytesRead } = await fh.read(buf, 0, toRead, localOffset); + readable.push(buf.subarray(0, bytesRead)); + remaining -= bytesRead; + globalOffset += bytesRead; + } + finally { + await fh.close(); + } + } + readable.push(null); // Signal end of stream + } + return reader; +} +//# sourceMappingURL=zip-reader.js.map \ No newline at end of file diff --git a/worker/dist/archive/zip-reader.js.map b/worker/dist/archive/zip-reader.js.map new file mode 100644 index 0000000..7af95cd --- /dev/null +++ b/worker/dist/archive/zip-reader.js.map @@ -0,0 +1 @@ +{"version":3,"file":"zip-reader.js","sourceRoot":"","sources":["../../src/archive/zip-reader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,IAAI,IAAI,MAAM,EAAE,IAAI,IAAI,MAAM,EAAE,MAAM,aAAa,CAAC;AAC7D,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,MAAM,GAAG,GAAG,WAAW,CAAC,YAAY,CAAC,CAAC;AAWtC;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,SAAmB;IAEnB,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,OAAO,aAAa,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;IACrC,CAAC;IAED,iDAAiD;IACjD,OAAO,gBAAgB,CAAC,SAAS,CAAC,CAAC;AACrC,CAAC;AAED,0CAA0C;AAC1C,SAAS,aAAa,CAAC,UAAkB;IACvC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE;YAC9E,IAAI,GAAG,EAAE,CAAC;gBACR,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,gCAAgC,CAAC,CAAC;gBACtE,OAAO,CAAC,EAAE,CAAC,CAAC;gBACZ,OAAO;YACT,CAAC;YAED,MAAM,OAAO,GAAgB,EAAE,CAAC;YAEhC,OAAO,CAAC,SAAS,EAAE,CAAC;YACpB,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAkB,EAAE,EAAE;gBACzC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;oBAClC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;oBACvD,OAAO,CAAC,IAAI,CAAC;wBACX,IAAI,EAAE,KAAK,CAAC,QAAQ;wBACpB,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC;wBACvC,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI;wBACpC,cAAc,EAAE,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC;wBAC5C,gBAAgB,EAAE,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC;wBAChD,KAAK,EAAE,KAAK,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI;qBAC5E,CAAC,CAAC;gBACL,CAAC;gBACD,OAAO,CAAC,SAAS,EAAE,CAAC;YACtB,CAAC,CAAC,CAAC;YAEH,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;YAC1C,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;gBAC5B,GAAG,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,2BAA2B,CAAC,CAAC;gBACnE,OAAO,CAAC,OAAO,CAAC,CAAC;YACnB,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,gBAAgB,CAAC,SAAmB;IACjD,yBAAyB;IACzB,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,KAAK,MAAM,EAAE,IAAI,SAAS,EAAE,CAAC;QAC3B,MAAM,CAAC,GAAG,MAAM,MAAM,CAAC,EAAE,CAAC,CAAC;QAC3B,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACzB,CAAC;IACD,MAAM,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IAEvD,GAAG,CAAC,KAAK,CACP,EAAE,KAAK,EAAE,SAAS,CAAC,MAAM,EAAE,SAAS,EAAE,EACtC,2CAA2C,CAC5C,CAAC;IAEF,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,MAAM,MAAM,GAAG,qBAAqB,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QAE3D,KAAK,CAAC,sBAAsB,CAC1B,MAAM,EACN,SAAS,EACT,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,EACtC,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE;YACf,IAAI,GAAG,EAAE,CAAC;gBACR,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,EAAE,0CAA0C,CAAC,CAAC;gBAC9D,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;gBACvB,OAAO,CAAC,EAAE,CAAC,CAAC;gBACZ,OAAO;YACT,CAAC;YAED,MAAM,OAAO,GAAgB,EAAE,CAAC;YAEhC,OAAO,CAAC,SAAS,EAAE,CAAC;YACpB,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAkB,EAAE,EAAE;gBACzC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;oBAClC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;oBACvD,OAAO,CAAC,IAAI,CAAC;wBACX,IAAI,EAAE,KAAK,CAAC,QAAQ;wBACpB,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC;wBACvC,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI;wBACpC,cAAc,EAAE,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC;wBAC5C,gBAAgB,EAAE,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC;wBAChD,KAAK,EAAE,KAAK,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI;qBAC5E,CAAC,CAAC;gBACL,CAAC;gBACD,OAAO,CAAC,SAAS,EAAE,CAAC;YACtB,CAAC,CAAC,CAAC;YAEH,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBACrB,GAAG,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,EAAE,4BAA4B,CAAC,CAAC;gBACpE,OAAO,CAAC,OAAO,CAAC,CAAC;YACnB,CAAC,CAAC,CAAC;YACH,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;gBAC5B,GAAG,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,EAAE,qCAAqC,CAAC,CAAC;gBAC3D,OAAO,CAAC,OAAO,CAAC,CAAC;YACnB,CAAC,CAAC,CAAC;QACL,CAAC,CACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;;;GAMG;AACH,SAAS,qBAAqB,CAC5B,SAAmB,EACnB,SAAmB;IAEnB,gCAAgC;IAChC,MAAM,WAAW,GAAa,EAAE,CAAC;IACjC,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACzB,MAAM,IAAI,IAAI,CAAC;IACjB,CAAC;IAED,uDAAuD;IACvD,8DAA8D;IAC9D,MAAM,MAAM,GAAG,IAAK,KAAK,CAAC,kBAA0B,EAA8B,CAAC;IAEnF,2EAA2E;IAC3E,8DAA8D;IAC7D,MAAc,CAAC,mBAAmB,GAAG,UAAU,KAAa,EAAE,GAAW;QACxE,MAAM,QAAQ,GAAG,IAAI,QAAQ,CAAC,EAAE,IAAI,KAAI,CAAC,EAAE,CAAC,CAAC;QAE7C,SAAS,CAAC,KAAK,EAAE,GAAG,EAAE,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YAC5C,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACxB,CAAC,CAAC,CAAC;QAEH,OAAO,QAAQ,CAAC;IAClB,CAAC,CAAC;IAEF,KAAK,UAAU,SAAS,CAAC,KAAa,EAAE,GAAW,EAAE,QAAkB;QACrE,IAAI,SAAS,GAAG,GAAG,GAAG,KAAK,CAAC;QAC5B,IAAI,YAAY,GAAG,KAAK,CAAC;QAEzB,OAAO,SAAS,GAAG,CAAC,EAAE,CAAC;YACrB,uCAAuC;YACvC,IAAI,OAAO,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC;YACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC5C,IAAI,CAAC,GAAG,CAAC,GAAG,WAAW,CAAC,MAAM,IAAI,YAAY,GAAG,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;oBACpE,OAAO,GAAG,CAAC,CAAC;oBACZ,MAAM;gBACR,CAAC;YACH,CAAC;YAED,MAAM,WAAW,GAAG,YAAY,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;YACxD,MAAM,aAAa,GAAG,SAAS,CAAC,OAAO,CAAC,GAAG,WAAW,CAAC;YACvD,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;YAElD,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,GAAG,CAAC,CAAC;YACjD,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBACjC,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;gBACjE,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC;gBAC1C,SAAS,IAAI,SAAS,CAAC;gBACvB,YAAY,IAAI,SAAS,CAAC;YAC5B,CAAC;oBAAS,CAAC;gBACT,MAAM,EAAE,CAAC,KAAK,EAAE,CAAC;YACnB,CAAC;QACH,CAAC;QAED,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,uBAAuB;IAC9C,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"} \ No newline at end of file diff --git a/worker/dist/db/client.d.ts b/worker/dist/db/client.d.ts new file mode 100644 index 0000000..f99ad7c --- /dev/null +++ b/worker/dist/db/client.d.ts @@ -0,0 +1,7 @@ +import { PrismaClient } from "@prisma/client"; +import { PrismaPg } from "@prisma/adapter-pg"; +declare const pool: import("pg").Pool; +export declare const db: PrismaClient<{ + adapter: PrismaPg; +}, never, import("@prisma/client/runtime/client").DefaultArgs>; +export { pool }; diff --git a/worker/dist/db/client.js b/worker/dist/db/client.js new file mode 100644 index 0000000..2307f7c --- /dev/null +++ b/worker/dist/db/client.js @@ -0,0 +1,12 @@ +import { PrismaClient } from "@prisma/client"; +import { PrismaPg } from "@prisma/adapter-pg"; +import pg from "pg"; +import { config } from "../util/config.js"; +const pool = new pg.Pool({ + connectionString: config.databaseUrl, + max: 5, +}); +const adapter = new PrismaPg(pool); +export const db = new PrismaClient({ adapter }); +export { pool }; +//# sourceMappingURL=client.js.map \ No newline at end of file diff --git a/worker/dist/db/client.js.map b/worker/dist/db/client.js.map new file mode 100644 index 0000000..5508b5b --- /dev/null +++ b/worker/dist/db/client.js.map @@ -0,0 +1 @@ +{"version":3,"file":"client.js","sourceRoot":"","sources":["../../src/db/client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,MAAM,IAAI,GAAG,IAAI,EAAE,CAAC,IAAI,CAAC;IACvB,gBAAgB,EAAE,MAAM,CAAC,WAAW;IACpC,GAAG,EAAE,CAAC;CACP,CAAC,CAAC;AAEH,MAAM,OAAO,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC;AACnC,MAAM,CAAC,MAAM,EAAE,GAAG,IAAI,YAAY,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC;AAEhD,OAAO,EAAE,IAAI,EAAE,CAAC"} \ No newline at end of file diff --git a/worker/dist/db/locks.d.ts b/worker/dist/db/locks.d.ts new file mode 100644 index 0000000..646c184 --- /dev/null +++ b/worker/dist/db/locks.d.ts @@ -0,0 +1,9 @@ +/** + * Try to acquire a PostgreSQL advisory lock for an account. + * Returns true if acquired, false if already held by another session. + */ +export declare function tryAcquireLock(accountId: string): Promise; +/** + * Release the advisory lock for an account. + */ +export declare function releaseLock(accountId: string): Promise; diff --git a/worker/dist/db/locks.js b/worker/dist/db/locks.js new file mode 100644 index 0000000..926011d --- /dev/null +++ b/worker/dist/db/locks.js @@ -0,0 +1,53 @@ +import { pool } from "./client.js"; +import { childLogger } from "../util/logger.js"; +const log = childLogger("locks"); +/** + * Derive a stable 32-bit integer lock ID from an account ID string. + * PostgreSQL advisory locks use bigint, but we use 32-bit for safety. + */ +function hashToLockId(accountId) { + let hash = 0; + for (let i = 0; i < accountId.length; i++) { + const char = accountId.charCodeAt(i); + hash = (hash << 5) - hash + char; + hash |= 0; // Convert to 32-bit integer + } + return Math.abs(hash); +} +/** + * Try to acquire a PostgreSQL advisory lock for an account. + * Returns true if acquired, false if already held by another session. + */ +export async function tryAcquireLock(accountId) { + const lockId = hashToLockId(accountId); + const client = await pool.connect(); + try { + const result = await client.query("SELECT pg_try_advisory_lock($1)", [lockId]); + const acquired = result.rows[0]?.pg_try_advisory_lock ?? false; + if (acquired) { + log.debug({ accountId, lockId }, "Advisory lock acquired"); + } + else { + log.debug({ accountId, lockId }, "Advisory lock already held"); + } + return acquired; + } + finally { + client.release(); + } +} +/** + * Release the advisory lock for an account. + */ +export async function releaseLock(accountId) { + const lockId = hashToLockId(accountId); + const client = await pool.connect(); + try { + await client.query("SELECT pg_advisory_unlock($1)", [lockId]); + log.debug({ accountId, lockId }, "Advisory lock released"); + } + finally { + client.release(); + } +} +//# sourceMappingURL=locks.js.map \ No newline at end of file diff --git a/worker/dist/db/locks.js.map b/worker/dist/db/locks.js.map new file mode 100644 index 0000000..2cade3d --- /dev/null +++ b/worker/dist/db/locks.js.map @@ -0,0 +1 @@ +{"version":3,"file":"locks.js","sourceRoot":"","sources":["../../src/db/locks.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AACnC,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,MAAM,GAAG,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;AAEjC;;;GAGG;AACH,SAAS,YAAY,CAAC,SAAiB;IACrC,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,MAAM,IAAI,GAAG,SAAS,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QACrC,IAAI,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;QACjC,IAAI,IAAI,CAAC,CAAC,CAAC,4BAA4B;IACzC,CAAC;IACD,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;AACxB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,SAAiB;IACpD,MAAM,MAAM,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC;IACvC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;IACpC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,KAAK,CAC/B,iCAAiC,EACjC,CAAC,MAAM,CAAC,CACT,CAAC;QACF,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,oBAAoB,IAAI,KAAK,CAAC;QAC/D,IAAI,QAAQ,EAAE,CAAC;YACb,GAAG,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,wBAAwB,CAAC,CAAC;QAC7D,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,4BAA4B,CAAC,CAAC;QACjE,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;YAAS,CAAC;QACT,MAAM,CAAC,OAAO,EAAE,CAAC;IACnB,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,SAAiB;IACjD,MAAM,MAAM,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC;IACvC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;IACpC,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,KAAK,CAAC,+BAA+B,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC;QAC9D,GAAG,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,wBAAwB,CAAC,CAAC;IAC7D,CAAC;YAAS,CAAC;QACT,MAAM,CAAC,OAAO,EAAE,CAAC;IACnB,CAAC;AACH,CAAC"} \ No newline at end of file diff --git a/worker/dist/db/queries.d.ts b/worker/dist/db/queries.d.ts new file mode 100644 index 0000000..7e6f292 --- /dev/null +++ b/worker/dist/db/queries.d.ts @@ -0,0 +1,356 @@ +import type { ArchiveType, FetchStatus } from "@prisma/client"; +export declare function getActiveAccounts(): Promise<{ + id: string; + phone: string; + displayName: string | null; + isActive: boolean; + authState: import("@prisma/client").$Enums.AuthState; + authCode: string | null; + lastSeenAt: Date | null; + createdAt: Date; + updatedAt: Date; +}[]>; +export declare function getPendingAccounts(): Promise<{ + id: string; + phone: string; + displayName: string | null; + isActive: boolean; + authState: import("@prisma/client").$Enums.AuthState; + authCode: string | null; + lastSeenAt: Date | null; + createdAt: Date; + updatedAt: Date; +}[]>; +export declare function hasAnyChannels(): Promise; +export declare function getSourceChannelMappings(accountId: string): Promise<({ + channel: { + id: string; + isActive: boolean; + createdAt: Date; + updatedAt: Date; + telegramId: bigint; + title: string; + type: import("@prisma/client").$Enums.ChannelType; + isForum: boolean; + }; +} & { + accountId: string; + id: string; + createdAt: Date; + channelId: string; + role: import("@prisma/client").$Enums.ChannelRole; + lastProcessedMessageId: bigint | null; +})[]>; +export declare function getGlobalDestinationChannel(): Promise<{ + id: string; + isActive: boolean; + createdAt: Date; + updatedAt: Date; + telegramId: bigint; + title: string; + type: import("@prisma/client").$Enums.ChannelType; + isForum: boolean; +} | null>; +export declare function getGlobalSetting(key: string): Promise; +export declare function setGlobalSetting(key: string, value: string): Promise<{ + updatedAt: Date; + key: string; + value: string; +}>; +export declare function packageExistsByHash(contentHash: string): Promise; +/** + * Check if a package already exists for a given source message ID + * AND was successfully uploaded to the destination (destMessageId is set). + * Used as an early skip before downloading. + */ +export declare function packageExistsBySourceMessage(sourceChannelId: string, sourceMessageId: bigint): Promise; +/** + * Delete orphaned Package rows that have the same content hash but never + * completed the upload (destMessageId is null). Called before creating a + * new complete record to avoid unique constraint violations. + */ +export declare function deleteOrphanedPackageByHash(contentHash: string): Promise; +export interface CreatePackageInput { + contentHash: string; + fileName: string; + fileSize: bigint; + archiveType: ArchiveType; + sourceChannelId: string; + sourceMessageId: bigint; + sourceTopicId?: bigint | null; + destChannelId?: string; + destMessageId?: bigint; + isMultipart: boolean; + partCount: number; + ingestionRunId: string; + creator?: string | null; + previewData?: Buffer | null; + previewMsgId?: bigint | null; + files: { + path: string; + fileName: string; + extension: string | null; + compressedSize: bigint; + uncompressedSize: bigint; + crc32: string | null; + }[]; +} +export declare function createPackageWithFiles(input: CreatePackageInput): Promise<{ + id: string; + createdAt: Date; + contentHash: string; + fileName: string; + fileSize: bigint; + archiveType: import("@prisma/client").$Enums.ArchiveType; + creator: string | null; + sourceChannelId: string; + sourceMessageId: bigint; + sourceTopicId: bigint | null; + destChannelId: string | null; + destMessageId: bigint | null; + isMultipart: boolean; + partCount: number; + fileCount: number; + previewData: import("@prisma/client/runtime/client").Bytes | null; + previewMsgId: bigint | null; + indexedAt: Date; + ingestionRunId: string | null; +}>; +export declare function createIngestionRun(accountId: string): Promise<{ + accountId: string; + id: string; + status: import("@prisma/client").$Enums.IngestionStatus; + startedAt: Date; + finishedAt: Date | null; + messagesScanned: number; + zipsFound: number; + zipsDuplicate: number; + zipsIngested: number; + errorMessage: string | null; + currentActivity: string | null; + currentStep: string | null; + currentChannel: string | null; + currentFile: string | null; + currentFileNum: number | null; + totalFiles: number | null; + downloadedBytes: bigint | null; + totalBytes: bigint | null; + downloadPercent: number | null; + lastActivityAt: Date | null; +}>; +export interface ActivityUpdate { + currentActivity: string; + currentStep: string; + currentChannel?: string | null; + currentFile?: string | null; + currentFileNum?: number | null; + totalFiles?: number | null; + downloadedBytes?: bigint | null; + totalBytes?: bigint | null; + downloadPercent?: number | null; + messagesScanned?: number; + zipsFound?: number; + zipsDuplicate?: number; + zipsIngested?: number; +} +export declare function updateRunActivity(runId: string, activity: ActivityUpdate): Promise<{ + accountId: string; + id: string; + status: import("@prisma/client").$Enums.IngestionStatus; + startedAt: Date; + finishedAt: Date | null; + messagesScanned: number; + zipsFound: number; + zipsDuplicate: number; + zipsIngested: number; + errorMessage: string | null; + currentActivity: string | null; + currentStep: string | null; + currentChannel: string | null; + currentFile: string | null; + currentFileNum: number | null; + totalFiles: number | null; + downloadedBytes: bigint | null; + totalBytes: bigint | null; + downloadPercent: number | null; + lastActivityAt: Date | null; +}>; +export declare function completeIngestionRun(runId: string, counters: { + messagesScanned: number; + zipsFound: number; + zipsDuplicate: number; + zipsIngested: number; +}): Promise<{ + accountId: string; + id: string; + status: import("@prisma/client").$Enums.IngestionStatus; + startedAt: Date; + finishedAt: Date | null; + messagesScanned: number; + zipsFound: number; + zipsDuplicate: number; + zipsIngested: number; + errorMessage: string | null; + currentActivity: string | null; + currentStep: string | null; + currentChannel: string | null; + currentFile: string | null; + currentFileNum: number | null; + totalFiles: number | null; + downloadedBytes: bigint | null; + totalBytes: bigint | null; + downloadPercent: number | null; + lastActivityAt: Date | null; +}>; +export declare function failIngestionRun(runId: string, errorMessage: string): Promise<{ + accountId: string; + id: string; + status: import("@prisma/client").$Enums.IngestionStatus; + startedAt: Date; + finishedAt: Date | null; + messagesScanned: number; + zipsFound: number; + zipsDuplicate: number; + zipsIngested: number; + errorMessage: string | null; + currentActivity: string | null; + currentStep: string | null; + currentChannel: string | null; + currentFile: string | null; + currentFileNum: number | null; + totalFiles: number | null; + downloadedBytes: bigint | null; + totalBytes: bigint | null; + downloadPercent: number | null; + lastActivityAt: Date | null; +}>; +export declare function updateLastProcessedMessage(mappingId: string, messageId: bigint): Promise<{ + accountId: string; + id: string; + createdAt: Date; + channelId: string; + role: import("@prisma/client").$Enums.ChannelRole; + lastProcessedMessageId: bigint | null; +}>; +export declare function markStaleRunsAsFailed(): Promise; +export declare function updateAccountAuthState(accountId: string, authState: "PENDING" | "AWAITING_CODE" | "AWAITING_PASSWORD" | "AUTHENTICATED" | "EXPIRED", authCode?: string | null): Promise<{ + id: string; + phone: string; + displayName: string | null; + isActive: boolean; + authState: import("@prisma/client").$Enums.AuthState; + authCode: string | null; + lastSeenAt: Date | null; + createdAt: Date; + updatedAt: Date; +}>; +export declare function getAccountAuthCode(accountId: string): Promise<{ + authState: import("@prisma/client").$Enums.AuthState; + authCode: string | null; +} | null>; +export interface UpsertChannelInput { + telegramId: bigint; + title: string; + type: "SOURCE" | "DESTINATION"; + isForum: boolean; + isActive?: boolean; +} +/** + * Upsert a channel by telegramId. Returns the channel record. + * If it already exists, update title and forum status. + * New channels default to disabled (isActive: false) so the admin must + * explicitly enable them before the worker processes them. + * Pass isActive: true for DESTINATION channels that must be active immediately. + */ +export declare function upsertChannel(input: UpsertChannelInput): Promise<{ + id: string; + isActive: boolean; + createdAt: Date; + updatedAt: Date; + telegramId: bigint; + title: string; + type: import("@prisma/client").$Enums.ChannelType; + isForum: boolean; +}>; +/** + * Link an account to a channel if not already linked. + * Uses a try/catch on unique constraint to make it idempotent. + */ +export declare function ensureAccountChannelLink(accountId: string, channelId: string, role: "READER" | "WRITER"): Promise<{ + accountId: string; + id: string; + createdAt: Date; + channelId: string; + role: import("@prisma/client").$Enums.ChannelRole; + lastProcessedMessageId: bigint | null; +} | null>; +export declare function setChannelForum(channelId: string, isForum: boolean): Promise<{ + id: string; + isActive: boolean; + createdAt: Date; + updatedAt: Date; + telegramId: bigint; + title: string; + type: import("@prisma/client").$Enums.ChannelType; + isForum: boolean; +}>; +export declare function getTopicProgress(mappingId: string): Promise<{ + id: string; + lastProcessedMessageId: bigint | null; + accountChannelMapId: string; + topicId: bigint; + topicName: string | null; +}[]>; +export declare function upsertTopicProgress(mappingId: string, topicId: bigint, topicName: string | null, lastProcessedMessageId: bigint): Promise<{ + id: string; + lastProcessedMessageId: bigint | null; + accountChannelMapId: string; + topicId: bigint; + topicName: string | null; +}>; +export declare function getChannelFetchRequest(requestId: string): Promise<({ + account: { + id: string; + phone: string; + displayName: string | null; + isActive: boolean; + authState: import("@prisma/client").$Enums.AuthState; + authCode: string | null; + lastSeenAt: Date | null; + createdAt: Date; + updatedAt: Date; + }; +} & { + error: string | null; + accountId: string; + id: string; + createdAt: Date; + updatedAt: Date; + status: import("@prisma/client").$Enums.FetchStatus; + resultJson: string | null; +}) | null>; +export declare function updateFetchRequestStatus(requestId: string, status: FetchStatus, extra?: { + resultJson?: string; + error?: string; +}): Promise<{ + error: string | null; + accountId: string; + id: string; + createdAt: Date; + updatedAt: Date; + status: import("@prisma/client").$Enums.FetchStatus; + resultJson: string | null; +}>; +export declare function getAccountLinkedChannelIds(accountId: string): Promise>; +export declare function getExistingChannelsByTelegramId(): Promise>; +export declare function getAccountById(accountId: string): Promise<{ + id: string; + phone: string; + displayName: string | null; + isActive: boolean; + authState: import("@prisma/client").$Enums.AuthState; + authCode: string | null; + lastSeenAt: Date | null; + createdAt: Date; + updatedAt: Date; +} | null>; diff --git a/worker/dist/db/queries.js b/worker/dist/db/queries.js new file mode 100644 index 0000000..b6e4850 --- /dev/null +++ b/worker/dist/db/queries.js @@ -0,0 +1,319 @@ +import { db } from "./client.js"; +export async function getActiveAccounts() { + return db.telegramAccount.findMany({ + where: { isActive: true, authState: "AUTHENTICATED" }, + }); +} +export async function getPendingAccounts() { + return db.telegramAccount.findMany({ + where: { isActive: true, authState: "PENDING" }, + }); +} +export async function hasAnyChannels() { + const count = await db.telegramChannel.count(); + return count > 0; +} +export async function getSourceChannelMappings(accountId) { + return db.accountChannelMap.findMany({ + where: { + accountId, + role: "READER", + channel: { type: "SOURCE", isActive: true }, + }, + include: { channel: true }, + }); +} +// ── Global destination channel ── +export async function getGlobalDestinationChannel() { + const setting = await db.globalSetting.findUnique({ + where: { key: "destination_channel_id" }, + }); + if (!setting) + return null; + return db.telegramChannel.findFirst({ + where: { id: setting.value, type: "DESTINATION", isActive: true }, + }); +} +export async function getGlobalSetting(key) { + const setting = await db.globalSetting.findUnique({ where: { key } }); + return setting?.value ?? null; +} +export async function setGlobalSetting(key, value) { + return db.globalSetting.upsert({ + where: { key }, + create: { key, value }, + update: { value }, + }); +} +export async function packageExistsByHash(contentHash) { + const pkg = await db.package.findFirst({ + where: { contentHash, destMessageId: { not: null } }, + select: { id: true }, + }); + return pkg !== null; +} +/** + * Check if a package already exists for a given source message ID + * AND was successfully uploaded to the destination (destMessageId is set). + * Used as an early skip before downloading. + */ +export async function packageExistsBySourceMessage(sourceChannelId, sourceMessageId) { + const pkg = await db.package.findFirst({ + where: { sourceChannelId, sourceMessageId, destMessageId: { not: null } }, + select: { id: true }, + }); + return pkg !== null; +} +/** + * Delete orphaned Package rows that have the same content hash but never + * completed the upload (destMessageId is null). Called before creating a + * new complete record to avoid unique constraint violations. + */ +export async function deleteOrphanedPackageByHash(contentHash) { + await db.package.deleteMany({ + where: { contentHash, destMessageId: null }, + }); +} +export async function createPackageWithFiles(input) { + const pkg = await db.package.create({ + data: { + contentHash: input.contentHash, + fileName: input.fileName, + fileSize: input.fileSize, + archiveType: input.archiveType, + sourceChannelId: input.sourceChannelId, + sourceMessageId: input.sourceMessageId, + sourceTopicId: input.sourceTopicId ?? undefined, + destChannelId: input.destChannelId, + destMessageId: input.destMessageId, + isMultipart: input.isMultipart, + partCount: input.partCount, + fileCount: input.files.length, + ingestionRunId: input.ingestionRunId, + creator: input.creator ?? undefined, + previewData: input.previewData ? new Uint8Array(input.previewData) : undefined, + previewMsgId: input.previewMsgId ?? undefined, + files: { + create: input.files, + }, + }, + }); + // Notify the bot service about the new package (for subscription alerts) + try { + await db.$queryRawUnsafe(`SELECT pg_notify('new_package', $1)`, JSON.stringify({ + packageId: pkg.id, + fileName: input.fileName, + creator: input.creator ?? null, + })); + } + catch { + // Best-effort — don't fail the ingestion if notification fails + } + return pkg; +} +export async function createIngestionRun(accountId) { + return db.ingestionRun.create({ + data: { + accountId, + status: "RUNNING", + currentActivity: "Starting ingestion run", + currentStep: "initializing", + lastActivityAt: new Date(), + }, + }); +} +export async function updateRunActivity(runId, activity) { + return db.ingestionRun.update({ + where: { id: runId }, + data: { + currentActivity: activity.currentActivity, + currentStep: activity.currentStep, + currentChannel: activity.currentChannel ?? undefined, + currentFile: activity.currentFile ?? undefined, + currentFileNum: activity.currentFileNum ?? undefined, + totalFiles: activity.totalFiles ?? undefined, + downloadedBytes: activity.downloadedBytes ?? undefined, + totalBytes: activity.totalBytes ?? undefined, + downloadPercent: activity.downloadPercent ?? undefined, + lastActivityAt: new Date(), + ...(activity.messagesScanned !== undefined && { messagesScanned: activity.messagesScanned }), + ...(activity.zipsFound !== undefined && { zipsFound: activity.zipsFound }), + ...(activity.zipsDuplicate !== undefined && { zipsDuplicate: activity.zipsDuplicate }), + ...(activity.zipsIngested !== undefined && { zipsIngested: activity.zipsIngested }), + }, + }); +} +const CLEAR_ACTIVITY = { + currentActivity: null, + currentStep: null, + currentChannel: null, + currentFile: null, + currentFileNum: null, + totalFiles: null, + downloadedBytes: null, + totalBytes: null, + downloadPercent: null, + lastActivityAt: new Date(), +}; +export async function completeIngestionRun(runId, counters) { + return db.ingestionRun.update({ + where: { id: runId }, + data: { + status: "COMPLETED", + finishedAt: new Date(), + ...counters, + ...CLEAR_ACTIVITY, + }, + }); +} +export async function failIngestionRun(runId, errorMessage) { + return db.ingestionRun.update({ + where: { id: runId }, + data: { + status: "FAILED", + finishedAt: new Date(), + errorMessage, + ...CLEAR_ACTIVITY, + }, + }); +} +export async function updateLastProcessedMessage(mappingId, messageId) { + return db.accountChannelMap.update({ + where: { id: mappingId }, + data: { lastProcessedMessageId: messageId }, + }); +} +export async function markStaleRunsAsFailed() { + return db.ingestionRun.updateMany({ + where: { status: "RUNNING" }, + data: { + status: "FAILED", + finishedAt: new Date(), + errorMessage: "Worker restarted — run was still marked as RUNNING", + }, + }); +} +export async function updateAccountAuthState(accountId, authState, authCode) { + return db.telegramAccount.update({ + where: { id: accountId }, + data: { authState, authCode, lastSeenAt: authState === "AUTHENTICATED" ? new Date() : undefined }, + }); +} +export async function getAccountAuthCode(accountId) { + const account = await db.telegramAccount.findUnique({ + where: { id: accountId }, + select: { authCode: true, authState: true }, + }); + return account; +} +/** + * Upsert a channel by telegramId. Returns the channel record. + * If it already exists, update title and forum status. + * New channels default to disabled (isActive: false) so the admin must + * explicitly enable them before the worker processes them. + * Pass isActive: true for DESTINATION channels that must be active immediately. + */ +export async function upsertChannel(input) { + return db.telegramChannel.upsert({ + where: { telegramId: input.telegramId }, + create: { + telegramId: input.telegramId, + title: input.title, + type: input.type, + isForum: input.isForum, + isActive: input.isActive ?? false, + }, + update: { + title: input.title, + isForum: input.isForum, + }, + }); +} +/** + * Link an account to a channel if not already linked. + * Uses a try/catch on unique constraint to make it idempotent. + */ +export async function ensureAccountChannelLink(accountId, channelId, role) { + try { + return await db.accountChannelMap.create({ + data: { accountId, channelId, role }, + }); + } + catch (err) { + // Already linked — ignore unique constraint violation + if (err instanceof Error && err.message.includes("Unique constraint")) { + return null; + } + throw err; + } +} +// ── Forum / Topic progress ── +export async function setChannelForum(channelId, isForum) { + return db.telegramChannel.update({ + where: { id: channelId }, + data: { isForum }, + }); +} +export async function getTopicProgress(mappingId) { + return db.topicProgress.findMany({ + where: { accountChannelMapId: mappingId }, + }); +} +export async function upsertTopicProgress(mappingId, topicId, topicName, lastProcessedMessageId) { + return db.topicProgress.upsert({ + where: { + accountChannelMapId_topicId: { + accountChannelMapId: mappingId, + topicId, + }, + }, + create: { + accountChannelMapId: mappingId, + topicId, + topicName, + lastProcessedMessageId, + }, + update: { + topicName, + lastProcessedMessageId, + }, + }); +} +// ── Channel fetch requests (DB-mediated communication with web app) ── +export async function getChannelFetchRequest(requestId) { + return db.channelFetchRequest.findUnique({ + where: { id: requestId }, + include: { account: true }, + }); +} +export async function updateFetchRequestStatus(requestId, status, extra) { + return db.channelFetchRequest.update({ + where: { id: requestId }, + data: { + status, + resultJson: extra?.resultJson ?? undefined, + error: extra?.error ?? undefined, + }, + }); +} +export async function getAccountLinkedChannelIds(accountId) { + const links = await db.accountChannelMap.findMany({ + where: { accountId }, + select: { channel: { select: { telegramId: true } } }, + }); + return new Set(links.map((l) => l.channel.telegramId.toString())); +} +export async function getExistingChannelsByTelegramId() { + const channels = await db.telegramChannel.findMany({ + select: { id: true, telegramId: true }, + }); + const map = new Map(); + for (const ch of channels) { + map.set(ch.telegramId.toString(), ch.id); + } + return map; +} +export async function getAccountById(accountId) { + return db.telegramAccount.findUnique({ where: { id: accountId } }); +} +//# sourceMappingURL=queries.js.map \ No newline at end of file diff --git a/worker/dist/db/queries.js.map b/worker/dist/db/queries.js.map new file mode 100644 index 0000000..fddee22 --- /dev/null +++ b/worker/dist/db/queries.js.map @@ -0,0 +1 @@ +{"version":3,"file":"queries.js","sourceRoot":"","sources":["../../src/db/queries.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,EAAE,EAAE,MAAM,aAAa,CAAC;AAGjC,MAAM,CAAC,KAAK,UAAU,iBAAiB;IACrC,OAAO,EAAE,CAAC,eAAe,CAAC,QAAQ,CAAC;QACjC,KAAK,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAE,eAAe,EAAE;KACtD,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB;IACtC,OAAO,EAAE,CAAC,eAAe,CAAC,QAAQ,CAAC;QACjC,KAAK,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE;KAChD,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc;IAClC,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;IAC/C,OAAO,KAAK,GAAG,CAAC,CAAC;AACnB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAAC,SAAiB;IAC9D,OAAO,EAAE,CAAC,iBAAiB,CAAC,QAAQ,CAAC;QACnC,KAAK,EAAE;YACL,SAAS;YACT,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE;SAC5C;QACD,OAAO,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE;KAC3B,CAAC,CAAC;AACL,CAAC;AAED,mCAAmC;AAEnC,MAAM,CAAC,KAAK,UAAU,2BAA2B;IAC/C,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,aAAa,CAAC,UAAU,CAAC;QAChD,KAAK,EAAE,EAAE,GAAG,EAAE,wBAAwB,EAAE;KACzC,CAAC,CAAC;IACH,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IAC1B,OAAO,EAAE,CAAC,eAAe,CAAC,SAAS,CAAC;QAClC,KAAK,EAAE,EAAE,EAAE,EAAE,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE,aAAa,EAAE,QAAQ,EAAE,IAAI,EAAE;KAClE,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,GAAW;IAChD,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,aAAa,CAAC,UAAU,CAAC,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;IACtE,OAAO,OAAO,EAAE,KAAK,IAAI,IAAI,CAAC;AAChC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,GAAW,EAAE,KAAa;IAC/D,OAAO,EAAE,CAAC,aAAa,CAAC,MAAM,CAAC;QAC7B,KAAK,EAAE,EAAE,GAAG,EAAE;QACd,MAAM,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE;QACtB,MAAM,EAAE,EAAE,KAAK,EAAE;KAClB,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,WAAmB;IAC3D,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC;QACrC,KAAK,EAAE,EAAE,WAAW,EAAE,aAAa,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE;QACpD,MAAM,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE;KACrB,CAAC,CAAC;IACH,OAAO,GAAG,KAAK,IAAI,CAAC;AACtB,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,4BAA4B,CAChD,eAAuB,EACvB,eAAuB;IAEvB,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC;QACrC,KAAK,EAAE,EAAE,eAAe,EAAE,eAAe,EAAE,aAAa,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE;QACzE,MAAM,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE;KACrB,CAAC,CAAC;IACH,OAAO,GAAG,KAAK,IAAI,CAAC;AACtB,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAAC,WAAmB;IACnE,MAAM,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC;QAC1B,KAAK,EAAE,EAAE,WAAW,EAAE,aAAa,EAAE,IAAI,EAAE;KAC5C,CAAC,CAAC;AACL,CAAC;AA4BD,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAC,KAAyB;IACpE,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC;QAClC,IAAI,EAAE;YACJ,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,eAAe,EAAE,KAAK,CAAC,eAAe;YACtC,eAAe,EAAE,KAAK,CAAC,eAAe;YACtC,aAAa,EAAE,KAAK,CAAC,aAAa,IAAI,SAAS;YAC/C,aAAa,EAAE,KAAK,CAAC,aAAa;YAClC,aAAa,EAAE,KAAK,CAAC,aAAa;YAClC,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM;YAC7B,cAAc,EAAE,KAAK,CAAC,cAAc;YACpC,OAAO,EAAE,KAAK,CAAC,OAAO,IAAI,SAAS;YACnC,WAAW,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,UAAU,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,SAAS;YAC9E,YAAY,EAAE,KAAK,CAAC,YAAY,IAAI,SAAS;YAC7C,KAAK,EAAE;gBACL,MAAM,EAAE,KAAK,CAAC,KAAK;aACpB;SACF;KACF,CAAC,CAAC;IAEH,yEAAyE;IACzE,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,eAAe,CACtB,qCAAqC,EACrC,IAAI,CAAC,SAAS,CAAC;YACb,SAAS,EAAE,GAAG,CAAC,EAAE;YACjB,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,OAAO,EAAE,KAAK,CAAC,OAAO,IAAI,IAAI;SAC/B,CAAC,CACH,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,+DAA+D;IACjE,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,SAAiB;IACxD,OAAO,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC;QAC5B,IAAI,EAAE;YACJ,SAAS;YACT,MAAM,EAAE,SAAS;YACjB,eAAe,EAAE,wBAAwB;YACzC,WAAW,EAAE,cAAc;YAC3B,cAAc,EAAE,IAAI,IAAI,EAAE;SAC3B;KACF,CAAC,CAAC;AACL,CAAC;AAkBD,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,KAAa,EACb,QAAwB;IAExB,OAAO,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC;QAC5B,KAAK,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE;QACpB,IAAI,EAAE;YACJ,eAAe,EAAE,QAAQ,CAAC,eAAe;YACzC,WAAW,EAAE,QAAQ,CAAC,WAAW;YACjC,cAAc,EAAE,QAAQ,CAAC,cAAc,IAAI,SAAS;YACpD,WAAW,EAAE,QAAQ,CAAC,WAAW,IAAI,SAAS;YAC9C,cAAc,EAAE,QAAQ,CAAC,cAAc,IAAI,SAAS;YACpD,UAAU,EAAE,QAAQ,CAAC,UAAU,IAAI,SAAS;YAC5C,eAAe,EAAE,QAAQ,CAAC,eAAe,IAAI,SAAS;YACtD,UAAU,EAAE,QAAQ,CAAC,UAAU,IAAI,SAAS;YAC5C,eAAe,EAAE,QAAQ,CAAC,eAAe,IAAI,SAAS;YACtD,cAAc,EAAE,IAAI,IAAI,EAAE;YAC1B,GAAG,CAAC,QAAQ,CAAC,eAAe,KAAK,SAAS,IAAI,EAAE,eAAe,EAAE,QAAQ,CAAC,eAAe,EAAE,CAAC;YAC5F,GAAG,CAAC,QAAQ,CAAC,SAAS,KAAK,SAAS,IAAI,EAAE,SAAS,EAAE,QAAQ,CAAC,SAAS,EAAE,CAAC;YAC1E,GAAG,CAAC,QAAQ,CAAC,aAAa,KAAK,SAAS,IAAI,EAAE,aAAa,EAAE,QAAQ,CAAC,aAAa,EAAE,CAAC;YACtF,GAAG,CAAC,QAAQ,CAAC,YAAY,KAAK,SAAS,IAAI,EAAE,YAAY,EAAE,QAAQ,CAAC,YAAY,EAAE,CAAC;SACpF;KACF,CAAC,CAAC;AACL,CAAC;AAED,MAAM,cAAc,GAAG;IACrB,eAAe,EAAE,IAAI;IACrB,WAAW,EAAE,IAAI;IACjB,cAAc,EAAE,IAAI;IACpB,WAAW,EAAE,IAAI;IACjB,cAAc,EAAE,IAAI;IACpB,UAAU,EAAE,IAAI;IAChB,eAAe,EAAE,IAAI;IACrB,UAAU,EAAE,IAAI;IAChB,eAAe,EAAE,IAAI;IACrB,cAAc,EAAE,IAAI,IAAI,EAAE;CAC3B,CAAC;AAEF,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,KAAa,EACb,QAKC;IAED,OAAO,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC;QAC5B,KAAK,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE;QACpB,IAAI,EAAE;YACJ,MAAM,EAAE,WAAW;YACnB,UAAU,EAAE,IAAI,IAAI,EAAE;YACtB,GAAG,QAAQ;YACX,GAAG,cAAc;SAClB;KACF,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,KAAa,EAAE,YAAoB;IACxE,OAAO,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC;QAC5B,KAAK,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE;QACpB,IAAI,EAAE;YACJ,MAAM,EAAE,QAAQ;YAChB,UAAU,EAAE,IAAI,IAAI,EAAE;YACtB,YAAY;YACZ,GAAG,cAAc;SAClB;KACF,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAC9C,SAAiB,EACjB,SAAiB;IAEjB,OAAO,EAAE,CAAC,iBAAiB,CAAC,MAAM,CAAC;QACjC,KAAK,EAAE,EAAE,EAAE,EAAE,SAAS,EAAE;QACxB,IAAI,EAAE,EAAE,sBAAsB,EAAE,SAAS,EAAE;KAC5C,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB;IACzC,OAAO,EAAE,CAAC,YAAY,CAAC,UAAU,CAAC;QAChC,KAAK,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE;QAC5B,IAAI,EAAE;YACJ,MAAM,EAAE,QAAQ;YAChB,UAAU,EAAE,IAAI,IAAI,EAAE;YACtB,YAAY,EAAE,oDAAoD;SACnE;KACF,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,SAAiB,EACjB,SAA0F,EAC1F,QAAwB;IAExB,OAAO,EAAE,CAAC,eAAe,CAAC,MAAM,CAAC;QAC/B,KAAK,EAAE,EAAE,EAAE,EAAE,SAAS,EAAE;QACxB,IAAI,EAAE,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU,EAAE,SAAS,KAAK,eAAe,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC,SAAS,EAAE;KAClG,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,SAAiB;IACxD,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,eAAe,CAAC,UAAU,CAAC;QAClD,KAAK,EAAE,EAAE,EAAE,EAAE,SAAS,EAAE;QACxB,MAAM,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE;KAC5C,CAAC,CAAC;IACH,OAAO,OAAO,CAAC;AACjB,CAAC;AAYD;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,KAAyB;IAC3D,OAAO,EAAE,CAAC,eAAe,CAAC,MAAM,CAAC;QAC/B,KAAK,EAAE,EAAE,UAAU,EAAE,KAAK,CAAC,UAAU,EAAE;QACvC,MAAM,EAAE;YACN,UAAU,EAAE,KAAK,CAAC,UAAU;YAC5B,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,QAAQ,EAAE,KAAK,CAAC,QAAQ,IAAI,KAAK;SAClC;QACD,MAAM,EAAE;YACN,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,OAAO,EAAE,KAAK,CAAC,OAAO;SACvB;KACF,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,SAAiB,EACjB,SAAiB,EACjB,IAAyB;IAEzB,IAAI,CAAC;QACH,OAAO,MAAM,EAAE,CAAC,iBAAiB,CAAC,MAAM,CAAC;YACvC,IAAI,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,IAAI,EAAE;SACrC,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,sDAAsD;QACtD,IAAI,GAAG,YAAY,KAAK,IAAI,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAC,EAAE,CAAC;YACtE,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,GAAG,CAAC;IACZ,CAAC;AACH,CAAC;AAED,+BAA+B;AAE/B,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,SAAiB,EAAE,OAAgB;IACvE,OAAO,EAAE,CAAC,eAAe,CAAC,MAAM,CAAC;QAC/B,KAAK,EAAE,EAAE,EAAE,EAAE,SAAS,EAAE;QACxB,IAAI,EAAE,EAAE,OAAO,EAAE;KAClB,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,SAAiB;IACtD,OAAO,EAAE,CAAC,aAAa,CAAC,QAAQ,CAAC;QAC/B,KAAK,EAAE,EAAE,mBAAmB,EAAE,SAAS,EAAE;KAC1C,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,SAAiB,EACjB,OAAe,EACf,SAAwB,EACxB,sBAA8B;IAE9B,OAAO,EAAE,CAAC,aAAa,CAAC,MAAM,CAAC;QAC7B,KAAK,EAAE;YACL,2BAA2B,EAAE;gBAC3B,mBAAmB,EAAE,SAAS;gBAC9B,OAAO;aACR;SACF;QACD,MAAM,EAAE;YACN,mBAAmB,EAAE,SAAS;YAC9B,OAAO;YACP,SAAS;YACT,sBAAsB;SACvB;QACD,MAAM,EAAE;YACN,SAAS;YACT,sBAAsB;SACvB;KACF,CAAC,CAAC;AACL,CAAC;AAED,wEAAwE;AAExE,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAC,SAAiB;IAC5D,OAAO,EAAE,CAAC,mBAAmB,CAAC,UAAU,CAAC;QACvC,KAAK,EAAE,EAAE,EAAE,EAAE,SAAS,EAAE;QACxB,OAAO,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE;KAC3B,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,SAAiB,EACjB,MAAmB,EACnB,KAA+C;IAE/C,OAAO,EAAE,CAAC,mBAAmB,CAAC,MAAM,CAAC;QACnC,KAAK,EAAE,EAAE,EAAE,EAAE,SAAS,EAAE;QACxB,IAAI,EAAE;YACJ,MAAM;YACN,UAAU,EAAE,KAAK,EAAE,UAAU,IAAI,SAAS;YAC1C,KAAK,EAAE,KAAK,EAAE,KAAK,IAAI,SAAS;SACjC;KACF,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAAC,SAAiB;IAChE,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,iBAAiB,CAAC,QAAQ,CAAC;QAChD,KAAK,EAAE,EAAE,SAAS,EAAE;QACpB,MAAM,EAAE,EAAE,OAAO,EAAE,EAAE,MAAM,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,EAAE,EAAE;KACtD,CAAC,CAAC;IACH,OAAO,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;AACpE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,+BAA+B;IACnD,MAAM,QAAQ,GAAG,MAAM,EAAE,CAAC,eAAe,CAAC,QAAQ,CAAC;QACjD,MAAM,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE;KACvC,CAAC,CAAC;IACH,MAAM,GAAG,GAAG,IAAI,GAAG,EAAkB,CAAC;IACtC,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1B,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;IAC3C,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,SAAiB;IACpD,OAAO,EAAE,CAAC,eAAe,CAAC,UAAU,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;AACrE,CAAC"} \ No newline at end of file diff --git a/worker/dist/fetch-listener.d.ts b/worker/dist/fetch-listener.d.ts new file mode 100644 index 0000000..462baf1 --- /dev/null +++ b/worker/dist/fetch-listener.d.ts @@ -0,0 +1,11 @@ +/** + * Start listening for pg_notify signals from the web app. + * + * Channels: + * - `channel_fetch` — payload = requestId → fetch channels for an account + * - `generate_invite` — payload = channelId → generate invite link for destination + * - `create_destination` — payload = JSON { requestId, title } → create supergroup via TDLib + * - `ingestion_trigger` — trigger an immediate ingestion cycle + */ +export declare function startFetchListener(): Promise; +export declare function stopFetchListener(): void; diff --git a/worker/dist/fetch-listener.js b/worker/dist/fetch-listener.js new file mode 100644 index 0000000..60dd305 --- /dev/null +++ b/worker/dist/fetch-listener.js @@ -0,0 +1,195 @@ +import { pool } from "./db/client.js"; +import { childLogger } from "./util/logger.js"; +import { withTdlibMutex } from "./util/mutex.js"; +import { processFetchRequest } from "./worker.js"; +import { generateInviteLink, createSupergroup } from "./tdlib/chats.js"; +import { createTdlibClient, closeTdlibClient } from "./tdlib/client.js"; +import { triggerImmediateCycle } from "./scheduler.js"; +import { getGlobalDestinationChannel, setGlobalSetting, getActiveAccounts, upsertChannel, ensureAccountChannelLink, } from "./db/queries.js"; +const log = childLogger("fetch-listener"); +let pgClient = null; +/** + * Start listening for pg_notify signals from the web app. + * + * Channels: + * - `channel_fetch` — payload = requestId → fetch channels for an account + * - `generate_invite` — payload = channelId → generate invite link for destination + * - `create_destination` — payload = JSON { requestId, title } → create supergroup via TDLib + * - `ingestion_trigger` — trigger an immediate ingestion cycle + */ +export async function startFetchListener() { + pgClient = await pool.connect(); + await pgClient.query("LISTEN channel_fetch"); + await pgClient.query("LISTEN generate_invite"); + await pgClient.query("LISTEN create_destination"); + await pgClient.query("LISTEN ingestion_trigger"); + pgClient.on("notification", (msg) => { + if (msg.channel === "channel_fetch" && msg.payload) { + handleChannelFetch(msg.payload); + } + else if (msg.channel === "generate_invite" && msg.payload) { + handleGenerateInvite(msg.payload); + } + else if (msg.channel === "create_destination" && msg.payload) { + handleCreateDestination(msg.payload); + } + else if (msg.channel === "ingestion_trigger") { + handleIngestionTrigger(); + } + }); + log.info("Fetch listener started (channel_fetch, generate_invite, create_destination, ingestion_trigger)"); +} +export function stopFetchListener() { + if (pgClient) { + pgClient.release(); + pgClient = null; + } + log.info("Fetch listener stopped"); +} +// ── Channel fetch handler ── +// Chain promises to ensure sequential execution +let fetchQueue = Promise.resolve(); +function handleChannelFetch(requestId) { + fetchQueue = fetchQueue.then(async () => { + try { + await withTdlibMutex("fetch-channels", () => processFetchRequest(requestId)); + } + catch (err) { + log.error({ err, requestId }, "Failed to process fetch request"); + } + }); +} +// ── Invite link generation handler ── +function handleGenerateInvite(channelId) { + fetchQueue = fetchQueue.then(async () => { + try { + await withTdlibMutex("generate-invite", async () => { + const destChannel = await getGlobalDestinationChannel(); + if (!destChannel || destChannel.id !== channelId) { + log.warn({ channelId }, "Destination channel mismatch, skipping invite generation"); + return; + } + // Use the first available authenticated account to generate the link + const accounts = await getActiveAccounts(); + if (accounts.length === 0) { + log.warn("No authenticated accounts to generate invite link"); + return; + } + const account = accounts[0]; + const client = await createTdlibClient({ id: account.id, phone: account.phone }); + try { + const link = await generateInviteLink(client, destChannel.telegramId); + await setGlobalSetting("destination_invite_link", link); + log.info({ link }, "Invite link generated and saved"); + } + finally { + await closeTdlibClient(client); + } + }); + } + catch (err) { + log.error({ err, channelId }, "Failed to generate invite link"); + } + }); +} +// ── Create destination supergroup handler ── +function handleCreateDestination(payload) { + fetchQueue = fetchQueue.then(async () => { + let requestId; + try { + const parsed = JSON.parse(payload); + requestId = parsed.requestId; + await withTdlibMutex("create-destination", async () => { + const { db } = await import("./db/client.js"); + // Mark the request as in-progress + await db.channelFetchRequest.update({ + where: { id: parsed.requestId }, + data: { status: "IN_PROGRESS" }, + }); + // Use the first available authenticated account + const accounts = await getActiveAccounts(); + if (accounts.length === 0) { + throw new Error("No authenticated accounts available to create the group"); + } + const account = accounts[0]; + const client = await createTdlibClient({ id: account.id, phone: account.phone }); + try { + // Create the supergroup via TDLib + const result = await createSupergroup(client, parsed.title); + log.info({ chatId: result.chatId.toString(), title: result.title }, "Supergroup created"); + // Upsert it as a DESTINATION channel in the DB (active by default) + const channel = await upsertChannel({ + telegramId: result.chatId, + title: result.title, + type: "DESTINATION", + isForum: false, + isActive: true, + }); + // Set as global destination + await setGlobalSetting("destination_channel_id", channel.id); + // Generate an invite link + const link = await generateInviteLink(client, result.chatId); + await setGlobalSetting("destination_invite_link", link); + log.info({ link }, "Invite link generated for new destination"); + // Link all authenticated accounts as WRITER + for (const acc of accounts) { + try { + await ensureAccountChannelLink(acc.id, channel.id, "WRITER"); + } + catch { + // Already linked + } + } + // Mark fetch request as completed with the channel info + await db.channelFetchRequest.update({ + where: { id: parsed.requestId }, + data: { + status: "COMPLETED", + resultJson: JSON.stringify({ + channelId: channel.id, + telegramId: result.chatId.toString(), + title: result.title, + inviteLink: link, + }), + }, + }); + log.info({ channelId: channel.id, telegramId: result.chatId.toString() }, "Destination channel created and configured"); + } + finally { + await closeTdlibClient(client); + } + }); + } + catch (err) { + log.error({ err, payload }, "Failed to create destination channel"); + if (requestId) { + try { + const { db } = await import("./db/client.js"); + await db.channelFetchRequest.update({ + where: { id: requestId }, + data: { + status: "FAILED", + error: err instanceof Error ? err.message : String(err), + }, + }); + } + catch { + // Best-effort + } + } + } + }); +} +// ── Ingestion trigger handler ── +function handleIngestionTrigger() { + fetchQueue = fetchQueue.then(async () => { + try { + log.info("Ingestion trigger received from UI"); + await triggerImmediateCycle(); + } + catch (err) { + log.error({ err }, "Failed to trigger immediate ingestion cycle"); + } + }); +} +//# sourceMappingURL=fetch-listener.js.map \ No newline at end of file diff --git a/worker/dist/fetch-listener.js.map b/worker/dist/fetch-listener.js.map new file mode 100644 index 0000000..cb513a9 --- /dev/null +++ b/worker/dist/fetch-listener.js.map @@ -0,0 +1 @@ +{"version":3,"file":"fetch-listener.js","sourceRoot":"","sources":["../src/fetch-listener.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAC;AACtC,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AACxE,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACxE,OAAO,EAAE,qBAAqB,EAAE,MAAM,gBAAgB,CAAC;AACvD,OAAO,EACL,2BAA2B,EAE3B,gBAAgB,EAChB,iBAAiB,EACjB,aAAa,EACb,wBAAwB,GACzB,MAAM,iBAAiB,CAAC;AAEzB,MAAM,GAAG,GAAG,WAAW,CAAC,gBAAgB,CAAC,CAAC;AAE1C,IAAI,QAAQ,GAAyB,IAAI,CAAC;AAE1C;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB;IACtC,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;IAChC,MAAM,QAAQ,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;IAC7C,MAAM,QAAQ,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC;IAC/C,MAAM,QAAQ,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAClD,MAAM,QAAQ,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;IAEjD,QAAQ,CAAC,EAAE,CAAC,cAAc,EAAE,CAAC,GAAG,EAAE,EAAE;QAClC,IAAI,GAAG,CAAC,OAAO,KAAK,eAAe,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;YACnD,kBAAkB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAClC,CAAC;aAAM,IAAI,GAAG,CAAC,OAAO,KAAK,iBAAiB,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;YAC5D,oBAAoB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACpC,CAAC;aAAM,IAAI,GAAG,CAAC,OAAO,KAAK,oBAAoB,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;YAC/D,uBAAuB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACvC,CAAC;aAAM,IAAI,GAAG,CAAC,OAAO,KAAK,mBAAmB,EAAE,CAAC;YAC/C,sBAAsB,EAAE,CAAC;QAC3B,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,GAAG,CAAC,IAAI,CAAC,gGAAgG,CAAC,CAAC;AAC7G,CAAC;AAED,MAAM,UAAU,iBAAiB;IAC/B,IAAI,QAAQ,EAAE,CAAC;QACb,QAAQ,CAAC,OAAO,EAAE,CAAC;QACnB,QAAQ,GAAG,IAAI,CAAC;IAClB,CAAC;IACD,GAAG,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;AACrC,CAAC;AAED,8BAA8B;AAE9B,gDAAgD;AAChD,IAAI,UAAU,GAAkB,OAAO,CAAC,OAAO,EAAE,CAAC;AAElD,SAAS,kBAAkB,CAAC,SAAiB;IAC3C,UAAU,GAAG,UAAU,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;QACtC,IAAI,CAAC;YACH,MAAM,cAAc,CAAC,gBAAgB,EAAE,GAAG,EAAE,CAC1C,mBAAmB,CAAC,SAAS,CAAC,CAC/B,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,EAAE,iCAAiC,CAAC,CAAC;QACnE,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AAED,uCAAuC;AAEvC,SAAS,oBAAoB,CAAC,SAAiB;IAC7C,UAAU,GAAG,UAAU,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;QACtC,IAAI,CAAC;YACH,MAAM,cAAc,CAAC,iBAAiB,EAAE,KAAK,IAAI,EAAE;gBACjD,MAAM,WAAW,GAAG,MAAM,2BAA2B,EAAE,CAAC;gBACxD,IAAI,CAAC,WAAW,IAAI,WAAW,CAAC,EAAE,KAAK,SAAS,EAAE,CAAC;oBACjD,GAAG,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,EAAE,0DAA0D,CAAC,CAAC;oBACpF,OAAO;gBACT,CAAC;gBAED,qEAAqE;gBACrE,MAAM,QAAQ,GAAG,MAAM,iBAAiB,EAAE,CAAC;gBAC3C,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAC1B,GAAG,CAAC,IAAI,CAAC,mDAAmD,CAAC,CAAC;oBAC9D,OAAO;gBACT,CAAC;gBAED,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;gBAC5B,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,EAAE,EAAE,EAAE,OAAO,CAAC,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;gBAEjF,IAAI,CAAC;oBACH,MAAM,IAAI,GAAG,MAAM,kBAAkB,CAAC,MAAM,EAAE,WAAW,CAAC,UAAU,CAAC,CAAC;oBACtE,MAAM,gBAAgB,CAAC,yBAAyB,EAAE,IAAI,CAAC,CAAC;oBACxD,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,EAAE,iCAAiC,CAAC,CAAC;gBACxD,CAAC;wBAAS,CAAC;oBACT,MAAM,gBAAgB,CAAC,MAAM,CAAC,CAAC;gBACjC,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,EAAE,gCAAgC,CAAC,CAAC;QAClE,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AAED,8CAA8C;AAE9C,SAAS,uBAAuB,CAAC,OAAe;IAC9C,UAAU,GAAG,UAAU,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;QACtC,IAAI,SAA6B,CAAC;QAClC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAyC,CAAC;YAC3E,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;YAE7B,MAAM,cAAc,CAAC,oBAAoB,EAAE,KAAK,IAAI,EAAE;gBACpD,MAAM,EAAE,EAAE,EAAE,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,CAAC;gBAE9C,kCAAkC;gBAClC,MAAM,EAAE,CAAC,mBAAmB,CAAC,MAAM,CAAC;oBAClC,KAAK,EAAE,EAAE,EAAE,EAAE,MAAM,CAAC,SAAS,EAAE;oBAC/B,IAAI,EAAE,EAAE,MAAM,EAAE,aAAa,EAAE;iBAChC,CAAC,CAAC;gBAEH,gDAAgD;gBAChD,MAAM,QAAQ,GAAG,MAAM,iBAAiB,EAAE,CAAC;gBAC3C,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAC1B,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;gBAC7E,CAAC;gBAED,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;gBAC5B,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,EAAE,EAAE,EAAE,OAAO,CAAC,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;gBAEjF,IAAI,CAAC;oBACH,kCAAkC;oBAClC,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;oBAC5D,GAAG,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,EAAE,oBAAoB,CAAC,CAAC;oBAE1F,mEAAmE;oBACnE,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC;wBAClC,UAAU,EAAE,MAAM,CAAC,MAAM;wBACzB,KAAK,EAAE,MAAM,CAAC,KAAK;wBACnB,IAAI,EAAE,aAAa;wBACnB,OAAO,EAAE,KAAK;wBACd,QAAQ,EAAE,IAAI;qBACf,CAAC,CAAC;oBAEH,4BAA4B;oBAC5B,MAAM,gBAAgB,CAAC,wBAAwB,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC;oBAE7D,0BAA0B;oBAC1B,MAAM,IAAI,GAAG,MAAM,kBAAkB,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;oBAC7D,MAAM,gBAAgB,CAAC,yBAAyB,EAAE,IAAI,CAAC,CAAC;oBACxD,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,EAAE,2CAA2C,CAAC,CAAC;oBAEhE,4CAA4C;oBAC5C,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;wBAC3B,IAAI,CAAC;4BACH,MAAM,wBAAwB,CAAC,GAAG,CAAC,EAAE,EAAE,OAAO,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;wBAC/D,CAAC;wBAAC,MAAM,CAAC;4BACP,iBAAiB;wBACnB,CAAC;oBACH,CAAC;oBAED,wDAAwD;oBACxD,MAAM,EAAE,CAAC,mBAAmB,CAAC,MAAM,CAAC;wBAClC,KAAK,EAAE,EAAE,EAAE,EAAE,MAAM,CAAC,SAAS,EAAE;wBAC/B,IAAI,EAAE;4BACJ,MAAM,EAAE,WAAW;4BACnB,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC;gCACzB,SAAS,EAAE,OAAO,CAAC,EAAE;gCACrB,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE;gCACpC,KAAK,EAAE,MAAM,CAAC,KAAK;gCACnB,UAAU,EAAE,IAAI;6BACjB,CAAC;yBACH;qBACF,CAAC,CAAC;oBAEH,GAAG,CAAC,IAAI,CACN,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,EAC/D,4CAA4C,CAC7C,CAAC;gBACJ,CAAC;wBAAS,CAAC;oBACT,MAAM,gBAAgB,CAAC,MAAM,CAAC,CAAC;gBACjC,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,sCAAsC,CAAC,CAAC;YACpE,IAAI,SAAS,EAAE,CAAC;gBACd,IAAI,CAAC;oBACH,MAAM,EAAE,EAAE,EAAE,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,CAAC;oBAC9C,MAAM,EAAE,CAAC,mBAAmB,CAAC,MAAM,CAAC;wBAClC,KAAK,EAAE,EAAE,EAAE,EAAE,SAAS,EAAE;wBACxB,IAAI,EAAE;4BACJ,MAAM,EAAE,QAAQ;4BAChB,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;yBACxD;qBACF,CAAC,CAAC;gBACL,CAAC;gBAAC,MAAM,CAAC;oBACP,cAAc;gBAChB,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AAED,kCAAkC;AAElC,SAAS,sBAAsB;IAC7B,UAAU,GAAG,UAAU,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;QACtC,IAAI,CAAC;YACH,GAAG,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;YAC/C,MAAM,qBAAqB,EAAE,CAAC;QAChC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,EAAE,6CAA6C,CAAC,CAAC;QACpE,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC"} \ No newline at end of file diff --git a/worker/dist/index.d.ts b/worker/dist/index.d.ts new file mode 100644 index 0000000..cb0ff5c --- /dev/null +++ b/worker/dist/index.d.ts @@ -0,0 +1 @@ +export {}; diff --git a/worker/dist/index.js b/worker/dist/index.js new file mode 100644 index 0000000..4b59786 --- /dev/null +++ b/worker/dist/index.js @@ -0,0 +1,50 @@ +import { mkdir } from "fs/promises"; +import { config } from "./util/config.js"; +import { logger } from "./util/logger.js"; +import { markStaleRunsAsFailed } from "./db/queries.js"; +import { cleanupTempDir } from "./worker.js"; +import { startScheduler, stopScheduler } from "./scheduler.js"; +import { startFetchListener, stopFetchListener } from "./fetch-listener.js"; +import { db, pool } from "./db/client.js"; +const log = logger.child({ module: "main" }); +async function main() { + log.info("DragonsStash Telegram Worker starting"); + log.info({ config: { ...config, databaseUrl: "***" } }, "Configuration loaded"); + if (!config.telegramApiId || !config.telegramApiHash) { + log.fatal("TELEGRAM_API_ID and TELEGRAM_API_HASH are both required"); + process.exit(1); + } + // Ensure temp directory exists + await mkdir(config.tempDir, { recursive: true }); + await mkdir(config.tdlibStateDir, { recursive: true }); + // Clean up stale state + await cleanupTempDir(); + await markStaleRunsAsFailed(); + // Start the fetch listener (pg_notify for on-demand channel fetching) + await startFetchListener(); + // Start the scheduler + await startScheduler(); +} +// Graceful shutdown +function shutdown(signal) { + log.info({ signal }, "Shutdown signal received"); + stopScheduler(); + stopFetchListener(); + // Close DB connections + Promise.all([db.$disconnect(), pool.end()]) + .then(() => { + log.info("Shutdown complete"); + process.exit(0); + }) + .catch((err) => { + log.error({ err }, "Error during shutdown"); + process.exit(1); + }); +} +process.on("SIGTERM", () => shutdown("SIGTERM")); +process.on("SIGINT", () => shutdown("SIGINT")); +main().catch((err) => { + log.fatal({ err }, "Worker failed to start"); + process.exit(1); +}); +//# sourceMappingURL=index.js.map \ No newline at end of file diff --git a/worker/dist/index.js.map b/worker/dist/index.js.map new file mode 100644 index 0000000..ceccbac --- /dev/null +++ b/worker/dist/index.js.map @@ -0,0 +1 @@ +{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,OAAO,EAAE,qBAAqB,EAAE,MAAM,iBAAiB,CAAC;AACxD,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/D,OAAO,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAC5E,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAC;AAE1C,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;AAE7C,KAAK,UAAU,IAAI;IACjB,GAAG,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;IAClD,GAAG,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,GAAG,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,EAAE,EAAE,sBAAsB,CAAC,CAAC;IAEhF,IAAI,CAAC,MAAM,CAAC,aAAa,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC;QACrD,GAAG,CAAC,KAAK,CAAC,yDAAyD,CAAC,CAAC;QACrE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,+BAA+B;IAC/B,MAAM,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACjD,MAAM,KAAK,CAAC,MAAM,CAAC,aAAa,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAEvD,uBAAuB;IACvB,MAAM,cAAc,EAAE,CAAC;IACvB,MAAM,qBAAqB,EAAE,CAAC;IAE9B,sEAAsE;IACtE,MAAM,kBAAkB,EAAE,CAAC;IAE3B,sBAAsB;IACtB,MAAM,cAAc,EAAE,CAAC;AACzB,CAAC;AAED,oBAAoB;AACpB,SAAS,QAAQ,CAAC,MAAc;IAC9B,GAAG,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,0BAA0B,CAAC,CAAC;IACjD,aAAa,EAAE,CAAC;IAChB,iBAAiB,EAAE,CAAC;IAEpB,uBAAuB;IACvB,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,WAAW,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;SACxC,IAAI,CAAC,GAAG,EAAE;QACT,GAAG,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;QAC9B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC;SACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;QACb,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,EAAE,uBAAuB,CAAC,CAAC;QAC5C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;AACP,CAAC;AAED,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;AACjD,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;AAE/C,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IACnB,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,EAAE,wBAAwB,CAAC,CAAC;IAC7C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"} \ No newline at end of file diff --git a/worker/dist/preview/match.d.ts b/worker/dist/preview/match.d.ts new file mode 100644 index 0000000..50201fd --- /dev/null +++ b/worker/dist/preview/match.d.ts @@ -0,0 +1,22 @@ +export interface TelegramPhoto { + id: bigint; + date: Date; + /** Caption text on the photo message (if any). */ + caption: string; + /** The smallest photo size available — used as thumbnail. */ + fileId: string; + fileSize: number; +} +export interface ArchiveRef { + baseName: string; + firstMessageId: bigint; + firstMessageDate: Date; +} +/** + * Try to match a photo message to an archive by: + * 1. Caption contains the archive baseName (without extension) + * 2. Photo was posted within ±10 messages (time-window: ±6 hours) + * + * Returns the best match (closest in time), or null. + */ +export declare function matchPreviewToArchive(photos: TelegramPhoto[], archives: ArchiveRef[]): Map; diff --git a/worker/dist/preview/match.js b/worker/dist/preview/match.js new file mode 100644 index 0000000..02dd716 --- /dev/null +++ b/worker/dist/preview/match.js @@ -0,0 +1,53 @@ +import { childLogger } from "../util/logger.js"; +const log = childLogger("preview-match"); +/** + * Try to match a photo message to an archive by: + * 1. Caption contains the archive baseName (without extension) + * 2. Photo was posted within ±10 messages (time-window: ±6 hours) + * + * Returns the best match (closest in time), or null. + */ +export function matchPreviewToArchive(photos, archives) { + const results = new Map(); + const TIME_WINDOW_MS = 6 * 60 * 60 * 1000; // 6 hours + for (const archive of archives) { + // Normalize the archive base name for matching + const normalizedBase = normalizeForMatch(archive.baseName); + if (!normalizedBase) + continue; + let bestMatch = null; + let bestTimeDiff = Infinity; + for (const photo of photos) { + const timeDiff = Math.abs(photo.date.getTime() - archive.firstMessageDate.getTime()); + // Must be within time window + if (timeDiff > TIME_WINDOW_MS) + continue; + // Check if the photo caption contains the archive base name + const normalizedCaption = normalizeForMatch(photo.caption); + if (!normalizedCaption) + continue; + const matches = normalizedCaption.includes(normalizedBase) || + normalizedBase.includes(normalizedCaption); + if (matches && timeDiff < bestTimeDiff) { + bestMatch = photo; + bestTimeDiff = timeDiff; + } + } + if (bestMatch) { + log.debug({ baseName: archive.baseName, photoId: bestMatch.id.toString() }, "Matched preview photo to archive"); + results.set(archive.baseName, bestMatch); + } + } + return results; +} +/** + * Strip extension, punctuation, and normalize for fuzzy matching. + */ +function normalizeForMatch(input) { + return input + .toLowerCase() + .replace(/\.[a-z0-9]{1,5}$/i, "") // strip extension + .replace(/[_\-.\s]+/g, " ") // normalize separators + .trim(); +} +//# sourceMappingURL=match.js.map \ No newline at end of file diff --git a/worker/dist/preview/match.js.map b/worker/dist/preview/match.js.map new file mode 100644 index 0000000..6e0d388 --- /dev/null +++ b/worker/dist/preview/match.js.map @@ -0,0 +1 @@ +{"version":3,"file":"match.js","sourceRoot":"","sources":["../../src/preview/match.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,MAAM,GAAG,GAAG,WAAW,CAAC,eAAe,CAAC,CAAC;AAkBzC;;;;;;GAMG;AACH,MAAM,UAAU,qBAAqB,CACnC,MAAuB,EACvB,QAAsB;IAEtB,MAAM,OAAO,GAAG,IAAI,GAAG,EAAyB,CAAC;IACjD,MAAM,cAAc,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,UAAU;IAErD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,+CAA+C;QAC/C,MAAM,cAAc,GAAG,iBAAiB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC3D,IAAI,CAAC,cAAc;YAAE,SAAS;QAE9B,IAAI,SAAS,GAAyB,IAAI,CAAC;QAC3C,IAAI,YAAY,GAAG,QAAQ,CAAC;QAE5B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CACvB,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,OAAO,CAAC,gBAAgB,CAAC,OAAO,EAAE,CAC1D,CAAC;YAEF,6BAA6B;YAC7B,IAAI,QAAQ,GAAG,cAAc;gBAAE,SAAS;YAExC,4DAA4D;YAC5D,MAAM,iBAAiB,GAAG,iBAAiB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAC3D,IAAI,CAAC,iBAAiB;gBAAE,SAAS;YAEjC,MAAM,OAAO,GACX,iBAAiB,CAAC,QAAQ,CAAC,cAAc,CAAC;gBAC1C,cAAc,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC;YAE7C,IAAI,OAAO,IAAI,QAAQ,GAAG,YAAY,EAAE,CAAC;gBACvC,SAAS,GAAG,KAAK,CAAC;gBAClB,YAAY,GAAG,QAAQ,CAAC;YAC1B,CAAC;QACH,CAAC;QAED,IAAI,SAAS,EAAE,CAAC;YACd,GAAG,CAAC,KAAK,CACP,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,OAAO,EAAE,SAAS,CAAC,EAAE,CAAC,QAAQ,EAAE,EAAE,EAChE,kCAAkC,CACnC,CAAC;YACF,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;QAC3C,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB,CAAC,KAAa;IACtC,OAAO,KAAK;SACT,WAAW,EAAE;SACb,OAAO,CAAC,mBAAmB,EAAE,EAAE,CAAC,CAAC,kBAAkB;SACnD,OAAO,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC,uBAAuB;SAClD,IAAI,EAAE,CAAC;AACZ,CAAC"} \ No newline at end of file diff --git a/worker/dist/scheduler.d.ts b/worker/dist/scheduler.d.ts new file mode 100644 index 0000000..1825681 --- /dev/null +++ b/worker/dist/scheduler.d.ts @@ -0,0 +1,13 @@ +/** + * Start the scheduler. Runs an immediate first cycle, then schedules subsequent ones. + */ +export declare function startScheduler(): Promise; +/** + * Trigger an immediate ingestion cycle (e.g. from the admin UI). + * If a cycle is already running, this is a no-op. + */ +export declare function triggerImmediateCycle(): Promise; +/** + * Stop the scheduler gracefully. + */ +export declare function stopScheduler(): void; diff --git a/worker/dist/scheduler.js b/worker/dist/scheduler.js new file mode 100644 index 0000000..58f5585 --- /dev/null +++ b/worker/dist/scheduler.js @@ -0,0 +1,121 @@ +import { config } from "./util/config.js"; +import { childLogger } from "./util/logger.js"; +import { withTdlibMutex } from "./util/mutex.js"; +import { getActiveAccounts, getPendingAccounts } from "./db/queries.js"; +import { runWorkerForAccount, authenticateAccount } from "./worker.js"; +const log = childLogger("scheduler"); +let running = false; +let timer = null; +let cycleCount = 0; +/** + * Maximum time for a single ingestion cycle (ms). + * After this, new accounts won't be started (in-progress work finishes). + * Default: 4 hours. Configurable via WORKER_CYCLE_TIMEOUT_MINUTES. + */ +const CYCLE_TIMEOUT_MS = (parseInt(process.env.WORKER_CYCLE_TIMEOUT_MINUTES ?? "240", 10)) * 60 * 1000; +/** + * Run one ingestion cycle: + * 1. Authenticate any PENDING accounts (triggers SMS code flow + auto-fetch channels) + * 2. Process all active AUTHENTICATED accounts for ingestion + * + * All TDLib operations are wrapped in the mutex to ensure only one client + * runs at a time (also shared with the fetch listener for on-demand requests). + * + * The cycle has a configurable timeout (WORKER_CYCLE_TIMEOUT_MINUTES, default 4h). + * Once the timeout elapses, no new accounts will be started but any in-progress + * account processing is allowed to finish its current archive set. + */ +async function runCycle() { + if (running) { + log.warn("Previous cycle still running, skipping"); + return; + } + running = true; + cycleCount++; + const cycleStart = Date.now(); + log.info({ cycle: cycleCount, timeoutMinutes: CYCLE_TIMEOUT_MS / 60_000 }, "Starting ingestion cycle"); + try { + // ── Phase 1: Authenticate pending accounts ── + const pendingAccounts = await getPendingAccounts(); + if (pendingAccounts.length > 0) { + log.info({ count: pendingAccounts.length }, "Found pending accounts, starting authentication"); + for (const account of pendingAccounts) { + if (Date.now() - cycleStart > CYCLE_TIMEOUT_MS) { + log.warn("Cycle timeout reached during authentication phase, stopping"); + break; + } + await withTdlibMutex(`auth:${account.phone}`, () => authenticateAccount(account)); + } + } + // ── Phase 2: Ingest for authenticated accounts ── + const accounts = await getActiveAccounts(); + if (accounts.length === 0) { + log.info("No active authenticated accounts, nothing to ingest"); + return; + } + log.info({ accountCount: accounts.length }, "Processing accounts"); + for (const account of accounts) { + if (Date.now() - cycleStart > CYCLE_TIMEOUT_MS) { + log.warn({ elapsed: Math.round((Date.now() - cycleStart) / 60_000), timeoutMinutes: CYCLE_TIMEOUT_MS / 60_000 }, "Cycle timeout reached, skipping remaining accounts"); + break; + } + await withTdlibMutex(`ingest:${account.phone}`, () => runWorkerForAccount(account)); + } + log.info({ elapsed: Math.round((Date.now() - cycleStart) / 1000) }, "Ingestion cycle complete"); + } + catch (err) { + log.error({ err }, "Ingestion cycle failed"); + } + finally { + running = false; + } +} +/** + * Schedule the next cycle with jitter. + */ +function scheduleNext() { + const intervalMs = config.workerIntervalMinutes * 60 * 1000; + const jitterMs = Math.random() * config.jitterMinutes * 60 * 1000; + const delay = intervalMs + jitterMs; + log.info({ nextRunInMinutes: Math.round(delay / 60000) }, "Next cycle scheduled"); + timer = setTimeout(async () => { + await runCycle(); + scheduleNext(); + }, delay); +} +/** + * Start the scheduler. Runs an immediate first cycle, then schedules subsequent ones. + */ +export async function startScheduler() { + log.info({ + intervalMinutes: config.workerIntervalMinutes, + jitterMinutes: config.jitterMinutes, + }, "Scheduler starting"); + // Run immediately on start + await runCycle(); + // Then schedule recurring cycles + scheduleNext(); +} +/** + * Trigger an immediate ingestion cycle (e.g. from the admin UI). + * If a cycle is already running, this is a no-op. + */ +export async function triggerImmediateCycle() { + if (running) { + log.info("Cycle already running, ignoring trigger"); + return; + } + log.info("Immediate cycle triggered via UI"); + await runCycle(); +} +/** + * Stop the scheduler gracefully. + */ +export function stopScheduler() { + if (timer) { + clearTimeout(timer); + timer = null; + } + log.info("Scheduler stopped"); +} +//# sourceMappingURL=scheduler.js.map \ No newline at end of file diff --git a/worker/dist/scheduler.js.map b/worker/dist/scheduler.js.map new file mode 100644 index 0000000..c43fa77 --- /dev/null +++ b/worker/dist/scheduler.js.map @@ -0,0 +1 @@ +{"version":3,"file":"scheduler.js","sourceRoot":"","sources":["../src/scheduler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AACxE,OAAO,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAEvE,MAAM,GAAG,GAAG,WAAW,CAAC,WAAW,CAAC,CAAC;AAErC,IAAI,OAAO,GAAG,KAAK,CAAC;AACpB,IAAI,KAAK,GAAyC,IAAI,CAAC;AACvD,IAAI,UAAU,GAAG,CAAC,CAAC;AAEnB;;;;GAIG;AACH,MAAM,gBAAgB,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,4BAA4B,IAAI,KAAK,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;AAEvG;;;;;;;;;;;GAWG;AACH,KAAK,UAAU,QAAQ;IACrB,IAAI,OAAO,EAAE,CAAC;QACZ,GAAG,CAAC,IAAI,CAAC,wCAAwC,CAAC,CAAC;QACnD,OAAO;IACT,CAAC;IAED,OAAO,GAAG,IAAI,CAAC;IACf,UAAU,EAAE,CAAC;IACb,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC9B,GAAG,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,cAAc,EAAE,gBAAgB,GAAG,MAAM,EAAE,EAAE,0BAA0B,CAAC,CAAC;IAEvG,IAAI,CAAC;QACH,+CAA+C;QAC/C,MAAM,eAAe,GAAG,MAAM,kBAAkB,EAAE,CAAC;QACnD,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/B,GAAG,CAAC,IAAI,CACN,EAAE,KAAK,EAAE,eAAe,CAAC,MAAM,EAAE,EACjC,iDAAiD,CAClD,CAAC;YACF,KAAK,MAAM,OAAO,IAAI,eAAe,EAAE,CAAC;gBACtC,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,GAAG,gBAAgB,EAAE,CAAC;oBAC/C,GAAG,CAAC,IAAI,CAAC,6DAA6D,CAAC,CAAC;oBACxE,MAAM;gBACR,CAAC;gBACD,MAAM,cAAc,CAAC,QAAQ,OAAO,CAAC,KAAK,EAAE,EAAE,GAAG,EAAE,CACjD,mBAAmB,CAAC,OAAO,CAAC,CAC7B,CAAC;YACJ,CAAC;QACH,CAAC;QAED,mDAAmD;QACnD,MAAM,QAAQ,GAAG,MAAM,iBAAiB,EAAE,CAAC;QAE3C,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,GAAG,CAAC,IAAI,CAAC,qDAAqD,CAAC,CAAC;YAChE,OAAO;QACT,CAAC;QAED,GAAG,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,QAAQ,CAAC,MAAM,EAAE,EAAE,qBAAqB,CAAC,CAAC;QAEnE,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,GAAG,gBAAgB,EAAE,CAAC;gBAC/C,GAAG,CAAC,IAAI,CACN,EAAE,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC,GAAG,MAAM,CAAC,EAAE,cAAc,EAAE,gBAAgB,GAAG,MAAM,EAAE,EACtG,oDAAoD,CACrD,CAAC;gBACF,MAAM;YACR,CAAC;YACD,MAAM,cAAc,CAAC,UAAU,OAAO,CAAC,KAAK,EAAE,EAAE,GAAG,EAAE,CACnD,mBAAmB,CAAC,OAAO,CAAC,CAC7B,CAAC;QACJ,CAAC;QAED,GAAG,CAAC,IAAI,CACN,EAAE,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC,GAAG,IAAI,CAAC,EAAE,EACzD,0BAA0B,CAC3B,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,EAAE,wBAAwB,CAAC,CAAC;IAC/C,CAAC;YAAS,CAAC;QACT,OAAO,GAAG,KAAK,CAAC;IAClB,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,YAAY;IACnB,MAAM,UAAU,GAAG,MAAM,CAAC,qBAAqB,GAAG,EAAE,GAAG,IAAI,CAAC;IAC5D,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC,aAAa,GAAG,EAAE,GAAG,IAAI,CAAC;IAClE,MAAM,KAAK,GAAG,UAAU,GAAG,QAAQ,CAAC;IAEpC,GAAG,CAAC,IAAI,CACN,EAAE,gBAAgB,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,EAAE,EAC/C,sBAAsB,CACvB,CAAC;IAEF,KAAK,GAAG,UAAU,CAAC,KAAK,IAAI,EAAE;QAC5B,MAAM,QAAQ,EAAE,CAAC;QACjB,YAAY,EAAE,CAAC;IACjB,CAAC,EAAE,KAAK,CAAC,CAAC;AACZ,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc;IAClC,GAAG,CAAC,IAAI,CACN;QACE,eAAe,EAAE,MAAM,CAAC,qBAAqB;QAC7C,aAAa,EAAE,MAAM,CAAC,aAAa;KACpC,EACD,oBAAoB,CACrB,CAAC;IAEF,2BAA2B;IAC3B,MAAM,QAAQ,EAAE,CAAC;IAEjB,iCAAiC;IACjC,YAAY,EAAE,CAAC;AACjB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB;IACzC,IAAI,OAAO,EAAE,CAAC;QACZ,GAAG,CAAC,IAAI,CAAC,yCAAyC,CAAC,CAAC;QACpD,OAAO;IACT,CAAC;IACD,GAAG,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;IAC7C,MAAM,QAAQ,EAAE,CAAC;AACnB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,aAAa;IAC3B,IAAI,KAAK,EAAE,CAAC;QACV,YAAY,CAAC,KAAK,CAAC,CAAC;QACpB,KAAK,GAAG,IAAI,CAAC;IACf,CAAC;IACD,GAAG,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;AAChC,CAAC"} \ No newline at end of file diff --git a/worker/dist/tdlib/chats.d.ts b/worker/dist/tdlib/chats.d.ts new file mode 100644 index 0000000..cf9b4e6 --- /dev/null +++ b/worker/dist/tdlib/chats.d.ts @@ -0,0 +1,31 @@ +import type { Client } from "tdl"; +export interface TelegramChatInfo { + chatId: bigint; + title: string; + type: "channel" | "supergroup" | "group" | "private" | "other"; + isForum: boolean; + memberCount?: number; +} +/** + * Fetch all chats the account is a member of. + * Uses TDLib's getChats to load the chat list, then getChat for details. + * Filters to channels and supergroups only (groups/privates are not useful for ingestion). + */ +export declare function getAccountChats(client: Client): Promise; +/** + * Generate an invite link for a chat. The account must be an admin or have + * invite link permissions. + */ +export declare function generateInviteLink(client: Client, chatId: bigint): Promise; +/** + * Create a new supergroup (private group) via TDLib. + * Returns the chat ID and title. + */ +export declare function createSupergroup(client: Client, title: string): Promise<{ + chatId: bigint; + title: string; +}>; +/** + * Join a chat using an invite link. + */ +export declare function joinChatByInviteLink(client: Client, inviteLink: string): Promise; diff --git a/worker/dist/tdlib/chats.js b/worker/dist/tdlib/chats.js new file mode 100644 index 0000000..4f8a305 --- /dev/null +++ b/worker/dist/tdlib/chats.js @@ -0,0 +1,124 @@ +import { childLogger } from "../util/logger.js"; +import { config } from "../util/config.js"; +const log = childLogger("chats"); +/** + * Fetch all chats the account is a member of. + * Uses TDLib's getChats to load the chat list, then getChat for details. + * Filters to channels and supergroups only (groups/privates are not useful for ingestion). + */ +export async function getAccountChats(client) { + const chats = []; + // Load main chat list — TDLib loads in batches + let offsetOrder = "9223372036854775807"; // max int64 as string + let offsetChatId = 0; + let hasMore = true; + while (hasMore) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const result = (await client.invoke({ + _: "getChats", + chat_list: { _: "chatListMain" }, + limit: 100, + })); + if (!result.chat_ids || result.chat_ids.length === 0) { + break; + } + for (const chatId of result.chat_ids) { + try { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const chat = (await client.invoke({ + _: "getChat", + chat_id: chatId, + })); + const chatType = chat.type?._; + let type = "other"; + let isForum = false; + if (chatType === "chatTypeSupergroup") { + // Get supergroup details to check if it's a channel or group + try { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const sg = (await client.invoke({ + _: "getSupergroup", + supergroup_id: chat.type.supergroup_id, + })); + type = sg.is_channel ? "channel" : "supergroup"; + isForum = sg.is_forum ?? false; + } + catch { + type = "supergroup"; + } + } + else if (chatType === "chatTypeBasicGroup") { + type = "group"; + } + else if (chatType === "chatTypePrivate" || chatType === "chatTypeSecret") { + type = "private"; + } + // Only include channels and supergroups + if (type === "channel" || type === "supergroup") { + chats.push({ + chatId: BigInt(chatId), + title: chat.title ?? `Chat ${chatId}`, + type, + isForum, + }); + } + } + catch (err) { + log.warn({ chatId, err }, "Failed to get chat details, skipping"); + } + } + // getChats with chatListMain returns all chats at once in newer TDLib versions + // So we break after the first batch + hasMore = false; + await sleep(config.apiDelayMs); + } + log.info({ total: chats.length }, "Fetched channels/supergroups from Telegram"); + return chats; +} +/** + * Generate an invite link for a chat. The account must be an admin or have + * invite link permissions. + */ +export async function generateInviteLink(client, chatId) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const result = (await client.invoke({ + _: "createChatInviteLink", + chat_id: Number(chatId), + name: "DragonsStash Auto-Join", + creates_join_request: false, + })); + const link = result.invite_link; + log.info({ chatId: chatId.toString(), link }, "Generated invite link"); + return link; +} +/** + * Create a new supergroup (private group) via TDLib. + * Returns the chat ID and title. + */ +export async function createSupergroup(client, title) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const result = (await client.invoke({ + _: "createNewSupergroupChat", + title, + is_forum: false, + is_channel: false, + description: "DragonsStash archive destination — all accounts write here", + })); + const chatId = BigInt(result.id); + log.info({ chatId: chatId.toString(), title }, "Created new supergroup"); + return { chatId, title: result.title ?? title }; +} +/** + * Join a chat using an invite link. + */ +export async function joinChatByInviteLink(client, inviteLink) { + await client.invoke({ + _: "joinChatByInviteLink", + invite_link: inviteLink, + }); + log.info({ inviteLink }, "Joined chat by invite link"); +} +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} +//# sourceMappingURL=chats.js.map \ No newline at end of file diff --git a/worker/dist/tdlib/chats.js.map b/worker/dist/tdlib/chats.js.map new file mode 100644 index 0000000..a5b7c30 --- /dev/null +++ b/worker/dist/tdlib/chats.js.map @@ -0,0 +1 @@ +{"version":3,"file":"chats.js","sourceRoot":"","sources":["../../src/tdlib/chats.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,MAAM,GAAG,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;AAUjC;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,MAAc;IAEd,MAAM,KAAK,GAAuB,EAAE,CAAC;IAErC,+CAA+C;IAC/C,IAAI,WAAW,GAAG,qBAAqB,CAAC,CAAC,sBAAsB;IAC/D,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,OAAO,GAAG,IAAI,CAAC;IAEnB,OAAO,OAAO,EAAE,CAAC;QACf,8DAA8D;QAC9D,MAAM,MAAM,GAAG,CAAC,MAAM,MAAM,CAAC,MAAM,CAAC;YAClC,CAAC,EAAE,UAAU;YACb,SAAS,EAAE,EAAE,CAAC,EAAE,cAAc,EAAE;YAChC,KAAK,EAAE,GAAG;SACX,CAAC,CAA2B,CAAC;QAE9B,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrD,MAAM;QACR,CAAC;QAED,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YACrC,IAAI,CAAC;gBACH,8DAA8D;gBAC9D,MAAM,IAAI,GAAG,CAAC,MAAM,MAAM,CAAC,MAAM,CAAC;oBAChC,CAAC,EAAE,SAAS;oBACZ,OAAO,EAAE,MAAM;iBAChB,CAAC,CAAQ,CAAC;gBAEX,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC9B,IAAI,IAAI,GAA6B,OAAO,CAAC;gBAC7C,IAAI,OAAO,GAAG,KAAK,CAAC;gBAEpB,IAAI,QAAQ,KAAK,oBAAoB,EAAE,CAAC;oBACtC,6DAA6D;oBAC7D,IAAI,CAAC;wBACH,8DAA8D;wBAC9D,MAAM,EAAE,GAAG,CAAC,MAAM,MAAM,CAAC,MAAM,CAAC;4BAC9B,CAAC,EAAE,eAAe;4BAClB,aAAa,EAAE,IAAI,CAAC,IAAI,CAAC,aAAa;yBACvC,CAAC,CAAQ,CAAC;wBAEX,IAAI,GAAG,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC;wBAChD,OAAO,GAAG,EAAE,CAAC,QAAQ,IAAI,KAAK,CAAC;oBACjC,CAAC;oBAAC,MAAM,CAAC;wBACP,IAAI,GAAG,YAAY,CAAC;oBACtB,CAAC;gBACH,CAAC;qBAAM,IAAI,QAAQ,KAAK,oBAAoB,EAAE,CAAC;oBAC7C,IAAI,GAAG,OAAO,CAAC;gBACjB,CAAC;qBAAM,IAAI,QAAQ,KAAK,iBAAiB,IAAI,QAAQ,KAAK,gBAAgB,EAAE,CAAC;oBAC3E,IAAI,GAAG,SAAS,CAAC;gBACnB,CAAC;gBAED,wCAAwC;gBACxC,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,YAAY,EAAE,CAAC;oBAChD,KAAK,CAAC,IAAI,CAAC;wBACT,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC;wBACtB,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,QAAQ,MAAM,EAAE;wBACrC,IAAI;wBACJ,OAAO;qBACR,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,GAAG,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,sCAAsC,CAAC,CAAC;YACpE,CAAC;QACH,CAAC;QAED,+EAA+E;QAC/E,oCAAoC;QACpC,OAAO,GAAG,KAAK,CAAC;QAEhB,MAAM,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IACjC,CAAC;IAED,GAAG,CAAC,IAAI,CACN,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,EACvB,4CAA4C,CAC7C,CAAC;IAEF,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,MAAc,EACd,MAAc;IAEd,8DAA8D;IAC9D,MAAM,MAAM,GAAG,CAAC,MAAM,MAAM,CAAC,MAAM,CAAC;QAClC,CAAC,EAAE,sBAAsB;QACzB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC;QACvB,IAAI,EAAE,wBAAwB;QAC9B,oBAAoB,EAAE,KAAK;KAC5B,CAAC,CAAQ,CAAC;IAEX,MAAM,IAAI,GAAG,MAAM,CAAC,WAAqB,CAAC;IAC1C,GAAG,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,EAAE,uBAAuB,CAAC,CAAC;IACvE,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,MAAc,EACd,KAAa;IAEb,8DAA8D;IAC9D,MAAM,MAAM,GAAG,CAAC,MAAM,MAAM,CAAC,MAAM,CAAC;QAClC,CAAC,EAAE,yBAAyB;QAC5B,KAAK;QACL,QAAQ,EAAE,KAAK;QACf,UAAU,EAAE,KAAK;QACjB,WAAW,EAAE,4DAA4D;KAC1E,CAAC,CAAQ,CAAC;IAEX,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACjC,GAAG,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,EAAE,wBAAwB,CAAC,CAAC;IACzE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,KAAK,EAAE,CAAC;AAClD,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,MAAc,EACd,UAAkB;IAElB,MAAM,MAAM,CAAC,MAAM,CAAC;QAClB,CAAC,EAAE,sBAAsB;QACzB,WAAW,EAAE,UAAU;KACxB,CAAC,CAAC;IACH,GAAG,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,EAAE,4BAA4B,CAAC,CAAC;AACzD,CAAC;AAED,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC"} \ No newline at end of file diff --git a/worker/dist/tdlib/client.d.ts b/worker/dist/tdlib/client.d.ts new file mode 100644 index 0000000..477bd64 --- /dev/null +++ b/worker/dist/tdlib/client.d.ts @@ -0,0 +1,18 @@ +import { type Client } from "tdl"; +interface AccountConfig { + id: string; + phone: string; +} +/** + * Create and authenticate a TDLib client for a Telegram account. + * Authentication flow communicates with the admin UI via the database: + * - Worker sets authState to AWAITING_CODE when TDLib asks for phone code + * - Admin enters the code via UI, which writes it to authCode field + * - Worker polls DB for the code and feeds it to TDLib + */ +export declare function createTdlibClient(account: AccountConfig): Promise; +/** + * Close a TDLib client gracefully. + */ +export declare function closeTdlibClient(client: Client): Promise; +export {}; diff --git a/worker/dist/tdlib/client.js b/worker/dist/tdlib/client.js new file mode 100644 index 0000000..9d64701 --- /dev/null +++ b/worker/dist/tdlib/client.js @@ -0,0 +1,96 @@ +import tdl, { createClient } from "tdl"; +import { getTdjson } from "prebuilt-tdlib"; +import path from "path"; +import { config } from "../util/config.js"; +import { childLogger } from "../util/logger.js"; +import { updateAccountAuthState, getAccountAuthCode, } from "../db/queries.js"; +const log = childLogger("tdlib-client"); +// Configure tdl to use the prebuilt tdjson shared library +tdl.configure({ tdjson: getTdjson() }); +/** + * Create and authenticate a TDLib client for a Telegram account. + * Authentication flow communicates with the admin UI via the database: + * - Worker sets authState to AWAITING_CODE when TDLib asks for phone code + * - Admin enters the code via UI, which writes it to authCode field + * - Worker polls DB for the code and feeds it to TDLib + */ +export async function createTdlibClient(account) { + const dbPath = path.join(config.tdlibStateDir, account.id); + const client = createClient({ + apiId: config.telegramApiId, + apiHash: config.telegramApiHash, + databaseDirectory: dbPath, + filesDirectory: path.join(dbPath, "files"), + }); + client.on("error", (err) => { + log.error({ err, accountId: account.id }, "TDLib client error"); + }); + try { + await client.login(() => ({ + getPhoneNumber: async () => { + log.info({ accountId: account.id }, "TDLib requesting phone number"); + return account.phone; + }, + getAuthCode: async () => { + log.info({ accountId: account.id }, "TDLib requesting auth code"); + await updateAccountAuthState(account.id, "AWAITING_CODE"); + // Poll database for the code entered via admin UI + const code = await pollForAuthCode(account.id); + if (!code) { + throw new Error("Auth code not provided within timeout"); + } + // Clear the code after reading + await updateAccountAuthState(account.id, "AUTHENTICATED", null); + return code; + }, + getPassword: async () => { + log.info({ accountId: account.id }, "TDLib requesting 2FA password"); + await updateAccountAuthState(account.id, "AWAITING_PASSWORD"); + // Poll database for the password entered via admin UI + const code = await pollForAuthCode(account.id); + if (!code) { + throw new Error("2FA password not provided within timeout"); + } + await updateAccountAuthState(account.id, "AUTHENTICATED", null); + return code; + }, + })); + await updateAccountAuthState(account.id, "AUTHENTICATED"); + log.info({ accountId: account.id }, "TDLib client authenticated"); + return client; + } + catch (err) { + log.error({ err, accountId: account.id }, "TDLib authentication failed"); + await updateAccountAuthState(account.id, "EXPIRED"); + throw err; + } +} +/** + * Poll the database every 5 seconds for an auth code, up to 5 minutes. + */ +async function pollForAuthCode(accountId, timeoutMs = 300_000) { + const start = Date.now(); + while (Date.now() - start < timeoutMs) { + const result = await getAccountAuthCode(accountId); + if (result?.authCode) { + return result.authCode; + } + await sleep(5000); + } + return null; +} +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} +/** + * Close a TDLib client gracefully. + */ +export async function closeTdlibClient(client) { + try { + await client.close(); + } + catch (err) { + log.warn({ err }, "Error closing TDLib client"); + } +} +//# sourceMappingURL=client.js.map \ No newline at end of file diff --git a/worker/dist/tdlib/client.js.map b/worker/dist/tdlib/client.js.map new file mode 100644 index 0000000..3b185cb --- /dev/null +++ b/worker/dist/tdlib/client.js.map @@ -0,0 +1 @@ +{"version":3,"file":"client.js","sourceRoot":"","sources":["../../src/tdlib/client.ts"],"names":[],"mappings":"AAAA,OAAO,GAAG,EAAE,EAAE,YAAY,EAAe,MAAM,KAAK,CAAC;AACrD,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EACL,sBAAsB,EACtB,kBAAkB,GACnB,MAAM,kBAAkB,CAAC;AAE1B,MAAM,GAAG,GAAG,WAAW,CAAC,cAAc,CAAC,CAAC;AAExC,0DAA0D;AAC1D,GAAG,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;AAOvC;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,OAAsB;IAEtB,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC;IAE3D,MAAM,MAAM,GAAG,YAAY,CAAC;QAC1B,KAAK,EAAE,MAAM,CAAC,aAAa;QAC3B,OAAO,EAAE,MAAM,CAAC,eAAe;QAC/B,iBAAiB,EAAE,MAAM;QACzB,cAAc,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC;KAC3C,CAAC,CAAC;IAEH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;QACzB,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,EAAE,oBAAoB,CAAC,CAAC;IAClE,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC;YACxB,cAAc,EAAE,KAAK,IAAI,EAAE;gBACzB,GAAG,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,EAAE,+BAA+B,CAAC,CAAC;gBACrE,OAAO,OAAO,CAAC,KAAK,CAAC;YACvB,CAAC;YACD,WAAW,EAAE,KAAK,IAAI,EAAE;gBACtB,GAAG,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,EAAE,4BAA4B,CAAC,CAAC;gBAClE,MAAM,sBAAsB,CAAC,OAAO,CAAC,EAAE,EAAE,eAAe,CAAC,CAAC;gBAE1D,kDAAkD;gBAClD,MAAM,IAAI,GAAG,MAAM,eAAe,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;gBAC/C,IAAI,CAAC,IAAI,EAAE,CAAC;oBACV,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;gBAC3D,CAAC;gBAED,+BAA+B;gBAC/B,MAAM,sBAAsB,CAAC,OAAO,CAAC,EAAE,EAAE,eAAe,EAAE,IAAI,CAAC,CAAC;gBAChE,OAAO,IAAI,CAAC;YACd,CAAC;YACD,WAAW,EAAE,KAAK,IAAI,EAAE;gBACtB,GAAG,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,EAAE,+BAA+B,CAAC,CAAC;gBACrE,MAAM,sBAAsB,CAAC,OAAO,CAAC,EAAE,EAAE,mBAAmB,CAAC,CAAC;gBAE9D,sDAAsD;gBACtD,MAAM,IAAI,GAAG,MAAM,eAAe,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;gBAC/C,IAAI,CAAC,IAAI,EAAE,CAAC;oBACV,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;gBAC9D,CAAC;gBAED,MAAM,sBAAsB,CAAC,OAAO,CAAC,EAAE,EAAE,eAAe,EAAE,IAAI,CAAC,CAAC;gBAChE,OAAO,IAAI,CAAC;YACd,CAAC;SACF,CAAC,CAAC,CAAC;QAEJ,MAAM,sBAAsB,CAAC,OAAO,CAAC,EAAE,EAAE,eAAe,CAAC,CAAC;QAC1D,GAAG,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,EAAE,4BAA4B,CAAC,CAAC;QAClE,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,EAAE,6BAA6B,CAAC,CAAC;QACzE,MAAM,sBAAsB,CAAC,OAAO,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;QACpD,MAAM,GAAG,CAAC;IACZ,CAAC;AACH,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,eAAe,CAC5B,SAAiB,EACjB,SAAS,GAAG,OAAO;IAEnB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,GAAG,SAAS,EAAE,CAAC;QACtC,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,SAAS,CAAC,CAAC;QACnD,IAAI,MAAM,EAAE,QAAQ,EAAE,CAAC;YACrB,OAAO,MAAM,CAAC,QAAQ,CAAC;QACzB,CAAC;QACD,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,MAAc;IACnD,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,EAAE,4BAA4B,CAAC,CAAC;IAClD,CAAC;AACH,CAAC"} \ No newline at end of file diff --git a/worker/dist/tdlib/download.d.ts b/worker/dist/tdlib/download.d.ts new file mode 100644 index 0000000..4813078 --- /dev/null +++ b/worker/dist/tdlib/download.d.ts @@ -0,0 +1,67 @@ +import type { Client } from "tdl"; +import type { TelegramMessage } from "../archive/multipart.js"; +import type { TelegramPhoto } from "../preview/match.js"; +/** Maximum number of pages to scan per channel/topic to prevent infinite loops */ +export declare const MAX_SCAN_PAGES = 5000; +/** Timeout for a single TDLib API call (ms) */ +export declare const INVOKE_TIMEOUT_MS = 120000; +export interface ChannelScanResult { + archives: TelegramMessage[]; + photos: TelegramPhoto[]; + totalScanned: number; +} +export type ScanProgressCallback = (messagesScanned: number) => void; +/** + * Invoke a TDLib method with a timeout to prevent indefinite hangs. + * If TDLib does not respond within the timeout, the promise rejects. + */ +export declare function invokeWithTimeout(client: Client, request: Record, timeoutMs?: number): Promise; +/** + * Fetch messages from a channel, stopping once we've scanned past the + * last-processed boundary (with one page of lookback for multipart safety). + * Collects both archive attachments AND photo messages (for preview matching). + * Returns messages in chronological order (oldest first). + * + * When `lastProcessedMessageId` is null (first run), scans everything. + * The worker applies a post-grouping filter to skip fully-processed sets, + * and keeps `packageExistsBySourceMessage` as a safety net. + * + * Safety features: + * - Max page limit to prevent infinite loops + * - Stuck detection: breaks if from_message_id stops advancing + * - Timeout on each TDLib API call + */ +export declare function getChannelMessages(client: Client, chatId: bigint, lastProcessedMessageId?: bigint | null, limit?: number, onProgress?: ScanProgressCallback): Promise; +/** + * Download a photo thumbnail from Telegram and return its raw bytes. + * Uses synchronous download (photos are small, typically < 100KB). + * Returns null if download fails (non-critical). + */ +export declare function downloadPhotoThumbnail(client: Client, fileId: string): Promise; +export interface DownloadProgress { + fileId: string; + fileName: string; + downloadedBytes: number; + totalBytes: number; + percent: number; + isComplete: boolean; +} +export type ProgressCallback = (progress: DownloadProgress) => void; +/** + * Download a file from Telegram to a local path with progress tracking + * and integrity verification. + * + * Progress flow: + * 1. Starts async download via TDLib + * 2. Listens for `updateFile` events to track download progress + * 3. Logs progress at every 10% increment + * 4. Once complete, verifies the local file size matches the expected size + * 5. Moves the file from TDLib's cache to the destination path + * + * Verification: + * - Compares actual file size on disk to the expected size from Telegram + * - Throws on mismatch (partial/corrupt download) + * - Throws on timeout (configurable, scales with file size) + * - Throws if download stops without completing (network error, etc.) + */ +export declare function downloadFile(client: Client, fileId: string, destPath: string, expectedSize: bigint, fileName: string, onProgress?: ProgressCallback): Promise; diff --git a/worker/dist/tdlib/download.js b/worker/dist/tdlib/download.js new file mode 100644 index 0000000..0b56c0f --- /dev/null +++ b/worker/dist/tdlib/download.js @@ -0,0 +1,307 @@ +import { readFile, rename, copyFile, unlink, stat } from "fs/promises"; +import { config } from "../util/config.js"; +import { childLogger } from "../util/logger.js"; +import { isArchiveAttachment } from "../archive/detect.js"; +const log = childLogger("download"); +/** Maximum number of pages to scan per channel/topic to prevent infinite loops */ +export const MAX_SCAN_PAGES = 5000; +/** Timeout for a single TDLib API call (ms) */ +export const INVOKE_TIMEOUT_MS = 120_000; // 2 minutes +/** + * Invoke a TDLib method with a timeout to prevent indefinite hangs. + * If TDLib does not respond within the timeout, the promise rejects. + */ +export async function invokeWithTimeout(client, +// eslint-disable-next-line @typescript-eslint/no-explicit-any +request, timeoutMs = INVOKE_TIMEOUT_MS) { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + reject(new Error(`TDLib invoke timed out after ${timeoutMs}ms for ${request._}`)); + }, timeoutMs); + client.invoke(request) + .then((result) => { + clearTimeout(timer); + resolve(result); + }) + .catch((err) => { + clearTimeout(timer); + reject(err); + }); + }); +} +/** + * Fetch messages from a channel, stopping once we've scanned past the + * last-processed boundary (with one page of lookback for multipart safety). + * Collects both archive attachments AND photo messages (for preview matching). + * Returns messages in chronological order (oldest first). + * + * When `lastProcessedMessageId` is null (first run), scans everything. + * The worker applies a post-grouping filter to skip fully-processed sets, + * and keeps `packageExistsBySourceMessage` as a safety net. + * + * Safety features: + * - Max page limit to prevent infinite loops + * - Stuck detection: breaks if from_message_id stops advancing + * - Timeout on each TDLib API call + */ +export async function getChannelMessages(client, chatId, lastProcessedMessageId, limit = 100, onProgress) { + const archives = []; + const photos = []; + const boundary = lastProcessedMessageId ? Number(lastProcessedMessageId) : null; + let currentFromId = 0; + let totalScanned = 0; + let pageCount = 0; + // eslint-disable-next-line no-constant-condition + while (true) { + if (pageCount >= MAX_SCAN_PAGES) { + log.warn({ chatId: chatId.toString(), pageCount, totalScanned }, "Hit max page limit for channel scan, stopping"); + break; + } + pageCount++; + const previousFromId = currentFromId; + const result = await invokeWithTimeout(client, { + _: "getChatHistory", + chat_id: Number(chatId), + from_message_id: currentFromId, + offset: 0, + limit: Math.min(limit, 100), + only_local: false, + }); + if (!result.messages || result.messages.length === 0) + break; + totalScanned += result.messages.length; + for (const msg of result.messages) { + // Check for archive documents + const doc = msg.content?.document; + if (doc?.file_name && doc.document && isArchiveAttachment(doc.file_name)) { + archives.push({ + id: BigInt(msg.id), + fileName: doc.file_name, + fileId: String(doc.document.id), + fileSize: BigInt(doc.document.size), + date: new Date(msg.date * 1000), + }); + continue; + } + // Check for photo messages (potential previews) + const photo = msg.content?.photo; + const caption = msg.content?.caption?.text ?? ""; + if (photo?.sizes && photo.sizes.length > 0) { + const smallest = photo.sizes[0]; + photos.push({ + id: BigInt(msg.id), + date: new Date(msg.date * 1000), + caption, + fileId: String(smallest.photo.id), + fileSize: smallest.photo.size || smallest.photo.expected_size, + }); + } + } + // Report scanning progress after each page + onProgress?.(totalScanned); + currentFromId = result.messages[result.messages.length - 1].id; + // Stuck detection: if from_message_id didn't advance, break to prevent infinite loop + if (currentFromId === previousFromId) { + log.warn({ chatId: chatId.toString(), currentFromId, totalScanned }, "Pagination stuck (from_message_id not advancing), breaking"); + break; + } + // Stop scanning once we've gone past the boundary (this page is the lookback) + if (boundary && currentFromId < boundary) + break; + if (result.messages.length < Math.min(limit, 100)) + break; + // Rate limit delay + await sleep(config.apiDelayMs); + } + log.info({ chatId: chatId.toString(), archives: archives.length, photos: photos.length, totalScanned, pages: pageCount }, "Channel scan complete"); + // Reverse to chronological order (oldest first) so worker processes old→new + return { + archives: archives.reverse(), + photos: photos.reverse(), + totalScanned, + }; +} +/** + * Download a photo thumbnail from Telegram and return its raw bytes. + * Uses synchronous download (photos are small, typically < 100KB). + * Returns null if download fails (non-critical). + */ +export async function downloadPhotoThumbnail(client, fileId) { + const numericId = parseInt(fileId, 10); + try { + const result = (await client.invoke({ + _: "downloadFile", + file_id: numericId, + priority: 1, // Low priority — thumbnails are nice-to-have + offset: 0, + limit: 0, + synchronous: true, // Small file — wait for it + })); + if (result?.local?.is_downloading_completed && result.local.path) { + const data = await readFile(result.local.path); + log.debug({ fileId, bytes: data.length }, "Downloaded photo thumbnail"); + return data; + } + } + catch (err) { + log.warn({ fileId, err }, "Failed to download photo thumbnail"); + } + return null; +} +/** + * Download a file from Telegram to a local path with progress tracking + * and integrity verification. + * + * Progress flow: + * 1. Starts async download via TDLib + * 2. Listens for `updateFile` events to track download progress + * 3. Logs progress at every 10% increment + * 4. Once complete, verifies the local file size matches the expected size + * 5. Moves the file from TDLib's cache to the destination path + * + * Verification: + * - Compares actual file size on disk to the expected size from Telegram + * - Throws on mismatch (partial/corrupt download) + * - Throws on timeout (configurable, scales with file size) + * - Throws if download stops without completing (network error, etc.) + */ +export async function downloadFile(client, fileId, destPath, expectedSize, fileName, onProgress) { + const numericId = parseInt(fileId, 10); + const totalBytes = Number(expectedSize); + log.info({ fileId, fileName, destPath, totalBytes }, "Starting file download"); + // Report initial progress + onProgress?.({ + fileId, + fileName, + downloadedBytes: 0, + totalBytes, + percent: 0, + isComplete: false, + }); + return new Promise((resolve, reject) => { + let lastLoggedPercent = 0; + let settled = false; + // Timeout: 10 minutes per GB, minimum 5 minutes + const timeoutMs = Math.max(5 * 60_000, (totalBytes / (1024 * 1024 * 1024)) * 10 * 60_000); + const timer = setTimeout(() => { + if (!settled) { + settled = true; + cleanup(); + reject(new Error(`Download timed out after ${Math.round(timeoutMs / 60_000)}min for ${fileName}`)); + } + }, timeoutMs); + // Listen for file update events to track progress + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const handleUpdate = (update) => { + if (update?._ !== "updateFile") + return; + const file = update.file; + if (!file || file.id !== numericId) + return; + const downloaded = file.local.downloaded_size; + const percent = totalBytes > 0 ? Math.round((downloaded / totalBytes) * 100) : 0; + // Log at every 10% increment + if (percent >= lastLoggedPercent + 10) { + lastLoggedPercent = percent - (percent % 10); + log.info({ fileId, fileName, downloaded, totalBytes, percent: `${percent}%` }, "Download progress"); + } + // Report to callback + onProgress?.({ + fileId, + fileName, + downloadedBytes: downloaded, + totalBytes, + percent, + isComplete: file.local.is_downloading_completed, + }); + // Download finished + if (file.local.is_downloading_completed) { + if (!settled) { + settled = true; + cleanup(); + verifyAndMove(file.local.path, destPath, totalBytes, fileName, fileId) + .then(resolve) + .catch(reject); + } + } + // Download stopped without completing (network error, cancelled, etc.) + if (!file.local.is_downloading_active && + !file.local.is_downloading_completed) { + if (!settled) { + settled = true; + cleanup(); + reject(new Error(`Download stopped unexpectedly for ${fileName} ` + + `(${downloaded}/${totalBytes} bytes, ${percent}%)`)); + } + } + }; + const cleanup = () => { + clearTimeout(timer); + client.off("update", handleUpdate); + }; + // Subscribe to updates BEFORE starting download + client.on("update", handleUpdate); + // Start async download (non-blocking — progress via updateFile events) + client + .invoke({ + _: "downloadFile", + file_id: numericId, + priority: 32, + offset: 0, + limit: 0, + synchronous: false, + }) + .then((result) => { + // If the file was already cached locally, invoke returns immediately + const file = result; + if (file?.local?.is_downloading_completed && !settled) { + settled = true; + cleanup(); + verifyAndMove(file.local.path, destPath, totalBytes, fileName, fileId) + .then(resolve) + .catch(reject); + } + }) + .catch((err) => { + if (!settled) { + settled = true; + cleanup(); + reject(err); + } + }); + }); +} +/** + * Verify the downloaded file's size matches the expected size, + * then move it to the destination path. + */ +async function verifyAndMove(localPath, destPath, expectedBytes, fileName, fileId) { + const stats = await stat(localPath); + const actualBytes = stats.size; + if (expectedBytes > 0 && actualBytes !== expectedBytes) { + log.error({ fileId, fileName, expectedBytes, actualBytes }, "Download size mismatch — file is incomplete or corrupted"); + throw new Error(`Download verification failed for ${fileName}: ` + + `expected ${expectedBytes} bytes, got ${actualBytes} bytes`); + } + log.info({ fileId, fileName, bytes: actualBytes, destPath }, "File verified and complete"); + // Move from TDLib's cache to our temp directory. + // Use rename first (fast, same filesystem), fall back to copy+delete + // when source and destination are on different filesystems (EXDEV). + try { + await rename(localPath, destPath); + } + catch (err) { + if (err.code === "EXDEV") { + log.debug({ fileId, fileName }, "Cross-device rename — falling back to copy + unlink"); + await copyFile(localPath, destPath); + await unlink(localPath); + } + else { + throw err; + } + } +} +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} +//# sourceMappingURL=download.js.map \ No newline at end of file diff --git a/worker/dist/tdlib/download.js.map b/worker/dist/tdlib/download.js.map new file mode 100644 index 0000000..5fc9f0b --- /dev/null +++ b/worker/dist/tdlib/download.js.map @@ -0,0 +1 @@ +{"version":3,"file":"download.js","sourceRoot":"","sources":["../../src/tdlib/download.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AACvE,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAI3D,MAAM,GAAG,GAAG,WAAW,CAAC,UAAU,CAAC,CAAC;AAEpC,kFAAkF;AAClF,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,CAAC;AAEnC,+CAA+C;AAC/C,MAAM,CAAC,MAAM,iBAAiB,GAAG,OAAO,CAAC,CAAC,YAAY;AAiEtD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,MAAc;AACd,8DAA8D;AAC9D,OAA4B,EAC5B,SAAS,GAAG,iBAAiB;IAE7B,OAAO,IAAI,OAAO,CAAI,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACxC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,MAAM,CAAC,IAAI,KAAK,CAAC,gCAAgC,SAAS,UAAU,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACpF,CAAC,EAAE,SAAS,CAAC,CAAC;QAEb,MAAM,CAAC,MAAM,CAAC,OAAO,CAAgB;aACnC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE;YACf,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,OAAO,CAAC,MAAM,CAAC,CAAC;QAClB,CAAC,CAAC;aACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACb,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,CAAC,GAAG,CAAC,CAAC;QACd,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,MAAc,EACd,MAAc,EACd,sBAAsC,EACtC,KAAK,GAAG,GAAG,EACX,UAAiC;IAEjC,MAAM,QAAQ,GAAsB,EAAE,CAAC;IACvC,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,MAAM,QAAQ,GAAG,sBAAsB,CAAC,CAAC,CAAC,MAAM,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAEhF,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,iDAAiD;IACjD,OAAO,IAAI,EAAE,CAAC;QACZ,IAAI,SAAS,IAAI,cAAc,EAAE,CAAC;YAChC,GAAG,CAAC,IAAI,CACN,EAAE,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,YAAY,EAAE,EACtD,+CAA+C,CAChD,CAAC;YACF,MAAM;QACR,CAAC;QACD,SAAS,EAAE,CAAC;QAEZ,MAAM,cAAc,GAAG,aAAa,CAAC;QAErC,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAA4B,MAAM,EAAE;YACxE,CAAC,EAAE,gBAAgB;YACnB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC;YACvB,eAAe,EAAE,aAAa;YAC9B,MAAM,EAAE,CAAC;YACT,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC;YAC3B,UAAU,EAAE,KAAK;SAClB,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM;QAE5D,YAAY,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;QAEvC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YAClC,8BAA8B;YAC9B,MAAM,GAAG,GAAG,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC;YAClC,IAAI,GAAG,EAAE,SAAS,IAAI,GAAG,CAAC,QAAQ,IAAI,mBAAmB,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;gBACzE,QAAQ,CAAC,IAAI,CAAC;oBACZ,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;oBAClB,QAAQ,EAAE,GAAG,CAAC,SAAS;oBACvB,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAC/B,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC;oBACnC,IAAI,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC;iBAChC,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;YAED,gDAAgD;YAChD,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC;YACjC,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC;YACjD,IAAI,KAAK,EAAE,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC3C,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAChC,MAAM,CAAC,IAAI,CAAC;oBACV,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;oBAClB,IAAI,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC;oBAC/B,OAAO;oBACP,MAAM,EAAE,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;oBACjC,QAAQ,EAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI,QAAQ,CAAC,KAAK,CAAC,aAAa;iBAC9D,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,2CAA2C;QAC3C,UAAU,EAAE,CAAC,YAAY,CAAC,CAAC;QAE3B,aAAa,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QAE/D,qFAAqF;QACrF,IAAI,aAAa,KAAK,cAAc,EAAE,CAAC;YACrC,GAAG,CAAC,IAAI,CACN,EAAE,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE,EAAE,aAAa,EAAE,YAAY,EAAE,EAC1D,4DAA4D,CAC7D,CAAC;YACF,MAAM;QACR,CAAC;QAED,8EAA8E;QAC9E,IAAI,QAAQ,IAAI,aAAa,GAAG,QAAQ;YAAE,MAAM;QAEhD,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC;YAAE,MAAM;QAEzD,mBAAmB;QACnB,MAAM,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IACjC,CAAC;IAED,GAAG,CAAC,IAAI,CACN,EAAE,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,EAAE,KAAK,EAAE,SAAS,EAAE,EAC/G,uBAAuB,CACxB,CAAC;IAEF,4EAA4E;IAC5E,OAAO;QACL,QAAQ,EAAE,QAAQ,CAAC,OAAO,EAAE;QAC5B,MAAM,EAAE,MAAM,CAAC,OAAO,EAAE;QACxB,YAAY;KACb,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,MAAc,EACd,MAAc;IAEd,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAEvC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,CAAC,MAAM,MAAM,CAAC,MAAM,CAAC;YAClC,CAAC,EAAE,cAAc;YACjB,OAAO,EAAE,SAAS;YAClB,QAAQ,EAAE,CAAC,EAAE,6CAA6C;YAC1D,MAAM,EAAE,CAAC;YACT,KAAK,EAAE,CAAC;YACR,WAAW,EAAE,IAAI,EAAE,2BAA2B;SAC/C,CAAC,CAAW,CAAC;QAEd,IAAI,MAAM,EAAE,KAAK,EAAE,wBAAwB,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;YACjE,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAC/C,GAAG,CAAC,KAAK,CACP,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,MAAM,EAAE,EAC9B,4BAA4B,CAC7B,CAAC;YACF,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,oCAAoC,CAAC,CAAC;IAClE,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAaD;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,MAAc,EACd,MAAc,EACd,QAAgB,EAChB,YAAoB,EACpB,QAAgB,EAChB,UAA6B;IAE7B,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACvC,MAAM,UAAU,GAAG,MAAM,CAAC,YAAY,CAAC,CAAC;IAExC,GAAG,CAAC,IAAI,CACN,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAE,EAC1C,wBAAwB,CACzB,CAAC;IAEF,0BAA0B;IAC1B,UAAU,EAAE,CAAC;QACX,MAAM;QACN,QAAQ;QACR,eAAe,EAAE,CAAC;QAClB,UAAU;QACV,OAAO,EAAE,CAAC;QACV,UAAU,EAAE,KAAK;KAClB,CAAC,CAAC;IAEH,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC3C,IAAI,iBAAiB,GAAG,CAAC,CAAC;QAC1B,IAAI,OAAO,GAAG,KAAK,CAAC;QAEpB,gDAAgD;QAChD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CACxB,CAAC,GAAG,MAAM,EACV,CAAC,UAAU,GAAG,CAAC,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,GAAG,EAAE,GAAG,MAAM,CAClD,CAAC;QACF,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,OAAO,GAAG,IAAI,CAAC;gBACf,OAAO,EAAE,CAAC;gBACV,MAAM,CACJ,IAAI,KAAK,CACP,4BAA4B,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,MAAM,CAAC,WAAW,QAAQ,EAAE,CAChF,CACF,CAAC;YACJ,CAAC;QACH,CAAC,EAAE,SAAS,CAAC,CAAC;QAEd,kDAAkD;QAClD,8DAA8D;QAC9D,MAAM,YAAY,GAAG,CAAC,MAAW,EAAE,EAAE;YACnC,IAAI,MAAM,EAAE,CAAC,KAAK,YAAY;gBAAE,OAAO;YACvC,MAAM,IAAI,GAAG,MAAM,CAAC,IAA0B,CAAC;YAC/C,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE,KAAK,SAAS;gBAAE,OAAO;YAE3C,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC;YAC9C,MAAM,OAAO,GACX,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,UAAU,GAAG,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAEnE,6BAA6B;YAC7B,IAAI,OAAO,IAAI,iBAAiB,GAAG,EAAE,EAAE,CAAC;gBACtC,iBAAiB,GAAG,OAAO,GAAG,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC;gBAC7C,GAAG,CAAC,IAAI,CACN,EAAE,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,OAAO,GAAG,EAAE,EACpE,mBAAmB,CACpB,CAAC;YACJ,CAAC;YAED,qBAAqB;YACrB,UAAU,EAAE,CAAC;gBACX,MAAM;gBACN,QAAQ;gBACR,eAAe,EAAE,UAAU;gBAC3B,UAAU;gBACV,OAAO;gBACP,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,wBAAwB;aAChD,CAAC,CAAC;YAEH,oBAAoB;YACpB,IAAI,IAAI,CAAC,KAAK,CAAC,wBAAwB,EAAE,CAAC;gBACxC,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,OAAO,GAAG,IAAI,CAAC;oBACf,OAAO,EAAE,CAAC;oBACV,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,CAAC;yBACnE,IAAI,CAAC,OAAO,CAAC;yBACb,KAAK,CAAC,MAAM,CAAC,CAAC;gBACnB,CAAC;YACH,CAAC;YAED,uEAAuE;YACvE,IACE,CAAC,IAAI,CAAC,KAAK,CAAC,qBAAqB;gBACjC,CAAC,IAAI,CAAC,KAAK,CAAC,wBAAwB,EACpC,CAAC;gBACD,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,OAAO,GAAG,IAAI,CAAC;oBACf,OAAO,EAAE,CAAC;oBACV,MAAM,CACJ,IAAI,KAAK,CACP,qCAAqC,QAAQ,GAAG;wBAC9C,IAAI,UAAU,IAAI,UAAU,WAAW,OAAO,IAAI,CACrD,CACF,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC,CAAC;QAEF,MAAM,OAAO,GAAG,GAAG,EAAE;YACnB,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;QACrC,CAAC,CAAC;QAEF,gDAAgD;QAChD,MAAM,CAAC,EAAE,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;QAElC,uEAAuE;QACvE,MAAM;aACH,MAAM,CAAC;YACN,CAAC,EAAE,cAAc;YACjB,OAAO,EAAE,SAAS;YAClB,QAAQ,EAAE,EAAE;YACZ,MAAM,EAAE,CAAC;YACT,KAAK,EAAE,CAAC;YACR,WAAW,EAAE,KAAK;SACnB,CAAC;aACD,IAAI,CAAC,CAAC,MAAe,EAAE,EAAE;YACxB,qEAAqE;YACrE,MAAM,IAAI,GAAG,MAA4B,CAAC;YAC1C,IAAI,IAAI,EAAE,KAAK,EAAE,wBAAwB,IAAI,CAAC,OAAO,EAAE,CAAC;gBACtD,OAAO,GAAG,IAAI,CAAC;gBACf,OAAO,EAAE,CAAC;gBACV,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,CAAC;qBACnE,IAAI,CAAC,OAAO,CAAC;qBACb,KAAK,CAAC,MAAM,CAAC,CAAC;YACnB,CAAC;QACH,CAAC,CAAC;aACD,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;YACtB,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,OAAO,GAAG,IAAI,CAAC;gBACf,OAAO,EAAE,CAAC;gBACV,MAAM,CAAC,GAAG,CAAC,CAAC;YACd,CAAC;QACH,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,aAAa,CAC1B,SAAiB,EACjB,QAAgB,EAChB,aAAqB,EACrB,QAAgB,EAChB,MAAc;IAEd,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,CAAC;IACpC,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC;IAE/B,IAAI,aAAa,GAAG,CAAC,IAAI,WAAW,KAAK,aAAa,EAAE,CAAC;QACvD,GAAG,CAAC,KAAK,CACP,EAAE,MAAM,EAAE,QAAQ,EAAE,aAAa,EAAE,WAAW,EAAE,EAChD,0DAA0D,CAC3D,CAAC;QACF,MAAM,IAAI,KAAK,CACb,oCAAoC,QAAQ,IAAI;YAC9C,YAAY,aAAa,eAAe,WAAW,QAAQ,CAC9D,CAAC;IACJ,CAAC;IAED,GAAG,CAAC,IAAI,CACN,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,EAClD,4BAA4B,CAC7B,CAAC;IAEF,iDAAiD;IACjD,qEAAqE;IACrE,oEAAoE;IACpE,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IACpC,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,IAAK,GAA6B,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YACpD,GAAG,CAAC,KAAK,CACP,EAAE,MAAM,EAAE,QAAQ,EAAE,EACpB,qDAAqD,CACtD,CAAC;YACF,MAAM,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;YACpC,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;QAC1B,CAAC;aAAM,CAAC;YACN,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC"} \ No newline at end of file diff --git a/worker/dist/tdlib/topics.d.ts b/worker/dist/tdlib/topics.d.ts new file mode 100644 index 0000000..23ecbae --- /dev/null +++ b/worker/dist/tdlib/topics.d.ts @@ -0,0 +1,32 @@ +import type { Client } from "tdl"; +import type { ChannelScanResult, ScanProgressCallback } from "./download.js"; +export interface ForumTopic { + topicId: bigint; + name: string; +} +/** + * Check if a chat is a forum supergroup (topics enabled). + */ +export declare function isChatForum(client: Client, chatId: bigint): Promise; +/** + * Get all forum topics in a supergroup. + * Includes stuck detection and timeout protection on API calls. + */ +export declare function getForumTopicList(client: Client, chatId: bigint): Promise; +/** + * Fetch messages from a specific forum topic (thread), stopping once + * we've scanned past the last-processed boundary (with one page of lookback). + * Uses searchChatMessages with message_thread_id to scan within a topic. + * + * Returns messages in chronological order (oldest first). + * + * When `lastProcessedMessageId` is null (first run), scans everything. + * The worker applies a post-grouping filter to skip fully-processed sets, + * and keeps `packageExistsBySourceMessage` as a safety net. + * + * Safety features: + * - Max page limit to prevent infinite loops + * - Stuck detection: breaks if from_message_id stops advancing + * - Timeout on each TDLib API call + */ +export declare function getTopicMessages(client: Client, chatId: bigint, topicId: bigint, lastProcessedMessageId?: bigint | null, limit?: number, onProgress?: ScanProgressCallback): Promise; diff --git a/worker/dist/tdlib/topics.js b/worker/dist/tdlib/topics.js new file mode 100644 index 0000000..328730f --- /dev/null +++ b/worker/dist/tdlib/topics.js @@ -0,0 +1,196 @@ +import { config } from "../util/config.js"; +import { childLogger } from "../util/logger.js"; +import { isArchiveAttachment } from "../archive/detect.js"; +import { invokeWithTimeout, MAX_SCAN_PAGES } from "./download.js"; +const log = childLogger("topics"); +/** + * Check if a chat is a forum supergroup (topics enabled). + */ +export async function isChatForum(client, chatId) { + try { + const chat = await invokeWithTimeout(client, { + _: "getChat", + chat_id: Number(chatId), + }); + if (chat.type?._ === "chatTypeSupergroup" && chat.type.is_forum) { + return true; + } + // Also check via getSupergroup for older TDLib versions + if (chat.type?._ === "chatTypeSupergroup" && chat.type.supergroup_id) { + const sg = await invokeWithTimeout(client, { + _: "getSupergroup", + supergroup_id: chat.type.supergroup_id, + }); + return sg.is_forum === true; + } + return false; + } + catch (err) { + log.warn({ err, chatId: chatId.toString() }, "Failed to check if chat is forum"); + return false; + } +} +/** + * Get all forum topics in a supergroup. + * Includes stuck detection and timeout protection on API calls. + */ +export async function getForumTopicList(client, chatId) { + const topics = []; + let offsetDate = 0; + let offsetMessageId = 0; + let offsetMessageThreadId = 0; + let pageCount = 0; + // eslint-disable-next-line no-constant-condition + while (true) { + if (pageCount >= MAX_SCAN_PAGES) { + log.warn({ chatId: chatId.toString(), pageCount, topicCount: topics.length }, "Hit max page limit for topic enumeration, stopping"); + break; + } + pageCount++; + const prevOffsetDate = offsetDate; + const prevOffsetMessageId = offsetMessageId; + const prevOffsetMessageThreadId = offsetMessageThreadId; + const result = await invokeWithTimeout(client, { + _: "getForumTopics", + chat_id: Number(chatId), + query: "", + offset_date: offsetDate, + offset_message_id: offsetMessageId, + offset_message_thread_id: offsetMessageThreadId, + limit: 100, + }); + if (!result.topics || result.topics.length === 0) + break; + for (const t of result.topics) { + if (!t.info?.message_thread_id) + continue; + // Skip the "General" topic — it's not creator-specific + if (t.info.is_general) + continue; + topics.push({ + topicId: BigInt(t.info.message_thread_id), + name: t.info.name ?? "Unnamed", + }); + } + // Check if there are more pages + if (!result.next_offset_date && + !result.next_offset_message_id && + !result.next_offset_message_thread_id) { + break; + } + offsetDate = result.next_offset_date ?? 0; + offsetMessageId = result.next_offset_message_id ?? 0; + offsetMessageThreadId = result.next_offset_message_thread_id ?? 0; + // Stuck detection: if offsets didn't advance, break + if (offsetDate === prevOffsetDate && + offsetMessageId === prevOffsetMessageId && + offsetMessageThreadId === prevOffsetMessageThreadId) { + log.warn({ chatId: chatId.toString(), topicCount: topics.length }, "Topic pagination stuck (offsets not advancing), breaking"); + break; + } + await sleep(config.apiDelayMs); + } + log.info({ chatId: chatId.toString(), topicCount: topics.length }, "Enumerated forum topics"); + return topics; +} +/** + * Fetch messages from a specific forum topic (thread), stopping once + * we've scanned past the last-processed boundary (with one page of lookback). + * Uses searchChatMessages with message_thread_id to scan within a topic. + * + * Returns messages in chronological order (oldest first). + * + * When `lastProcessedMessageId` is null (first run), scans everything. + * The worker applies a post-grouping filter to skip fully-processed sets, + * and keeps `packageExistsBySourceMessage` as a safety net. + * + * Safety features: + * - Max page limit to prevent infinite loops + * - Stuck detection: breaks if from_message_id stops advancing + * - Timeout on each TDLib API call + */ +export async function getTopicMessages(client, chatId, topicId, lastProcessedMessageId, limit = 100, onProgress) { + const archives = []; + const photos = []; + const boundary = lastProcessedMessageId ? Number(lastProcessedMessageId) : null; + let currentFromId = 0; + let totalScanned = 0; + let pageCount = 0; + // eslint-disable-next-line no-constant-condition + while (true) { + if (pageCount >= MAX_SCAN_PAGES) { + log.warn({ chatId: chatId.toString(), topicId: topicId.toString(), pageCount, totalScanned }, "Hit max page limit for topic scan, stopping"); + break; + } + pageCount++; + const previousFromId = currentFromId; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const result = await invokeWithTimeout(client, { + _: "searchChatMessages", + chat_id: Number(chatId), + query: "", + message_thread_id: Number(topicId), + from_message_id: currentFromId, + offset: 0, + limit: Math.min(limit, 100), + filter: null, + sender_id: null, + saved_messages_topic_id: 0, + }); + if (!result.messages || result.messages.length === 0) + break; + totalScanned += result.messages.length; + for (const msg of result.messages) { + // Check for archive documents + const doc = msg.content?.document; + if (doc?.file_name && doc.document && isArchiveAttachment(doc.file_name)) { + archives.push({ + id: BigInt(msg.id), + fileName: doc.file_name, + fileId: String(doc.document.id), + fileSize: BigInt(doc.document.size), + date: new Date(msg.date * 1000), + }); + continue; + } + // Check for photo messages (potential previews) + const photo = msg.content?.photo; + const caption = msg.content?.caption?.text ?? ""; + if (photo?.sizes && photo.sizes.length > 0) { + const smallest = photo.sizes[0]; + photos.push({ + id: BigInt(msg.id), + date: new Date(msg.date * 1000), + caption, + fileId: String(smallest.photo.id), + fileSize: smallest.photo.size || smallest.photo.expected_size, + }); + } + } + // Report scanning progress after each page + onProgress?.(totalScanned); + currentFromId = result.messages[result.messages.length - 1].id; + // Stuck detection: if from_message_id didn't advance, break to prevent infinite loop + if (currentFromId === previousFromId) { + log.warn({ chatId: chatId.toString(), topicId: topicId.toString(), currentFromId, totalScanned }, "Topic pagination stuck (from_message_id not advancing), breaking"); + break; + } + // Stop scanning once we've gone past the boundary (this page is the lookback) + if (boundary && currentFromId < boundary) + break; + if (result.messages.length < Math.min(limit, 100)) + break; + await sleep(config.apiDelayMs); + } + log.info({ chatId: chatId.toString(), topicId: topicId.toString(), archives: archives.length, photos: photos.length, totalScanned, pages: pageCount }, "Topic scan complete"); + // Reverse to chronological order (oldest first) so worker processes old→new + return { + archives: archives.reverse(), + photos: photos.reverse(), + totalScanned, + }; +} +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} +//# sourceMappingURL=topics.js.map \ No newline at end of file diff --git a/worker/dist/tdlib/topics.js.map b/worker/dist/tdlib/topics.js.map new file mode 100644 index 0000000..940c3cf --- /dev/null +++ b/worker/dist/tdlib/topics.js.map @@ -0,0 +1 @@ +{"version":3,"file":"topics.js","sourceRoot":"","sources":["../../src/tdlib/topics.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAI3D,OAAO,EAAE,iBAAiB,EAAE,cAAc,EAAqB,MAAM,eAAe,CAAC;AAErF,MAAM,GAAG,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC;AAOlC;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,MAAc,EACd,MAAc;IAEd,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAMjC,MAAM,EAAE;YACT,CAAC,EAAE,SAAS;YACZ,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC;SACxB,CAAC,CAAC;QAEH,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,oBAAoB,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YAChE,OAAO,IAAI,CAAC;QACd,CAAC;QAED,wDAAwD;QACxD,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,oBAAoB,IAAI,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACrE,MAAM,EAAE,GAAG,MAAM,iBAAiB,CAAyB,MAAM,EAAE;gBACjE,CAAC,EAAE,eAAe;gBAClB,aAAa,EAAE,IAAI,CAAC,IAAI,CAAC,aAAa;aACvC,CAAC,CAAC;YACH,OAAO,EAAE,CAAC,QAAQ,KAAK,IAAI,CAAC;QAC9B,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE,EAAE,EAAE,kCAAkC,CAAC,CAAC;QACjF,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,MAAc,EACd,MAAc;IAEd,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,IAAI,qBAAqB,GAAG,CAAC,CAAC;IAC9B,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,iDAAiD;IACjD,OAAO,IAAI,EAAE,CAAC;QACZ,IAAI,SAAS,IAAI,cAAc,EAAE,CAAC;YAChC,GAAG,CAAC,IAAI,CACN,EAAE,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,EACnE,oDAAoD,CACrD,CAAC;YACF,MAAM;QACR,CAAC;QACD,SAAS,EAAE,CAAC;QAEZ,MAAM,cAAc,GAAG,UAAU,CAAC;QAClC,MAAM,mBAAmB,GAAG,eAAe,CAAC;QAC5C,MAAM,yBAAyB,GAAG,qBAAqB,CAAC;QAExD,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAWnC,MAAM,EAAE;YACT,CAAC,EAAE,gBAAgB;YACnB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC;YACvB,KAAK,EAAE,EAAE;YACT,WAAW,EAAE,UAAU;YACvB,iBAAiB,EAAE,eAAe;YAClC,wBAAwB,EAAE,qBAAqB;YAC/C,KAAK,EAAE,GAAG;SACX,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM;QAExD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YAC9B,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,iBAAiB;gBAAE,SAAS;YACzC,uDAAuD;YACvD,IAAI,CAAC,CAAC,IAAI,CAAC,UAAU;gBAAE,SAAS;YAEhC,MAAM,CAAC,IAAI,CAAC;gBACV,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC;gBACzC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,SAAS;aAC/B,CAAC,CAAC;QACL,CAAC;QAED,gCAAgC;QAChC,IACE,CAAC,MAAM,CAAC,gBAAgB;YACxB,CAAC,MAAM,CAAC,sBAAsB;YAC9B,CAAC,MAAM,CAAC,6BAA6B,EACrC,CAAC;YACD,MAAM;QACR,CAAC;QAED,UAAU,GAAG,MAAM,CAAC,gBAAgB,IAAI,CAAC,CAAC;QAC1C,eAAe,GAAG,MAAM,CAAC,sBAAsB,IAAI,CAAC,CAAC;QACrD,qBAAqB,GAAG,MAAM,CAAC,6BAA6B,IAAI,CAAC,CAAC;QAElE,oDAAoD;QACpD,IACE,UAAU,KAAK,cAAc;YAC7B,eAAe,KAAK,mBAAmB;YACvC,qBAAqB,KAAK,yBAAyB,EACnD,CAAC;YACD,GAAG,CAAC,IAAI,CACN,EAAE,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE,EAAE,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,EACxD,0DAA0D,CAC3D,CAAC;YACF,MAAM;QACR,CAAC;QAED,MAAM,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IACjC,CAAC;IAED,GAAG,CAAC,IAAI,CACN,EAAE,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE,EAAE,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,EACxD,yBAAyB,CAC1B,CAAC;IAEF,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,MAAc,EACd,MAAc,EACd,OAAe,EACf,sBAAsC,EACtC,KAAK,GAAG,GAAG,EACX,UAAiC;IAEjC,MAAM,QAAQ,GAAsB,EAAE,CAAC;IACvC,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,MAAM,QAAQ,GAAG,sBAAsB,CAAC,CAAC,CAAC,MAAM,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAEhF,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,iDAAiD;IACjD,OAAO,IAAI,EAAE,CAAC;QACZ,IAAI,SAAS,IAAI,cAAc,EAAE,CAAC;YAChC,GAAG,CAAC,IAAI,CACN,EAAE,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,YAAY,EAAE,EACnF,6CAA6C,CAC9C,CAAC;YACF,MAAM;QACR,CAAC;QACD,SAAS,EAAE,CAAC;QAEZ,MAAM,cAAc,GAAG,aAAa,CAAC;QAErC,8DAA8D;QAC9D,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAwBnC,MAAM,EAAE;YACT,CAAC,EAAE,oBAAoB;YACvB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC;YACvB,KAAK,EAAE,EAAE;YACT,iBAAiB,EAAE,MAAM,CAAC,OAAO,CAAC;YAClC,eAAe,EAAE,aAAa;YAC9B,MAAM,EAAE,CAAC;YACT,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC;YAC3B,MAAM,EAAE,IAAI;YACZ,SAAS,EAAE,IAAI;YACf,uBAAuB,EAAE,CAAC;SAC3B,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM;QAE5D,YAAY,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;QAEvC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YAClC,8BAA8B;YAC9B,MAAM,GAAG,GAAG,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC;YAClC,IAAI,GAAG,EAAE,SAAS,IAAI,GAAG,CAAC,QAAQ,IAAI,mBAAmB,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;gBACzE,QAAQ,CAAC,IAAI,CAAC;oBACZ,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;oBAClB,QAAQ,EAAE,GAAG,CAAC,SAAS;oBACvB,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAC/B,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC;oBACnC,IAAI,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC;iBAChC,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;YAED,gDAAgD;YAChD,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC;YACjC,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC;YACjD,IAAI,KAAK,EAAE,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC3C,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAChC,MAAM,CAAC,IAAI,CAAC;oBACV,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;oBAClB,IAAI,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC;oBAC/B,OAAO;oBACP,MAAM,EAAE,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;oBACjC,QAAQ,EAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI,QAAQ,CAAC,KAAK,CAAC,aAAa;iBAC9D,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,2CAA2C;QAC3C,UAAU,EAAE,CAAC,YAAY,CAAC,CAAC;QAE3B,aAAa,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QAE/D,qFAAqF;QACrF,IAAI,aAAa,KAAK,cAAc,EAAE,CAAC;YACrC,GAAG,CAAC,IAAI,CACN,EAAE,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,QAAQ,EAAE,EAAE,aAAa,EAAE,YAAY,EAAE,EACvF,kEAAkE,CACnE,CAAC;YACF,MAAM;QACR,CAAC;QAED,8EAA8E;QAC9E,IAAI,QAAQ,IAAI,aAAa,GAAG,QAAQ;YAAE,MAAM;QAEhD,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC;YAAE,MAAM;QAEzD,MAAM,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IACjC,CAAC;IAED,GAAG,CAAC,IAAI,CACN,EAAE,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,EAAE,KAAK,EAAE,SAAS,EAAE,EAC5I,qBAAqB,CACtB,CAAC;IAEF,4EAA4E;IAC5E,OAAO;QACL,QAAQ,EAAE,QAAQ,CAAC,OAAO,EAAE;QAC5B,MAAM,EAAE,MAAM,CAAC,OAAO,EAAE;QACxB,YAAY;KACb,CAAC;AACJ,CAAC;AAED,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC"} \ No newline at end of file diff --git a/worker/dist/upload/channel.d.ts b/worker/dist/upload/channel.d.ts new file mode 100644 index 0000000..c2db2a9 --- /dev/null +++ b/worker/dist/upload/channel.d.ts @@ -0,0 +1,16 @@ +import type { Client } from "tdl"; +export interface UploadResult { + messageId: bigint; +} +/** + * Upload one or more files to a destination Telegram channel. + * For multipart archives, each file is sent as a separate message. + * Returns the **final** (server-assigned) message ID of the first uploaded message. + * + * IMPORTANT: `sendMessage` returns a *temporary* message immediately. + * The actual file upload happens asynchronously in TDLib. We listen for + * `updateMessageSendSucceeded` to get the real server-side message ID and + * to make sure the upload is fully committed before we clean up temp files + * or close the TDLib client (which would cancel pending uploads). + */ +export declare function uploadToChannel(client: Client, chatId: bigint, filePaths: string[], caption?: string): Promise; diff --git a/worker/dist/upload/channel.js b/worker/dist/upload/channel.js new file mode 100644 index 0000000..5a4a045 --- /dev/null +++ b/worker/dist/upload/channel.js @@ -0,0 +1,137 @@ +import path from "path"; +import { stat } from "fs/promises"; +import { config } from "../util/config.js"; +import { childLogger } from "../util/logger.js"; +const log = childLogger("upload"); +/** + * Upload one or more files to a destination Telegram channel. + * For multipart archives, each file is sent as a separate message. + * Returns the **final** (server-assigned) message ID of the first uploaded message. + * + * IMPORTANT: `sendMessage` returns a *temporary* message immediately. + * The actual file upload happens asynchronously in TDLib. We listen for + * `updateMessageSendSucceeded` to get the real server-side message ID and + * to make sure the upload is fully committed before we clean up temp files + * or close the TDLib client (which would cancel pending uploads). + */ +export async function uploadToChannel(client, chatId, filePaths, caption) { + let firstMessageId = null; + for (let i = 0; i < filePaths.length; i++) { + const filePath = filePaths[i]; + const fileCaption = i === 0 && caption ? caption : undefined; + const fileName = path.basename(filePath); + let fileSizeMB = 0; + try { + const s = await stat(filePath); + fileSizeMB = Math.round(s.size / (1024 * 1024)); + } + catch { + // Non-critical + } + log.info({ chatId: Number(chatId), fileName, sizeMB: fileSizeMB, part: i + 1, total: filePaths.length }, "Uploading file to channel"); + const serverMsgId = await sendAndWaitForUpload(client, chatId, filePath, fileCaption, fileName, fileSizeMB); + if (i === 0) { + firstMessageId = serverMsgId; + } + // Rate limit delay between uploads + if (i < filePaths.length - 1) { + await sleep(config.apiDelayMs); + } + } + if (firstMessageId === null) { + throw new Error("Upload failed: no messages sent"); + } + log.info({ chatId: Number(chatId), messageId: Number(firstMessageId), files: filePaths.length }, "All uploads confirmed by Telegram"); + return { messageId: firstMessageId }; +} +/** + * Send a single file message and wait for Telegram to confirm the upload. + * Returns the final server-assigned message ID. + */ +async function sendAndWaitForUpload(client, chatId, filePath, caption, fileName, fileSizeMB) { + // Send the message — this returns a temporary message immediately + const tempMsg = (await client.invoke({ + _: "sendMessage", + chat_id: Number(chatId), + input_message_content: { + _: "inputMessageDocument", + document: { + _: "inputFileLocal", + path: filePath, + }, + caption: caption + ? { + _: "formattedText", + text: caption, + } + : undefined, + }, + })); + const tempMsgId = tempMsg.id; + log.debug({ fileName, tempMsgId }, "Message queued, waiting for upload confirmation"); + // Wait for the actual upload to complete + return new Promise((resolve, reject) => { + let settled = false; + let lastLoggedPercent = 0; + // Timeout: 10 minutes per GB, minimum 10 minutes + const timeoutMs = Math.max(10 * 60_000, (fileSizeMB / 1024) * 10 * 60_000); + const timer = setTimeout(() => { + if (!settled) { + settled = true; + cleanup(); + reject(new Error(`Upload timed out after ${Math.round(timeoutMs / 60_000)}min for ${fileName}`)); + } + }, timeoutMs); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const handleUpdate = (update) => { + // Track upload progress via updateFile events + if (update?._ === "updateFile") { + const file = update.file; + if (file?.remote?.is_uploading_active && file.expected_size > 0) { + const uploaded = file.remote.uploaded_size ?? 0; + const total = file.expected_size; + const percent = Math.round((uploaded / total) * 100); + if (percent >= lastLoggedPercent + 20) { + lastLoggedPercent = percent - (percent % 20); + log.info({ fileName, uploaded, total, percent: `${percent}%` }, "Upload progress"); + } + } + } + // The money event: upload succeeded, we get the final server message ID + if (update?._ === "updateMessageSendSucceeded") { + const msg = update.message; + const oldMsgId = update.old_message_id; + if (oldMsgId === tempMsgId) { + if (!settled) { + settled = true; + cleanup(); + const finalId = BigInt(msg.id); + log.info({ fileName, tempMsgId, finalMsgId: Number(finalId) }, "Upload confirmed by Telegram"); + resolve(finalId); + } + } + } + // Upload failed + if (update?._ === "updateMessageSendFailed") { + const oldMsgId = update.old_message_id; + if (oldMsgId === tempMsgId) { + if (!settled) { + settled = true; + cleanup(); + const errorMsg = update.error?.message ?? "Unknown upload error"; + reject(new Error(`Upload failed for ${fileName}: ${errorMsg}`)); + } + } + } + }; + const cleanup = () => { + clearTimeout(timer); + client.off("update", handleUpdate); + }; + client.on("update", handleUpdate); + }); +} +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} +//# sourceMappingURL=channel.js.map \ No newline at end of file diff --git a/worker/dist/upload/channel.js.map b/worker/dist/upload/channel.js.map new file mode 100644 index 0000000..d2692bd --- /dev/null +++ b/worker/dist/upload/channel.js.map @@ -0,0 +1 @@ +{"version":3,"file":"channel.js","sourceRoot":"","sources":["../../src/upload/channel.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AAEnC,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,MAAM,GAAG,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC;AAMlC;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,MAAc,EACd,MAAc,EACd,SAAmB,EACnB,OAAgB;IAEhB,IAAI,cAAc,GAAkB,IAAI,CAAC;IAEzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;QAC9B,MAAM,WAAW,GACf,CAAC,KAAK,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;QAE3C,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACzC,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,CAAC;YACH,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC/B,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC;QAClD,CAAC;QAAC,MAAM,CAAC;YACP,eAAe;QACjB,CAAC;QAED,GAAG,CAAC,IAAI,CACN,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,SAAS,CAAC,MAAM,EAAE,EAC9F,2BAA2B,CAC5B,CAAC;QAEF,MAAM,WAAW,GAAG,MAAM,oBAAoB,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;QAE5G,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YACZ,cAAc,GAAG,WAAW,CAAC;QAC/B,CAAC;QAED,mCAAmC;QACnC,IAAI,CAAC,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7B,MAAM,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;IAED,IAAI,cAAc,KAAK,IAAI,EAAE,CAAC;QAC5B,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;IACrD,CAAC;IAED,GAAG,CAAC,IAAI,CACN,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,cAAc,CAAC,EAAE,KAAK,EAAE,SAAS,CAAC,MAAM,EAAE,EACtF,mCAAmC,CACpC,CAAC;IAEF,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,CAAC;AACvC,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,oBAAoB,CACjC,MAAc,EACd,MAAc,EACd,QAAgB,EAChB,OAA2B,EAC3B,QAAgB,EAChB,UAAkB;IAElB,kEAAkE;IAClE,MAAM,OAAO,GAAG,CAAC,MAAM,MAAM,CAAC,MAAM,CAAC;QACnC,CAAC,EAAE,aAAa;QAChB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC;QACvB,qBAAqB,EAAE;YACrB,CAAC,EAAE,sBAAsB;YACzB,QAAQ,EAAE;gBACR,CAAC,EAAE,gBAAgB;gBACnB,IAAI,EAAE,QAAQ;aACf;YACD,OAAO,EAAE,OAAO;gBACd,CAAC,CAAC;oBACE,CAAC,EAAE,eAAe;oBAClB,IAAI,EAAE,OAAO;iBACd;gBACH,CAAC,CAAC,SAAS;SACd;KACF,CAAC,CAAmB,CAAC;IAEtB,MAAM,SAAS,GAAG,OAAO,CAAC,EAAE,CAAC;IAE7B,GAAG,CAAC,KAAK,CACP,EAAE,QAAQ,EAAE,SAAS,EAAE,EACvB,iDAAiD,CAClD,CAAC;IAEF,yCAAyC;IACzC,OAAO,IAAI,OAAO,CAAS,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC7C,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,IAAI,iBAAiB,GAAG,CAAC,CAAC;QAE1B,iDAAiD;QACjD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CACxB,EAAE,GAAG,MAAM,EACX,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAClC,CAAC;QAEF,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,OAAO,GAAG,IAAI,CAAC;gBACf,OAAO,EAAE,CAAC;gBACV,MAAM,CACJ,IAAI,KAAK,CACP,0BAA0B,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,MAAM,CAAC,WAAW,QAAQ,EAAE,CAC9E,CACF,CAAC;YACJ,CAAC;QACH,CAAC,EAAE,SAAS,CAAC,CAAC;QAEd,8DAA8D;QAC9D,MAAM,YAAY,GAAG,CAAC,MAAW,EAAE,EAAE;YACnC,8CAA8C;YAC9C,IAAI,MAAM,EAAE,CAAC,KAAK,YAAY,EAAE,CAAC;gBAC/B,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;gBACzB,IAAI,IAAI,EAAE,MAAM,EAAE,mBAAmB,IAAI,IAAI,CAAC,aAAa,GAAG,CAAC,EAAE,CAAC;oBAChE,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa,IAAI,CAAC,CAAC;oBAChD,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC;oBACjC,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC;oBACrD,IAAI,OAAO,IAAI,iBAAiB,GAAG,EAAE,EAAE,CAAC;wBACtC,iBAAiB,GAAG,OAAO,GAAG,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC;wBAC7C,GAAG,CAAC,IAAI,CACN,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,OAAO,GAAG,EAAE,EACrD,iBAAiB,CAClB,CAAC;oBACJ,CAAC;gBACH,CAAC;YACH,CAAC;YAED,wEAAwE;YACxE,IAAI,MAAM,EAAE,CAAC,KAAK,4BAA4B,EAAE,CAAC;gBAC/C,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC;gBAC3B,MAAM,QAAQ,GAAG,MAAM,CAAC,cAAc,CAAC;gBACvC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;oBAC3B,IAAI,CAAC,OAAO,EAAE,CAAC;wBACb,OAAO,GAAG,IAAI,CAAC;wBACf,OAAO,EAAE,CAAC;wBACV,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;wBAC/B,GAAG,CAAC,IAAI,CACN,EAAE,QAAQ,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,CAAC,OAAO,CAAC,EAAE,EACpD,8BAA8B,CAC/B,CAAC;wBACF,OAAO,CAAC,OAAO,CAAC,CAAC;oBACnB,CAAC;gBACH,CAAC;YACH,CAAC;YAED,gBAAgB;YAChB,IAAI,MAAM,EAAE,CAAC,KAAK,yBAAyB,EAAE,CAAC;gBAC5C,MAAM,QAAQ,GAAG,MAAM,CAAC,cAAc,CAAC;gBACvC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;oBAC3B,IAAI,CAAC,OAAO,EAAE,CAAC;wBACb,OAAO,GAAG,IAAI,CAAC;wBACf,OAAO,EAAE,CAAC;wBACV,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,EAAE,OAAO,IAAI,sBAAsB,CAAC;wBACjE,MAAM,CAAC,IAAI,KAAK,CAAC,qBAAqB,QAAQ,KAAK,QAAQ,EAAE,CAAC,CAAC,CAAC;oBAClE,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC,CAAC;QAEF,MAAM,OAAO,GAAG,GAAG,EAAE;YACnB,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;QACrC,CAAC,CAAC;QAEF,MAAM,CAAC,EAAE,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IACpC,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC"} \ No newline at end of file diff --git a/worker/dist/util/config.d.ts b/worker/dist/util/config.d.ts new file mode 100644 index 0000000..c83bdf9 --- /dev/null +++ b/worker/dist/util/config.d.ts @@ -0,0 +1,18 @@ +export declare const config: { + readonly databaseUrl: string; + readonly workerIntervalMinutes: number; + readonly tempDir: string; + readonly tdlibStateDir: string; + readonly maxZipSizeMB: number; + readonly logLevel: "debug" | "info" | "warn" | "error"; + readonly telegramApiId: number; + readonly telegramApiHash: string; + /** Maximum jitter added to scheduler interval (in minutes) */ + readonly jitterMinutes: 5; + /** Maximum time span for multipart archive parts (in hours). 0 = no limit. */ + readonly multipartTimeoutHours: number; + /** Delay between Telegram API calls (in ms) to avoid rate limits */ + readonly apiDelayMs: 1000; + /** Max retries for rate-limited requests */ + readonly maxRetries: 5; +}; diff --git a/worker/dist/util/config.js b/worker/dist/util/config.js new file mode 100644 index 0000000..2719058 --- /dev/null +++ b/worker/dist/util/config.js @@ -0,0 +1,19 @@ +export const config = { + databaseUrl: process.env.DATABASE_URL ?? "", + workerIntervalMinutes: parseInt(process.env.WORKER_INTERVAL_MINUTES ?? "60", 10), + tempDir: process.env.WORKER_TEMP_DIR ?? "/tmp/zips", + tdlibStateDir: process.env.TDLIB_STATE_DIR ?? "/data/tdlib", + maxZipSizeMB: parseInt(process.env.WORKER_MAX_ZIP_SIZE_MB ?? "4096", 10), + logLevel: (process.env.LOG_LEVEL ?? "info"), + telegramApiId: parseInt(process.env.TELEGRAM_API_ID ?? "0", 10), + telegramApiHash: process.env.TELEGRAM_API_HASH ?? "", + /** Maximum jitter added to scheduler interval (in minutes) */ + jitterMinutes: 5, + /** Maximum time span for multipart archive parts (in hours). 0 = no limit. */ + multipartTimeoutHours: parseInt(process.env.MULTIPART_TIMEOUT_HOURS ?? "0", 10), + /** Delay between Telegram API calls (in ms) to avoid rate limits */ + apiDelayMs: 1000, + /** Max retries for rate-limited requests */ + maxRetries: 5, +}; +//# sourceMappingURL=config.js.map \ No newline at end of file diff --git a/worker/dist/util/config.js.map b/worker/dist/util/config.js.map new file mode 100644 index 0000000..d14cfed --- /dev/null +++ b/worker/dist/util/config.js.map @@ -0,0 +1 @@ +{"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/util/config.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,MAAM,GAAG;IACpB,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,EAAE;IAC3C,qBAAqB,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB,IAAI,IAAI,EAAE,EAAE,CAAC;IAChF,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,WAAW;IACnD,aAAa,EAAE,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,aAAa;IAC3D,YAAY,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,sBAAsB,IAAI,MAAM,EAAE,EAAE,CAAC;IACxE,QAAQ,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,MAAM,CAAwC;IAClF,aAAa,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,GAAG,EAAE,EAAE,CAAC;IAC/D,eAAe,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,EAAE;IACpD,8DAA8D;IAC9D,aAAa,EAAE,CAAC;IAChB,8EAA8E;IAC9E,qBAAqB,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB,IAAI,GAAG,EAAE,EAAE,CAAC;IAC/E,oEAAoE;IACpE,UAAU,EAAE,IAAI;IAChB,4CAA4C;IAC5C,UAAU,EAAE,CAAC;CACL,CAAC"} \ No newline at end of file diff --git a/worker/dist/util/logger.d.ts b/worker/dist/util/logger.d.ts new file mode 100644 index 0000000..17f45d7 --- /dev/null +++ b/worker/dist/util/logger.d.ts @@ -0,0 +1,3 @@ +import pino from "pino"; +export declare const logger: pino.Logger; +export declare function childLogger(name: string, extra?: Record): pino.Logger; diff --git a/worker/dist/util/logger.js b/worker/dist/util/logger.js new file mode 100644 index 0000000..c593d46 --- /dev/null +++ b/worker/dist/util/logger.js @@ -0,0 +1,12 @@ +import pino from "pino"; +import { config } from "./config.js"; +export const logger = pino({ + level: config.logLevel, + transport: config.logLevel === "debug" + ? { target: "pino/file", options: { destination: 1 } } + : undefined, +}); +export function childLogger(name, extra) { + return logger.child({ module: name, ...extra }); +} +//# sourceMappingURL=logger.js.map \ No newline at end of file diff --git a/worker/dist/util/logger.js.map b/worker/dist/util/logger.js.map new file mode 100644 index 0000000..8d0066f --- /dev/null +++ b/worker/dist/util/logger.js.map @@ -0,0 +1 @@ +{"version":3,"file":"logger.js","sourceRoot":"","sources":["../../src/util/logger.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC,MAAM,CAAC,MAAM,MAAM,GAAG,IAAI,CAAC;IACzB,KAAK,EAAE,MAAM,CAAC,QAAQ;IACtB,SAAS,EACP,MAAM,CAAC,QAAQ,KAAK,OAAO;QACzB,CAAC,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,EAAE;QACtD,CAAC,CAAC,SAAS;CAChB,CAAC,CAAC;AAEH,MAAM,UAAU,WAAW,CAAC,IAAY,EAAE,KAA+B;IACvE,OAAO,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,KAAK,EAAE,CAAC,CAAC;AAClD,CAAC"} \ No newline at end of file diff --git a/worker/dist/util/mutex.d.ts b/worker/dist/util/mutex.d.ts new file mode 100644 index 0000000..24fabf6 --- /dev/null +++ b/worker/dist/util/mutex.d.ts @@ -0,0 +1,8 @@ +/** + * Ensures only one TDLib client runs at a time across the entire worker process. + * Both the scheduler (auth, ingestion) and the fetch listener acquire this + * before creating any TDLib client. + * + * Includes a wait timeout to prevent indefinite blocking if the current holder hangs. + */ +export declare function withTdlibMutex(label: string, fn: () => Promise): Promise; diff --git a/worker/dist/util/mutex.js b/worker/dist/util/mutex.js new file mode 100644 index 0000000..e72c5dd --- /dev/null +++ b/worker/dist/util/mutex.js @@ -0,0 +1,61 @@ +import { childLogger } from "./logger.js"; +const log = childLogger("mutex"); +let locked = false; +let holder = ""; +const queue = []; +/** + * Maximum time to wait for the TDLib mutex (ms). + * If the mutex is not available within this time, the operation is rejected. + * Default: 30 minutes (long enough for large downloads, short enough to detect hangs). + */ +const MUTEX_WAIT_TIMEOUT_MS = 30 * 60 * 1000; +/** + * Ensures only one TDLib client runs at a time across the entire worker process. + * Both the scheduler (auth, ingestion) and the fetch listener acquire this + * before creating any TDLib client. + * + * Includes a wait timeout to prevent indefinite blocking if the current holder hangs. + */ +export async function withTdlibMutex(label, fn) { + if (locked) { + log.info({ waiting: label, holder }, "Waiting for TDLib mutex"); + await new Promise((resolve, reject) => { + const entry = { resolve, reject, label }; + queue.push(entry); + // Timeout: reject if we've been waiting too long + const timer = setTimeout(() => { + const idx = queue.indexOf(entry); + if (idx !== -1) { + queue.splice(idx, 1); + reject(new Error(`TDLib mutex wait timeout after ${MUTEX_WAIT_TIMEOUT_MS / 60_000}min ` + + `(waiting: ${label}, holder: ${holder})`)); + } + }, MUTEX_WAIT_TIMEOUT_MS); + // Wrap resolve to clear the timer + const origResolve = entry.resolve; + entry.resolve = () => { + clearTimeout(timer); + origResolve(); + }; + }); + } + locked = true; + holder = label; + log.debug({ label }, "TDLib mutex acquired"); + try { + return await fn(); + } + finally { + locked = false; + holder = ""; + const next = queue.shift(); + if (next) { + log.debug({ next: next.label }, "TDLib mutex releasing to next waiter"); + next.resolve(); + } + else { + log.debug({ label }, "TDLib mutex released"); + } + } +} +//# sourceMappingURL=mutex.js.map \ No newline at end of file diff --git a/worker/dist/util/mutex.js.map b/worker/dist/util/mutex.js.map new file mode 100644 index 0000000..db05e69 --- /dev/null +++ b/worker/dist/util/mutex.js.map @@ -0,0 +1 @@ +{"version":3,"file":"mutex.js","sourceRoot":"","sources":["../../src/util/mutex.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAE1C,MAAM,GAAG,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;AAEjC,IAAI,MAAM,GAAG,KAAK,CAAC;AACnB,IAAI,MAAM,GAAG,EAAE,CAAC;AAChB,MAAM,KAAK,GAAgF,EAAE,CAAC;AAE9F;;;;GAIG;AACH,MAAM,qBAAqB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAE7C;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,KAAa,EACb,EAAoB;IAEpB,IAAI,MAAM,EAAE,CAAC;QACX,GAAG,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,yBAAyB,CAAC,CAAC;QAChE,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC1C,MAAM,KAAK,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;YACzC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAElB,iDAAiD;YACjD,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;gBAC5B,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;gBACjC,IAAI,GAAG,KAAK,CAAC,CAAC,EAAE,CAAC;oBACf,KAAK,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;oBACrB,MAAM,CAAC,IAAI,KAAK,CACd,kCAAkC,qBAAqB,GAAG,MAAM,MAAM;wBACtE,aAAa,KAAK,aAAa,MAAM,GAAG,CACzC,CAAC,CAAC;gBACL,CAAC;YACH,CAAC,EAAE,qBAAqB,CAAC,CAAC;YAE1B,kCAAkC;YAClC,MAAM,WAAW,GAAG,KAAK,CAAC,OAAO,CAAC;YAClC,KAAK,CAAC,OAAO,GAAG,GAAG,EAAE;gBACnB,YAAY,CAAC,KAAK,CAAC,CAAC;gBACpB,WAAW,EAAE,CAAC;YAChB,CAAC,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IAED,MAAM,GAAG,IAAI,CAAC;IACd,MAAM,GAAG,KAAK,CAAC;IACf,GAAG,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,EAAE,sBAAsB,CAAC,CAAC;IAE7C,IAAI,CAAC;QACH,OAAO,MAAM,EAAE,EAAE,CAAC;IACpB,CAAC;YAAS,CAAC;QACT,MAAM,GAAG,KAAK,CAAC;QACf,MAAM,GAAG,EAAE,CAAC;QACZ,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;QAC3B,IAAI,IAAI,EAAE,CAAC;YACT,GAAG,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,KAAK,EAAE,EAAE,sCAAsC,CAAC,CAAC;YACxE,IAAI,CAAC,OAAO,EAAE,CAAC;QACjB,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,EAAE,sBAAsB,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;AACH,CAAC"} \ No newline at end of file diff --git a/worker/dist/worker.d.ts b/worker/dist/worker.d.ts new file mode 100644 index 0000000..2185fa9 --- /dev/null +++ b/worker/dist/worker.d.ts @@ -0,0 +1,28 @@ +import type { TelegramAccount } from "@prisma/client"; +/** + * Authenticate a PENDING account by creating a TDLib client. + * TDLib will send an SMS code to the phone number, and the client.login() + * callbacks set the authState to AWAITING_CODE. Once the admin enters the + * code via the UI, pollForAuthCode picks it up and completes the login. + * + * After successful auth: + * 1. Fetches channels from Telegram and writes as a ChannelFetchRequest + * (so the admin can select sources in the UI) + * 2. Auto-joins the destination group if an invite link is configured + */ +export declare function authenticateAccount(account: TelegramAccount): Promise; +/** + * Process a ChannelFetchRequest: fetch channels from Telegram, + * enrich with DB state, and write the result JSON. + * Called by the fetch listener (pg_notify) and by authenticateAccount. + */ +export declare function processFetchRequest(requestId: string): Promise; +/** + * Run a full ingestion cycle for a single Telegram account. + * Every step writes live activity to the DB so the admin UI can display it. + */ +export declare function runWorkerForAccount(account: TelegramAccount): Promise; +/** + * Clean up any leftover temp files/directories from previous runs. + */ +export declare function cleanupTempDir(): Promise; diff --git a/worker/dist/worker.js b/worker/dist/worker.js new file mode 100644 index 0000000..c6f336e --- /dev/null +++ b/worker/dist/worker.js @@ -0,0 +1,745 @@ +import path from "path"; +import { unlink, readdir, mkdir, rm } from "fs/promises"; +import { config } from "./util/config.js"; +import { childLogger } from "./util/logger.js"; +import { tryAcquireLock, releaseLock } from "./db/locks.js"; +import { getSourceChannelMappings, getGlobalDestinationChannel, packageExistsByHash, packageExistsBySourceMessage, createPackageWithFiles, createIngestionRun, completeIngestionRun, failIngestionRun, updateLastProcessedMessage, updateRunActivity, setChannelForum, getTopicProgress, upsertTopicProgress, upsertChannel, ensureAccountChannelLink, getGlobalSetting, getChannelFetchRequest, updateFetchRequestStatus, getAccountLinkedChannelIds, getExistingChannelsByTelegramId, deleteOrphanedPackageByHash, } from "./db/queries.js"; +import { createTdlibClient, closeTdlibClient } from "./tdlib/client.js"; +import { getAccountChats, joinChatByInviteLink } from "./tdlib/chats.js"; +import { getChannelMessages, downloadFile, downloadPhotoThumbnail } from "./tdlib/download.js"; +import { isChatForum, getForumTopicList, getTopicMessages } from "./tdlib/topics.js"; +import { matchPreviewToArchive } from "./preview/match.js"; +import { groupArchiveSets } from "./archive/multipart.js"; +import { extractCreatorFromFileName } from "./archive/creator.js"; +import { hashParts } from "./archive/hash.js"; +import { readZipCentralDirectory } from "./archive/zip-reader.js"; +import { readRarContents } from "./archive/rar-reader.js"; +import { byteLevelSplit, concatenateFiles } from "./archive/split.js"; +import { uploadToChannel } from "./upload/channel.js"; +const log = childLogger("worker"); +/** + * Authenticate a PENDING account by creating a TDLib client. + * TDLib will send an SMS code to the phone number, and the client.login() + * callbacks set the authState to AWAITING_CODE. Once the admin enters the + * code via the UI, pollForAuthCode picks it up and completes the login. + * + * After successful auth: + * 1. Fetches channels from Telegram and writes as a ChannelFetchRequest + * (so the admin can select sources in the UI) + * 2. Auto-joins the destination group if an invite link is configured + */ +export async function authenticateAccount(account) { + const aLog = childLogger("auth", { accountId: account.id, phone: account.phone }); + aLog.info("Starting authentication flow"); + let client; + try { + client = await createTdlibClient({ + id: account.id, + phone: account.phone, + }); + aLog.info("Authentication successful"); + // Auto-fetch channels and create a fetch request result + aLog.info("Fetching channels from Telegram..."); + await createAutoFetchRequest(client, account.id, aLog); + // Auto-join the destination group if an invite link exists + const inviteLink = await getGlobalSetting("destination_invite_link"); + if (inviteLink) { + aLog.info("Attempting to join destination group via invite link..."); + try { + await joinChatByInviteLink(client, inviteLink); + // Link this account as WRITER to the destination channel + const destChannel = await getGlobalDestinationChannel(); + if (destChannel) { + await ensureAccountChannelLink(account.id, destChannel.id, "WRITER"); + aLog.info({ destChannel: destChannel.title }, "Joined destination group and linked as WRITER"); + } + } + catch (err) { + // May already be a member — that's fine + aLog.warn({ err }, "Could not join destination group (may already be a member)"); + // Still try to link as WRITER + const destChannel = await getGlobalDestinationChannel(); + if (destChannel) { + await ensureAccountChannelLink(account.id, destChannel.id, "WRITER"); + } + } + } + } + catch (err) { + aLog.error({ err }, "Authentication failed"); + } + finally { + if (client) { + await closeTdlibClient(client); + } + } +} +/** + * Process a ChannelFetchRequest: fetch channels from Telegram, + * enrich with DB state, and write the result JSON. + * Called by the fetch listener (pg_notify) and by authenticateAccount. + */ +export async function processFetchRequest(requestId) { + const aLog = childLogger("fetch-request", { requestId }); + const request = await getChannelFetchRequest(requestId); + if (!request || request.status !== "PENDING") { + aLog.warn("Fetch request not found or not pending, skipping"); + return; + } + await updateFetchRequestStatus(requestId, "IN_PROGRESS"); + aLog.info({ accountId: request.accountId }, "Processing fetch request"); + const client = await createTdlibClient({ + id: request.account.id, + phone: request.account.phone, + }); + try { + const chats = await getAccountChats(client); + // Enrich with DB state + const linkedTelegramIds = await getAccountLinkedChannelIds(request.accountId); + const existingChannels = await getExistingChannelsByTelegramId(); + const enrichedChats = chats.map((chat) => { + const telegramIdStr = chat.chatId.toString(); + return { + chatId: telegramIdStr, + title: chat.title, + type: chat.type, + isForum: chat.isForum, + memberCount: chat.memberCount ?? null, + alreadyLinked: linkedTelegramIds.has(telegramIdStr), + existingChannelId: existingChannels.get(telegramIdStr) ?? null, + }; + }); + // Also upsert channel metadata while we have the data + for (const chat of chats) { + try { + await upsertChannel({ + telegramId: chat.chatId, + title: chat.title, + type: "SOURCE", + isForum: chat.isForum, + }); + } + catch { + // Non-critical — metadata sync can fail silently + } + } + await updateFetchRequestStatus(requestId, "COMPLETED", { + resultJson: JSON.stringify(enrichedChats), + }); + aLog.info({ total: chats.length, linked: [...linkedTelegramIds].length }, "Fetch request completed"); + } + catch (err) { + const message = err instanceof Error ? err.message : String(err); + aLog.error({ err }, "Fetch request failed"); + await updateFetchRequestStatus(requestId, "FAILED", { error: message }); + } + finally { + await closeTdlibClient(client); + } +} +/** + * Internal helper called after authentication to auto-create a fetch request + * with the channel list (so the UI can show the picker immediately). + */ +async function createAutoFetchRequest(client, accountId, aLog) { + const chats = await getAccountChats(client); + const linkedTelegramIds = await getAccountLinkedChannelIds(accountId); + const existingChannels = await getExistingChannelsByTelegramId(); + const enrichedChats = chats.map((chat) => { + const telegramIdStr = chat.chatId.toString(); + return { + chatId: telegramIdStr, + title: chat.title, + type: chat.type, + isForum: chat.isForum, + memberCount: chat.memberCount ?? null, + alreadyLinked: linkedTelegramIds.has(telegramIdStr), + existingChannelId: existingChannels.get(telegramIdStr) ?? null, + }; + }); + // Upsert channel metadata + for (const chat of chats) { + try { + await upsertChannel({ + telegramId: chat.chatId, + title: chat.title, + type: "SOURCE", + isForum: chat.isForum, + }); + } + catch { + // Non-critical + } + } + // Create the fetch request record with the result already filled in + const { db } = await import("./db/client.js"); + await db.channelFetchRequest.create({ + data: { + accountId, + status: "COMPLETED", + resultJson: JSON.stringify(enrichedChats), + }, + }); + aLog.info({ total: chats.length }, "Auto-fetch request created with channel list"); +} +/** + * Throttle DB writes for download progress to avoid hammering the DB. + * Only writes if at least 2 seconds have passed since the last write. + */ +function createThrottledActivityUpdater(runId, minIntervalMs = 2000) { + let lastWriteTime = 0; + let pendingUpdate = null; + let flushTimer = null; + const flush = async () => { + if (pendingUpdate) { + const update = pendingUpdate; + pendingUpdate = null; + lastWriteTime = Date.now(); + await updateRunActivity(runId, update).catch(() => { }); + } + }; + return { + update: (activity) => { + pendingUpdate = activity; + const elapsed = Date.now() - lastWriteTime; + if (elapsed >= minIntervalMs) { + if (flushTimer) + clearTimeout(flushTimer); + flush(); + } + else if (!flushTimer) { + flushTimer = setTimeout(() => { + flushTimer = null; + flush(); + }, minIntervalMs - elapsed); + } + }, + flush, + }; +} +/** + * Run a full ingestion cycle for a single Telegram account. + * Every step writes live activity to the DB so the admin UI can display it. + */ +export async function runWorkerForAccount(account) { + const accountLog = childLogger("worker", { accountId: account.id, phone: account.phone }); + // 1. Acquire advisory lock + const acquired = await tryAcquireLock(account.id); + if (!acquired) { + accountLog.info("Account already locked, skipping"); + return; + } + let runId; + try { + // 2. Create ingestion run + const run = await createIngestionRun(account.id); + runId = run.id; + const activeRunId = runId; + accountLog.info({ runId }, "Ingestion run started"); + const throttled = createThrottledActivityUpdater(activeRunId); + // 3. Initialize TDLib client + await updateRunActivity(activeRunId, { + currentActivity: "Connecting to Telegram", + currentStep: "connecting", + }); + const client = await createTdlibClient({ + id: account.id, + phone: account.phone, + }); + const counters = { + messagesScanned: 0, + zipsFound: 0, + zipsDuplicate: 0, + zipsIngested: 0, + }; + try { + // 4. Get assigned source channels and global destination + const channelMappings = await getSourceChannelMappings(account.id); + const destChannel = await getGlobalDestinationChannel(); + if (!destChannel) { + throw new Error("No global destination channel configured — set one in the admin UI"); + } + const totalChannels = channelMappings.length; + for (let chIdx = 0; chIdx < channelMappings.length; chIdx++) { + const mapping = channelMappings[chIdx]; + const channel = mapping.channel; + const channelLabel = totalChannels > 1 + ? `[${chIdx + 1}/${totalChannels}] ${channel.title}` + : channel.title; + try { + // ── Check if channel is a forum ── + const forum = await isChatForum(client, channel.telegramId); + if (forum !== channel.isForum) { + await setChannelForum(channel.id, forum); + accountLog.info({ channelId: channel.id, title: channel.title, isForum: forum }, "Updated channel forum status"); + } + const pipelineCtx = { + client, + runId: activeRunId, + channelTitle: channel.title, + channel, + destChannelTelegramId: destChannel.telegramId, + destChannelId: destChannel.id, + throttled, + counters, + topicCreator: null, + sourceTopicId: null, + accountLog, + }; + if (forum) { + // ── Forum channel: scan per-topic ── + await updateRunActivity(activeRunId, { + currentActivity: `Enumerating topics in "${channelLabel}"`, + currentStep: "scanning", + currentChannel: channelLabel, + currentFile: null, + currentFileNum: null, + totalFiles: null, + downloadedBytes: null, + totalBytes: null, + downloadPercent: null, + messagesScanned: counters.messagesScanned, + }); + const topics = await getForumTopicList(client, channel.telegramId); + const topicProgressList = await getTopicProgress(mapping.id); + accountLog.info({ channelId: channel.id, title: channel.title, topicCount: topics.length }, "Scanning forum channel by topic"); + for (let tIdx = 0; tIdx < topics.length; tIdx++) { + const topic = topics[tIdx]; + try { + const progress = topicProgressList.find((tp) => tp.topicId === topic.topicId); + const topicLabel = `${channel.title} › ${topic.name}`; + const topicProgress = topics.length > 1 + ? ` (topic ${tIdx + 1}/${topics.length})` + : ""; + await updateRunActivity(activeRunId, { + currentActivity: `Scanning "${topicLabel}"${topicProgress}`, + currentStep: "scanning", + currentChannel: channelLabel, + currentFile: null, + currentFileNum: null, + totalFiles: null, + downloadedBytes: null, + totalBytes: null, + downloadPercent: null, + messagesScanned: counters.messagesScanned, + }); + const scanResult = await getTopicMessages(client, channel.telegramId, topic.topicId, progress?.lastProcessedMessageId, 100, (scanned) => { + throttled.update({ + currentActivity: `Scanning "${topicLabel}"${topicProgress} — ${scanned} messages scanned`, + currentStep: "scanning", + currentChannel: channelLabel, + messagesScanned: counters.messagesScanned + scanned, + }); + }); + // Add scanned messages to global counter + counters.messagesScanned += scanResult.totalScanned; + if (scanResult.archives.length === 0) { + accountLog.debug({ channelId: channel.id, topic: topic.name }, "No new archives in topic"); + continue; + } + accountLog.info({ topic: topic.name, archives: scanResult.archives.length, photos: scanResult.photos.length }, "Found messages in topic"); + // Process archives with topic creator + pipelineCtx.topicCreator = topic.name; + pipelineCtx.sourceTopicId = topic.topicId; + pipelineCtx.channelTitle = `${channel.title} › ${topic.name}`; + const maxProcessedId = await processArchiveSets(pipelineCtx, scanResult, run.id, progress?.lastProcessedMessageId); + // Only advance progress to the highest successfully processed message + if (maxProcessedId) { + await upsertTopicProgress(mapping.id, topic.topicId, topic.name, maxProcessedId); + } + } + catch (topicErr) { + accountLog.warn({ err: topicErr, channelId: channel.id, topic: topic.name, topicId: topic.topicId.toString() }, "Failed to process topic, skipping"); + } + } + } + else { + // ── Non-forum channel: flat scan (existing behavior) ── + await updateRunActivity(activeRunId, { + currentActivity: `Scanning "${channelLabel}" for new archives`, + currentStep: "scanning", + currentChannel: channelLabel, + currentFile: null, + currentFileNum: null, + totalFiles: null, + downloadedBytes: null, + totalBytes: null, + downloadPercent: null, + messagesScanned: counters.messagesScanned, + }); + accountLog.info({ channelId: channel.id, title: channel.title }, "Processing source channel"); + const scanResult = await getChannelMessages(client, channel.telegramId, mapping.lastProcessedMessageId, 100, (scanned) => { + throttled.update({ + currentActivity: `Scanning "${channelLabel}" — ${scanned} messages scanned`, + currentStep: "scanning", + currentChannel: channelLabel, + messagesScanned: counters.messagesScanned + scanned, + }); + }); + // Add scanned messages to global counter + counters.messagesScanned += scanResult.totalScanned; + if (scanResult.archives.length === 0) { + accountLog.debug({ channelId: channel.id }, "No new archives"); + continue; + } + accountLog.info({ archives: scanResult.archives.length, photos: scanResult.photos.length }, "Found messages in channel"); + // For non-forum, creator comes from filename (set to null, resolved per-archive) + pipelineCtx.topicCreator = null; + pipelineCtx.sourceTopicId = null; + pipelineCtx.channelTitle = channel.title; + const maxProcessedId = await processArchiveSets(pipelineCtx, scanResult, run.id, mapping.lastProcessedMessageId); + // Only advance progress to the highest successfully processed message + if (maxProcessedId) { + await updateLastProcessedMessage(mapping.id, maxProcessedId); + } + } + } + catch (channelErr) { + accountLog.warn({ err: channelErr, channelId: channel.id, title: channel.title }, "Failed to process channel, skipping to next"); + } + } + // ── Done ── + await completeIngestionRun(activeRunId, counters); + accountLog.info({ counters }, "Ingestion run completed"); + } + finally { + await closeTdlibClient(client); + } + } + catch (err) { + const message = err instanceof Error ? err.message : String(err); + accountLog.error({ err }, "Ingestion run failed"); + if (runId) { + await failIngestionRun(runId, message).catch((e) => accountLog.error({ e }, "Failed to mark run as failed")); + } + } + finally { + await releaseLock(account.id); + } +} +/** + * Process a scan result through the archive pipeline: + * group → download → hash → dedup → metadata → split → upload → preview → index. + * + * Returns the highest message ID that was successfully processed (ingested or + * confirmed duplicate). The caller should only advance the progress boundary + * to this value — never to the max of all scanned messages. + */ +async function processArchiveSets(ctx, scanResult, ingestionRunId, lastProcessedMessageId) { + const { client, runId, channelTitle, channel, throttled, counters, accountLog } = ctx; + // Group into archive sets + let archiveSets = groupArchiveSets(scanResult.archives); + // Filter out sets where ALL parts are at or below the boundary (already processed) + if (lastProcessedMessageId) { + const totalBefore = archiveSets.length; + archiveSets = archiveSets.filter((set) => set.parts.some((p) => p.id > lastProcessedMessageId)); + const filtered = totalBefore - archiveSets.length; + if (filtered > 0) { + accountLog.info({ filtered, remaining: archiveSets.length }, "Filtered out already-processed archive sets"); + } + } + counters.zipsFound += archiveSets.length; + // Match preview photos to archive sets + const previewMatches = matchPreviewToArchive(scanResult.photos, archiveSets.map((s) => ({ + baseName: s.baseName, + firstMessageId: s.parts[0].id, + firstMessageDate: s.parts[0].date, + }))); + if (previewMatches.size > 0) { + accountLog.info({ matched: previewMatches.size, total: archiveSets.length }, "Matched preview photos to archives"); + } + await updateRunActivity(runId, { + currentActivity: `Found ${archiveSets.length} archive(s) in "${channelTitle}"`, + currentStep: "scanning", + currentChannel: channelTitle, + totalFiles: archiveSets.length, + zipsFound: counters.zipsFound, + messagesScanned: counters.messagesScanned, + }); + // Track the highest message ID that was successfully processed + let maxProcessedId = null; + for (let setIdx = 0; setIdx < archiveSets.length; setIdx++) { + try { + await processOneArchiveSet(ctx, archiveSets[setIdx], setIdx, archiveSets.length, previewMatches, ingestionRunId); + // Set completed (ingested or confirmed duplicate) — advance watermark + const setMaxId = archiveSets[setIdx].parts.reduce((max, p) => (p.id > max ? p.id : max), 0n); + if (setMaxId > (maxProcessedId ?? 0n)) { + maxProcessedId = setMaxId; + } + } + catch (setErr) { + // If a set fails, do NOT advance the watermark past it + accountLog.warn({ err: setErr, baseName: archiveSets[setIdx].baseName }, "Archive set failed, watermark will not advance past this set"); + } + } + return maxProcessedId; +} +/** + * Process a single archive set through the full pipeline. + */ +async function processOneArchiveSet(ctx, archiveSet, setIdx, totalSets, previewMatches, ingestionRunId) { + const { client, runId, channelTitle, channel, destChannelTelegramId, destChannelId, throttled, counters, topicCreator, sourceTopicId, accountLog, } = ctx; + const archiveName = archiveSet.parts[0].fileName; + // ── Early skip: check if this archive set was already ingested ── + // This avoids re-downloading large archives that were processed in a prior run. + const alreadyIngested = await packageExistsBySourceMessage(channel.id, archiveSet.parts[0].id); + if (alreadyIngested) { + counters.zipsDuplicate++; + accountLog.debug({ fileName: archiveName, sourceMessageId: Number(archiveSet.parts[0].id) }, "Archive already ingested (by source message), skipping"); + await updateRunActivity(runId, { + currentActivity: `Skipped ${archiveName} (already ingested)`, + currentStep: "deduplicating", + currentChannel: channelTitle, + currentFile: archiveName, + currentFileNum: setIdx + 1, + totalFiles: totalSets, + zipsDuplicate: counters.zipsDuplicate, + }); + return; + } + const tempPaths = []; + let splitPaths = []; + // Per-set subdirectory so uploaded files keep their original filenames + const setDir = path.join(config.tempDir, `${ingestionRunId}_${archiveSet.parts[0].id}`); + await mkdir(setDir, { recursive: true }); + try { + // ── Downloading ── + for (let partIdx = 0; partIdx < archiveSet.parts.length; partIdx++) { + const part = archiveSet.parts[partIdx]; + const tempPath = path.join(setDir, part.fileName); + const partLabel = archiveSet.parts.length > 1 + ? ` (part ${partIdx + 1}/${archiveSet.parts.length})` + : ""; + await updateRunActivity(runId, { + currentActivity: `Downloading ${part.fileName}${partLabel}`, + currentStep: "downloading", + currentChannel: channelTitle, + currentFile: part.fileName, + currentFileNum: setIdx + 1, + totalFiles: totalSets, + downloadedBytes: 0n, + totalBytes: part.fileSize, + downloadPercent: 0, + messagesScanned: counters.messagesScanned, + }); + accountLog.info({ + fileName: part.fileName, + fileSize: Number(part.fileSize), + part: partIdx + 1, + totalParts: archiveSet.parts.length, + }, "Downloading archive part"); + await downloadFile(client, part.fileId, tempPath, part.fileSize, part.fileName, (progress) => { + throttled.update({ + currentActivity: `Downloading ${part.fileName}${partLabel} — ${progress.percent}%`, + currentStep: "downloading", + currentChannel: channelTitle, + currentFile: part.fileName, + currentFileNum: setIdx + 1, + totalFiles: totalSets, + downloadedBytes: BigInt(progress.downloadedBytes), + totalBytes: BigInt(progress.totalBytes), + downloadPercent: progress.percent, + }); + }); + await throttled.flush(); + tempPaths.push(tempPath); + } + // ── Hashing ── + await updateRunActivity(runId, { + currentActivity: `Computing hash for ${archiveName}`, + currentStep: "hashing", + currentChannel: channelTitle, + currentFile: archiveName, + currentFileNum: setIdx + 1, + totalFiles: totalSets, + downloadedBytes: null, + totalBytes: null, + downloadPercent: null, + }); + const contentHash = await hashParts(tempPaths); + // ── Deduplicating ── + await updateRunActivity(runId, { + currentActivity: `Checking if ${archiveName} is a duplicate`, + currentStep: "deduplicating", + currentChannel: channelTitle, + currentFile: archiveName, + currentFileNum: setIdx + 1, + totalFiles: totalSets, + }); + const exists = await packageExistsByHash(contentHash); + if (exists) { + counters.zipsDuplicate++; + accountLog.debug({ contentHash }, "Duplicate archive, skipping"); + await updateRunActivity(runId, { + currentActivity: `Skipped ${archiveName} (duplicate)`, + currentStep: "deduplicating", + currentChannel: channelTitle, + currentFile: archiveName, + currentFileNum: setIdx + 1, + totalFiles: totalSets, + zipsDuplicate: counters.zipsDuplicate, + }); + return; + } + // ── Reading metadata ── + await updateRunActivity(runId, { + currentActivity: `Reading file list from ${archiveName}`, + currentStep: "reading_metadata", + currentChannel: channelTitle, + currentFile: archiveName, + currentFileNum: setIdx + 1, + totalFiles: totalSets, + }); + let entries = []; + try { + if (archiveSet.type === "ZIP") { + entries = await readZipCentralDirectory(tempPaths); + } + else { + entries = await readRarContents(tempPaths[0]); + } + } + catch (err) { + accountLog.warn({ err, baseName: archiveSet.baseName }, "Failed to read archive metadata, ingesting without file list"); + } + // ── Splitting / Repacking (if needed) ── + let uploadPaths = [...tempPaths]; + const totalSize = archiveSet.parts.reduce((sum, p) => sum + p.fileSize, 0n); + const MAX_UPLOAD_SIZE = 2n * 1024n * 1024n * 1024n; + const hasOversizedPart = archiveSet.parts.some((p) => p.fileSize > MAX_UPLOAD_SIZE); + if (hasOversizedPart) { + // Full repack: concatenate all parts → single file → re-split into uniform 2GB chunks + await updateRunActivity(runId, { + currentActivity: `Repacking ${archiveName} (parts >2GB, concatenating + re-splitting)`, + currentStep: "splitting", + currentChannel: channelTitle, + currentFile: archiveName, + currentFileNum: setIdx + 1, + totalFiles: totalSets, + }); + const concatPath = path.join(setDir, `${archiveSet.baseName}.concat`); + await concatenateFiles(tempPaths, concatPath); + splitPaths = await byteLevelSplit(concatPath); + uploadPaths = splitPaths; + // Clean up the concat intermediate file + await unlink(concatPath).catch(() => { }); + } + else if (!archiveSet.isMultipart && totalSize > MAX_UPLOAD_SIZE) { + // Single file >2GB: split directly + await updateRunActivity(runId, { + currentActivity: `Splitting ${archiveName} for upload (>2GB)`, + currentStep: "splitting", + currentChannel: channelTitle, + currentFile: archiveName, + currentFileNum: setIdx + 1, + totalFiles: totalSets, + }); + splitPaths = await byteLevelSplit(tempPaths[0]); + uploadPaths = splitPaths; + } + // ── Uploading ── + const uploadLabel = uploadPaths.length > 1 + ? ` (${uploadPaths.length} parts)` + : ""; + await updateRunActivity(runId, { + currentActivity: `Uploading ${archiveName} to archive channel${uploadLabel}`, + currentStep: "uploading", + currentChannel: channelTitle, + currentFile: archiveName, + currentFileNum: setIdx + 1, + totalFiles: totalSets, + }); + const destResult = await uploadToChannel(client, destChannelTelegramId, uploadPaths); + // ── Preview thumbnail ── + let previewData = null; + let previewMsgId = null; + const matchedPhoto = previewMatches.get(archiveSet.baseName); + if (matchedPhoto) { + await updateRunActivity(runId, { + currentActivity: `Downloading preview image for ${archiveName}`, + currentStep: "preview", + currentChannel: channelTitle, + currentFile: archiveName, + currentFileNum: setIdx + 1, + totalFiles: totalSets, + }); + previewData = await downloadPhotoThumbnail(client, matchedPhoto.fileId); + previewMsgId = matchedPhoto.id; + } + // ── Resolve creator: topic name > filename extraction > null ── + const creator = topicCreator ?? extractCreatorFromFileName(archiveName) ?? null; + // ── Indexing ── + await updateRunActivity(runId, { + currentActivity: `Saving metadata for ${archiveName} (${entries.length} files)`, + currentStep: "indexing", + currentChannel: channelTitle, + currentFile: archiveName, + currentFileNum: setIdx + 1, + totalFiles: totalSets, + }); + // Clean up any orphaned record (same hash but no dest upload) before creating + await deleteOrphanedPackageByHash(contentHash); + await createPackageWithFiles({ + contentHash, + fileName: archiveName, + fileSize: totalSize, + archiveType: archiveSet.type, + sourceChannelId: channel.id, + sourceMessageId: archiveSet.parts[0].id, + sourceTopicId, + destChannelId, + destMessageId: destResult.messageId, + isMultipart: archiveSet.parts.length > 1 || uploadPaths.length > 1, + partCount: uploadPaths.length, + ingestionRunId, + creator, + previewData, + previewMsgId, + files: entries, + }); + counters.zipsIngested++; + await updateRunActivity(runId, { + currentActivity: `Ingested ${archiveName} (${entries.length} files indexed)`, + currentStep: "complete", + currentChannel: channelTitle, + currentFile: archiveName, + currentFileNum: setIdx + 1, + totalFiles: totalSets, + zipsIngested: counters.zipsIngested, + }); + accountLog.info({ fileName: archiveName, contentHash, fileCount: entries.length, creator }, "Archive ingested"); + } + finally { + // ALWAYS delete temp files and the set directory + await deleteFiles([...tempPaths, ...splitPaths]); + await rm(setDir, { recursive: true, force: true }).catch(() => { }); + } +} +async function deleteFiles(paths) { + for (const p of paths) { + try { + await unlink(p); + } + catch { + // File may already be deleted or never created + } + } +} +/** + * Clean up any leftover temp files/directories from previous runs. + */ +export async function cleanupTempDir() { + try { + const entries = await readdir(config.tempDir); + for (const entry of entries) { + await rm(path.join(config.tempDir, entry), { recursive: true, force: true }).catch(() => { }); + } + if (entries.length > 0) { + log.info({ count: entries.length }, "Cleaned up stale temp files"); + } + } + catch { + // Directory might not exist yet + } +} +//# sourceMappingURL=worker.js.map \ No newline at end of file diff --git a/worker/dist/worker.js.map b/worker/dist/worker.js.map new file mode 100644 index 0000000..663c5b5 --- /dev/null +++ b/worker/dist/worker.js.map @@ -0,0 +1 @@ +{"version":3,"file":"worker.js","sourceRoot":"","sources":["../src/worker.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,aAAa,CAAC;AACzD,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC5D,OAAO,EACL,wBAAwB,EACxB,2BAA2B,EAC3B,mBAAmB,EACnB,4BAA4B,EAC5B,sBAAsB,EACtB,kBAAkB,EAClB,oBAAoB,EACpB,gBAAgB,EAChB,0BAA0B,EAC1B,iBAAiB,EACjB,eAAe,EACf,gBAAgB,EAChB,mBAAmB,EACnB,aAAa,EACb,wBAAwB,EACxB,gBAAgB,EAChB,sBAAsB,EACtB,wBAAwB,EACxB,0BAA0B,EAC1B,+BAA+B,EAE/B,2BAA2B,GAC5B,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACxE,OAAO,EAAE,eAAe,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AACzE,OAAO,EAAE,kBAAkB,EAAE,YAAY,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAE/F,OAAO,EAAE,WAAW,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACrF,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAE1D,OAAO,EAAE,0BAA0B,EAAE,MAAM,sBAAsB,CAAC;AAClE,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EAAE,uBAAuB,EAAE,MAAM,yBAAyB,CAAC;AAClE,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAC1D,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtE,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAItD,MAAM,GAAG,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC;AAElC;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,OAAwB;IAExB,MAAM,IAAI,GAAG,WAAW,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;IAClF,IAAI,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;IAE1C,IAAI,MAA0B,CAAC;IAC/B,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,iBAAiB,CAAC;YAC/B,EAAE,EAAE,OAAO,CAAC,EAAE;YACd,KAAK,EAAE,OAAO,CAAC,KAAK;SACrB,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QAEvC,wDAAwD;QACxD,IAAI,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;QAChD,MAAM,sBAAsB,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QAEvD,2DAA2D;QAC3D,MAAM,UAAU,GAAG,MAAM,gBAAgB,CAAC,yBAAyB,CAAC,CAAC;QACrE,IAAI,UAAU,EAAE,CAAC;YACf,IAAI,CAAC,IAAI,CAAC,yDAAyD,CAAC,CAAC;YACrE,IAAI,CAAC;gBACH,MAAM,oBAAoB,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;gBAC/C,yDAAyD;gBACzD,MAAM,WAAW,GAAG,MAAM,2BAA2B,EAAE,CAAC;gBACxD,IAAI,WAAW,EAAE,CAAC;oBAChB,MAAM,wBAAwB,CAAC,OAAO,CAAC,EAAE,EAAE,WAAW,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;oBACrE,IAAI,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,WAAW,CAAC,KAAK,EAAE,EAAE,+CAA+C,CAAC,CAAC;gBACjG,CAAC;YACH,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,wCAAwC;gBACxC,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,EAAE,4DAA4D,CAAC,CAAC;gBACjF,8BAA8B;gBAC9B,MAAM,WAAW,GAAG,MAAM,2BAA2B,EAAE,CAAC;gBACxD,IAAI,WAAW,EAAE,CAAC;oBAChB,MAAM,wBAAwB,CAAC,OAAO,CAAC,EAAE,EAAE,WAAW,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;gBACvE,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,EAAE,uBAAuB,CAAC,CAAC;IAC/C,CAAC;YAAS,CAAC;QACT,IAAI,MAAM,EAAE,CAAC;YACX,MAAM,gBAAgB,CAAC,MAAM,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,SAAiB;IACzD,MAAM,IAAI,GAAG,WAAW,CAAC,eAAe,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;IACzD,MAAM,OAAO,GAAG,MAAM,sBAAsB,CAAC,SAAS,CAAC,CAAC;IAExD,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QAC7C,IAAI,CAAC,IAAI,CAAC,kDAAkD,CAAC,CAAC;QAC9D,OAAO;IACT,CAAC;IAED,MAAM,wBAAwB,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;IACzD,IAAI,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,EAAE,0BAA0B,CAAC,CAAC;IAExE,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC;QACrC,EAAE,EAAE,OAAO,CAAC,OAAO,CAAC,EAAE;QACtB,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,KAAK;KAC7B,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,CAAC;QAE5C,uBAAuB;QACvB,MAAM,iBAAiB,GAAG,MAAM,0BAA0B,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAC9E,MAAM,gBAAgB,GAAG,MAAM,+BAA+B,EAAE,CAAC;QAEjE,MAAM,aAAa,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACvC,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;YAC7C,OAAO;gBACL,MAAM,EAAE,aAAa;gBACrB,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,IAAI;gBACrC,aAAa,EAAE,iBAAiB,CAAC,GAAG,CAAC,aAAa,CAAC;gBACnD,iBAAiB,EAAE,gBAAgB,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI,IAAI;aAC/D,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,sDAAsD;QACtD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,CAAC;gBACH,MAAM,aAAa,CAAC;oBAClB,UAAU,EAAE,IAAI,CAAC,MAAM;oBACvB,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,IAAI,EAAE,QAAQ;oBACd,OAAO,EAAE,IAAI,CAAC,OAAO;iBACtB,CAAC,CAAC;YACL,CAAC;YAAC,MAAM,CAAC;gBACP,iDAAiD;YACnD,CAAC;QACH,CAAC;QAED,MAAM,wBAAwB,CAAC,SAAS,EAAE,WAAW,EAAE;YACrD,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC;SAC1C,CAAC,CAAC;QAEH,IAAI,CAAC,IAAI,CACP,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,GAAG,iBAAiB,CAAC,CAAC,MAAM,EAAE,EAC9D,yBAAyB,CAC1B,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjE,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,EAAE,sBAAsB,CAAC,CAAC;QAC5C,MAAM,wBAAwB,CAAC,SAAS,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;IAC1E,CAAC;YAAS,CAAC;QACT,MAAM,gBAAgB,CAAC,MAAM,CAAC,CAAC;IACjC,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,sBAAsB,CACnC,MAAc,EACd,SAAiB,EACjB,IAAoC;IAEpC,MAAM,KAAK,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,CAAC;IAE5C,MAAM,iBAAiB,GAAG,MAAM,0BAA0B,CAAC,SAAS,CAAC,CAAC;IACtE,MAAM,gBAAgB,GAAG,MAAM,+BAA+B,EAAE,CAAC;IAEjE,MAAM,aAAa,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACvC,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;QAC7C,OAAO;YACL,MAAM,EAAE,aAAa;YACrB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,IAAI;YACrC,aAAa,EAAE,iBAAiB,CAAC,GAAG,CAAC,aAAa,CAAC;YACnD,iBAAiB,EAAE,gBAAgB,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI,IAAI;SAC/D,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,0BAA0B;IAC1B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC;YACH,MAAM,aAAa,CAAC;gBAClB,UAAU,EAAE,IAAI,CAAC,MAAM;gBACvB,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,IAAI,CAAC,OAAO;aACtB,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,eAAe;QACjB,CAAC;IACH,CAAC;IAED,oEAAoE;IACpE,MAAM,EAAE,EAAE,EAAE,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,CAAC;IAC9C,MAAM,EAAE,CAAC,mBAAmB,CAAC,MAAM,CAAC;QAClC,IAAI,EAAE;YACJ,SAAS;YACT,MAAM,EAAE,WAAW;YACnB,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC;SAC1C;KACF,CAAC,CAAC;IAEH,IAAI,CAAC,IAAI,CACP,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,EACvB,8CAA8C,CAC/C,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,SAAS,8BAA8B,CAAC,KAAa,EAAE,aAAa,GAAG,IAAI;IACzE,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,aAAa,GAA0B,IAAI,CAAC;IAChD,IAAI,UAAU,GAAyC,IAAI,CAAC;IAE5D,MAAM,KAAK,GAAG,KAAK,IAAI,EAAE;QACvB,IAAI,aAAa,EAAE,CAAC;YAClB,MAAM,MAAM,GAAG,aAAa,CAAC;YAC7B,aAAa,GAAG,IAAI,CAAC;YACrB,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC3B,MAAM,iBAAiB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACzD,CAAC;IACH,CAAC,CAAC;IAEF,OAAO;QACL,MAAM,EAAE,CAAC,QAAwB,EAAE,EAAE;YACnC,aAAa,GAAG,QAAQ,CAAC;YACzB,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,aAAa,CAAC;YAC3C,IAAI,OAAO,IAAI,aAAa,EAAE,CAAC;gBAC7B,IAAI,UAAU;oBAAE,YAAY,CAAC,UAAU,CAAC,CAAC;gBACzC,KAAK,EAAE,CAAC;YACV,CAAC;iBAAM,IAAI,CAAC,UAAU,EAAE,CAAC;gBACvB,UAAU,GAAG,UAAU,CAAC,GAAG,EAAE;oBAC3B,UAAU,GAAG,IAAI,CAAC;oBAClB,KAAK,EAAE,CAAC;gBACV,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC;QACD,KAAK;KACN,CAAC;AACJ,CAAC;AAwBD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,OAAwB;IAExB,MAAM,UAAU,GAAG,WAAW,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;IAE1F,2BAA2B;IAC3B,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAClD,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,UAAU,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;QACpD,OAAO;IACT,CAAC;IAED,IAAI,KAAyB,CAAC;IAE9B,IAAI,CAAC;QACH,0BAA0B;QAC1B,MAAM,GAAG,GAAG,MAAM,kBAAkB,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QACjD,KAAK,GAAG,GAAG,CAAC,EAAE,CAAC;QACf,MAAM,WAAW,GAAG,KAAK,CAAC;QAC1B,UAAU,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,EAAE,uBAAuB,CAAC,CAAC;QAEpD,MAAM,SAAS,GAAG,8BAA8B,CAAC,WAAW,CAAC,CAAC;QAE9D,6BAA6B;QAC7B,MAAM,iBAAiB,CAAC,WAAW,EAAE;YACnC,eAAe,EAAE,wBAAwB;YACzC,WAAW,EAAE,YAAY;SAC1B,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC;YACrC,EAAE,EAAE,OAAO,CAAC,EAAE;YACd,KAAK,EAAE,OAAO,CAAC,KAAK;SACrB,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG;YACf,eAAe,EAAE,CAAC;YAClB,SAAS,EAAE,CAAC;YACZ,aAAa,EAAE,CAAC;YAChB,YAAY,EAAE,CAAC;SAChB,CAAC;QAEF,IAAI,CAAC;YACH,yDAAyD;YACzD,MAAM,eAAe,GAAG,MAAM,wBAAwB,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YACnE,MAAM,WAAW,GAAG,MAAM,2BAA2B,EAAE,CAAC;YAExD,IAAI,CAAC,WAAW,EAAE,CAAC;gBACjB,MAAM,IAAI,KAAK,CAAC,oEAAoE,CAAC,CAAC;YACxF,CAAC;YAED,MAAM,aAAa,GAAG,eAAe,CAAC,MAAM,CAAC;YAE7C,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,eAAe,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;gBAC5D,MAAM,OAAO,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;gBACvC,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;gBAChC,MAAM,YAAY,GAAG,aAAa,GAAG,CAAC;oBACpC,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,aAAa,KAAK,OAAO,CAAC,KAAK,EAAE;oBACpD,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;gBAElB,IAAI,CAAC;oBACL,oCAAoC;oBACpC,MAAM,KAAK,GAAG,MAAM,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;oBAC5D,IAAI,KAAK,KAAK,OAAO,CAAC,OAAO,EAAE,CAAC;wBAC9B,MAAM,eAAe,CAAC,OAAO,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;wBACzC,UAAU,CAAC,IAAI,CACb,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,EAC/D,8BAA8B,CAC/B,CAAC;oBACJ,CAAC;oBAED,MAAM,WAAW,GAAoB;wBACnC,MAAM;wBACN,KAAK,EAAE,WAAW;wBAClB,YAAY,EAAE,OAAO,CAAC,KAAK;wBAC3B,OAAO;wBACP,qBAAqB,EAAE,WAAW,CAAC,UAAU;wBAC7C,aAAa,EAAE,WAAW,CAAC,EAAE;wBAC7B,SAAS;wBACT,QAAQ;wBACR,YAAY,EAAE,IAAI;wBAClB,aAAa,EAAE,IAAI;wBACnB,UAAU;qBACX,CAAC;oBAEF,IAAI,KAAK,EAAE,CAAC;wBACV,sCAAsC;wBACtC,MAAM,iBAAiB,CAAC,WAAW,EAAE;4BACnC,eAAe,EAAE,0BAA0B,YAAY,GAAG;4BAC1D,WAAW,EAAE,UAAU;4BACvB,cAAc,EAAE,YAAY;4BAC5B,WAAW,EAAE,IAAI;4BACjB,cAAc,EAAE,IAAI;4BACpB,UAAU,EAAE,IAAI;4BAChB,eAAe,EAAE,IAAI;4BACrB,UAAU,EAAE,IAAI;4BAChB,eAAe,EAAE,IAAI;4BACrB,eAAe,EAAE,QAAQ,CAAC,eAAe;yBAC1C,CAAC,CAAC;wBAEH,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;wBACnE,MAAM,iBAAiB,GAAG,MAAM,gBAAgB,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;wBAE7D,UAAU,CAAC,IAAI,CACb,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,EAC1E,iCAAiC,CAClC,CAAC;wBAEF,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC;4BAChD,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;4BAC3B,IAAI,CAAC;gCACH,MAAM,QAAQ,GAAG,iBAAiB,CAAC,IAAI,CACrC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,OAAO,KAAK,KAAK,CAAC,OAAO,CACrC,CAAC;gCAEF,MAAM,UAAU,GAAG,GAAG,OAAO,CAAC,KAAK,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;gCACtD,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC;oCACrC,CAAC,CAAC,WAAW,IAAI,GAAG,CAAC,IAAI,MAAM,CAAC,MAAM,GAAG;oCACzC,CAAC,CAAC,EAAE,CAAC;gCAEP,MAAM,iBAAiB,CAAC,WAAW,EAAE;oCACnC,eAAe,EAAE,aAAa,UAAU,IAAI,aAAa,EAAE;oCAC3D,WAAW,EAAE,UAAU;oCACvB,cAAc,EAAE,YAAY;oCAC5B,WAAW,EAAE,IAAI;oCACjB,cAAc,EAAE,IAAI;oCACpB,UAAU,EAAE,IAAI;oCAChB,eAAe,EAAE,IAAI;oCACrB,UAAU,EAAE,IAAI;oCAChB,eAAe,EAAE,IAAI;oCACrB,eAAe,EAAE,QAAQ,CAAC,eAAe;iCAC1C,CAAC,CAAC;gCAEH,MAAM,UAAU,GAAG,MAAM,gBAAgB,CACvC,MAAM,EACN,OAAO,CAAC,UAAU,EAClB,KAAK,CAAC,OAAO,EACb,QAAQ,EAAE,sBAAsB,EAChC,GAAG,EACH,CAAC,OAAO,EAAE,EAAE;oCACV,SAAS,CAAC,MAAM,CAAC;wCACf,eAAe,EAAE,aAAa,UAAU,IAAI,aAAa,MAAM,OAAO,mBAAmB;wCACzF,WAAW,EAAE,UAAU;wCACvB,cAAc,EAAE,YAAY;wCAC5B,eAAe,EAAE,QAAQ,CAAC,eAAe,GAAG,OAAO;qCACpD,CAAC,CAAC;gCACL,CAAC,CACF,CAAC;gCAEF,yCAAyC;gCACzC,QAAQ,CAAC,eAAe,IAAI,UAAU,CAAC,YAAY,CAAC;gCAEpD,IAAI,UAAU,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oCACrC,UAAU,CAAC,KAAK,CACd,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,EAC5C,0BAA0B,CAC3B,CAAC;oCACF,SAAS;gCACX,CAAC;gCAED,UAAU,CAAC,IAAI,CACb,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE,UAAU,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,CAAC,MAAM,EAAE,EAC7F,yBAAyB,CAC1B,CAAC;gCAEF,sCAAsC;gCACtC,WAAW,CAAC,YAAY,GAAG,KAAK,CAAC,IAAI,CAAC;gCACtC,WAAW,CAAC,aAAa,GAAG,KAAK,CAAC,OAAO,CAAC;gCAC1C,WAAW,CAAC,YAAY,GAAG,GAAG,OAAO,CAAC,KAAK,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;gCAE9D,MAAM,cAAc,GAAG,MAAM,kBAAkB,CAAC,WAAW,EAAE,UAAU,EAAE,GAAG,CAAC,EAAE,EAAE,QAAQ,EAAE,sBAAsB,CAAC,CAAC;gCAEnH,sEAAsE;gCACtE,IAAI,cAAc,EAAE,CAAC;oCACnB,MAAM,mBAAmB,CACvB,OAAO,CAAC,EAAE,EACV,KAAK,CAAC,OAAO,EACb,KAAK,CAAC,IAAI,EACV,cAAc,CACf,CAAC;gCACJ,CAAC;4BACH,CAAC;4BAAC,OAAO,QAAQ,EAAE,CAAC;gCAClB,UAAU,CAAC,IAAI,CACb,EAAE,GAAG,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,EAC9F,mCAAmC,CACpC,CAAC;4BACJ,CAAC;wBACH,CAAC;oBACH,CAAC;yBAAM,CAAC;wBACN,yDAAyD;wBACzD,MAAM,iBAAiB,CAAC,WAAW,EAAE;4BACnC,eAAe,EAAE,aAAa,YAAY,oBAAoB;4BAC9D,WAAW,EAAE,UAAU;4BACvB,cAAc,EAAE,YAAY;4BAC5B,WAAW,EAAE,IAAI;4BACjB,cAAc,EAAE,IAAI;4BACpB,UAAU,EAAE,IAAI;4BAChB,eAAe,EAAE,IAAI;4BACrB,UAAU,EAAE,IAAI;4BAChB,eAAe,EAAE,IAAI;4BACrB,eAAe,EAAE,QAAQ,CAAC,eAAe;yBAC1C,CAAC,CAAC;wBAEH,UAAU,CAAC,IAAI,CACb,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,EAC/C,2BAA2B,CAC5B,CAAC;wBAEF,MAAM,UAAU,GAAG,MAAM,kBAAkB,CACzC,MAAM,EACN,OAAO,CAAC,UAAU,EAClB,OAAO,CAAC,sBAAsB,EAC9B,GAAG,EACH,CAAC,OAAO,EAAE,EAAE;4BACV,SAAS,CAAC,MAAM,CAAC;gCACf,eAAe,EAAE,aAAa,YAAY,OAAO,OAAO,mBAAmB;gCAC3E,WAAW,EAAE,UAAU;gCACvB,cAAc,EAAE,YAAY;gCAC5B,eAAe,EAAE,QAAQ,CAAC,eAAe,GAAG,OAAO;6BACpD,CAAC,CAAC;wBACL,CAAC,CACF,CAAC;wBAEF,yCAAyC;wBACzC,QAAQ,CAAC,eAAe,IAAI,UAAU,CAAC,YAAY,CAAC;wBAEpD,IAAI,UAAU,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;4BACrC,UAAU,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,EAAE,iBAAiB,CAAC,CAAC;4BAC/D,SAAS;wBACX,CAAC;wBAED,UAAU,CAAC,IAAI,CACb,EAAE,QAAQ,EAAE,UAAU,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,CAAC,MAAM,EAAE,EAC1E,2BAA2B,CAC5B,CAAC;wBAEF,iFAAiF;wBACjF,WAAW,CAAC,YAAY,GAAG,IAAI,CAAC;wBAChC,WAAW,CAAC,aAAa,GAAG,IAAI,CAAC;wBACjC,WAAW,CAAC,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC;wBAEzC,MAAM,cAAc,GAAG,MAAM,kBAAkB,CAAC,WAAW,EAAE,UAAU,EAAE,GAAG,CAAC,EAAE,EAAE,OAAO,CAAC,sBAAsB,CAAC,CAAC;wBAEjH,sEAAsE;wBACtE,IAAI,cAAc,EAAE,CAAC;4BACnB,MAAM,0BAA0B,CAAC,OAAO,CAAC,EAAE,EAAE,cAAc,CAAC,CAAC;wBAC/D,CAAC;oBACH,CAAC;gBACD,CAAC;gBAAC,OAAO,UAAU,EAAE,CAAC;oBACpB,UAAU,CAAC,IAAI,CACb,EAAE,GAAG,EAAE,UAAU,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,EAChE,6CAA6C,CAC9C,CAAC;gBACJ,CAAC;YACH,CAAC;YAED,aAAa;YACb,MAAM,oBAAoB,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;YAClD,UAAU,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,EAAE,yBAAyB,CAAC,CAAC;QAC3D,CAAC;gBAAS,CAAC;YACT,MAAM,gBAAgB,CAAC,MAAM,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjE,UAAU,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,EAAE,sBAAsB,CAAC,CAAC;QAClD,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,gBAAgB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CACjD,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,8BAA8B,CAAC,CACxD,CAAC;QACJ,CAAC;IACH,CAAC;YAAS,CAAC;QACT,MAAM,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAChC,CAAC;AACH,CAAC;AAED;;;;;;;GAOG;AACH,KAAK,UAAU,kBAAkB,CAC/B,GAAoB,EACpB,UAA6B,EAC7B,cAAsB,EACtB,sBAAsC;IAEtC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,YAAY,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU,EAAE,GAAG,GAAG,CAAC;IAEtF,0BAA0B;IAC1B,IAAI,WAAW,GAAG,gBAAgB,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IAExD,mFAAmF;IACnF,IAAI,sBAAsB,EAAE,CAAC;QAC3B,MAAM,WAAW,GAAG,WAAW,CAAC,MAAM,CAAC;QACvC,WAAW,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CACvC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,sBAAsB,CAAC,CACrD,CAAC;QACF,MAAM,QAAQ,GAAG,WAAW,GAAG,WAAW,CAAC,MAAM,CAAC;QAClD,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;YACjB,UAAU,CAAC,IAAI,CACb,EAAE,QAAQ,EAAE,SAAS,EAAE,WAAW,CAAC,MAAM,EAAE,EAC3C,6CAA6C,CAC9C,CAAC;QACJ,CAAC;IACH,CAAC;IAED,QAAQ,CAAC,SAAS,IAAI,WAAW,CAAC,MAAM,CAAC;IAEzC,uCAAuC;IACvC,MAAM,cAAc,GAAG,qBAAqB,CAC1C,UAAU,CAAC,MAAM,EACjB,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACtB,QAAQ,EAAE,CAAC,CAAC,QAAQ;QACpB,cAAc,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE;QAC7B,gBAAgB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI;KAClC,CAAC,CAAC,CACJ,CAAC;IAEF,IAAI,cAAc,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;QAC5B,UAAU,CAAC,IAAI,CACb,EAAE,OAAO,EAAE,cAAc,CAAC,IAAI,EAAE,KAAK,EAAE,WAAW,CAAC,MAAM,EAAE,EAC3D,oCAAoC,CACrC,CAAC;IACJ,CAAC;IAED,MAAM,iBAAiB,CAAC,KAAK,EAAE;QAC7B,eAAe,EAAE,SAAS,WAAW,CAAC,MAAM,mBAAmB,YAAY,GAAG;QAC9E,WAAW,EAAE,UAAU;QACvB,cAAc,EAAE,YAAY;QAC5B,UAAU,EAAE,WAAW,CAAC,MAAM;QAC9B,SAAS,EAAE,QAAQ,CAAC,SAAS;QAC7B,eAAe,EAAE,QAAQ,CAAC,eAAe;KAC1C,CAAC,CAAC;IAEH,+DAA+D;IAC/D,IAAI,cAAc,GAAkB,IAAI,CAAC;IAEzC,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC;QAC3D,IAAI,CAAC;YACH,MAAM,oBAAoB,CACxB,GAAG,EACH,WAAW,CAAC,MAAM,CAAC,EACnB,MAAM,EACN,WAAW,CAAC,MAAM,EAClB,cAAc,EACd,cAAc,CACf,CAAC;YAEF,sEAAsE;YACtE,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,MAAM,CAC/C,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,EACrC,EAAE,CACH,CAAC;YACF,IAAI,QAAQ,GAAG,CAAC,cAAc,IAAI,EAAE,CAAC,EAAE,CAAC;gBACtC,cAAc,GAAG,QAAQ,CAAC;YAC5B,CAAC;QACH,CAAC;QAAC,OAAO,MAAM,EAAE,CAAC;YAChB,uDAAuD;YACvD,UAAU,CAAC,IAAI,CACb,EAAE,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EACvD,8DAA8D,CAC/D,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,cAAc,CAAC;AACxB,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,oBAAoB,CACjC,GAAoB,EACpB,UAAsB,EACtB,MAAc,EACd,SAAiB,EACjB,cAA2D,EAC3D,cAAsB;IAEtB,MAAM,EACJ,MAAM,EAAE,KAAK,EAAE,YAAY,EAAE,OAAO,EACpC,qBAAqB,EAAE,aAAa,EACpC,SAAS,EAAE,QAAQ,EAAE,YAAY,EAAE,aAAa,EAAE,UAAU,GAC7D,GAAG,GAAG,CAAC;IAER,MAAM,WAAW,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;IAEjD,mEAAmE;IACnE,gFAAgF;IAChF,MAAM,eAAe,GAAG,MAAM,4BAA4B,CACxD,OAAO,CAAC,EAAE,EACV,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CACvB,CAAC;IACF,IAAI,eAAe,EAAE,CAAC;QACpB,QAAQ,CAAC,aAAa,EAAE,CAAC;QACzB,UAAU,CAAC,KAAK,CACd,EAAE,QAAQ,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAC1E,wDAAwD,CACzD,CAAC;QACF,MAAM,iBAAiB,CAAC,KAAK,EAAE;YAC7B,eAAe,EAAE,WAAW,WAAW,qBAAqB;YAC5D,WAAW,EAAE,eAAe;YAC5B,cAAc,EAAE,YAAY;YAC5B,WAAW,EAAE,WAAW;YACxB,cAAc,EAAE,MAAM,GAAG,CAAC;YAC1B,UAAU,EAAE,SAAS;YACrB,aAAa,EAAE,QAAQ,CAAC,aAAa;SACtC,CAAC,CAAC;QACH,OAAO;IACT,CAAC;IAED,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,IAAI,UAAU,GAAa,EAAE,CAAC;IAE9B,uEAAuE;IACvE,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,cAAc,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACxF,MAAM,KAAK,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAEzC,IAAI,CAAC;QACH,oBAAoB;QACpB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,UAAU,CAAC,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC;YACnE,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;YAElD,MAAM,SAAS,GAAG,UAAU,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC;gBAC3C,CAAC,CAAC,UAAU,OAAO,GAAG,CAAC,IAAI,UAAU,CAAC,KAAK,CAAC,MAAM,GAAG;gBACrD,CAAC,CAAC,EAAE,CAAC;YAEP,MAAM,iBAAiB,CAAC,KAAK,EAAE;gBAC7B,eAAe,EAAE,eAAe,IAAI,CAAC,QAAQ,GAAG,SAAS,EAAE;gBAC3D,WAAW,EAAE,aAAa;gBAC1B,cAAc,EAAE,YAAY;gBAC5B,WAAW,EAAE,IAAI,CAAC,QAAQ;gBAC1B,cAAc,EAAE,MAAM,GAAG,CAAC;gBAC1B,UAAU,EAAE,SAAS;gBACrB,eAAe,EAAE,EAAE;gBACnB,UAAU,EAAE,IAAI,CAAC,QAAQ;gBACzB,eAAe,EAAE,CAAC;gBAClB,eAAe,EAAE,QAAQ,CAAC,eAAe;aAC1C,CAAC,CAAC;YAEH,UAAU,CAAC,IAAI,CACb;gBACE,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;gBAC/B,IAAI,EAAE,OAAO,GAAG,CAAC;gBACjB,UAAU,EAAE,UAAU,CAAC,KAAK,CAAC,MAAM;aACpC,EACD,0BAA0B,CAC3B,CAAC;YAEF,MAAM,YAAY,CAChB,MAAM,EACN,IAAI,CAAC,MAAM,EACX,QAAQ,EACR,IAAI,CAAC,QAAQ,EACb,IAAI,CAAC,QAAQ,EACb,CAAC,QAA0B,EAAE,EAAE;gBAC7B,SAAS,CAAC,MAAM,CAAC;oBACf,eAAe,EAAE,eAAe,IAAI,CAAC,QAAQ,GAAG,SAAS,MAAM,QAAQ,CAAC,OAAO,GAAG;oBAClF,WAAW,EAAE,aAAa;oBAC1B,cAAc,EAAE,YAAY;oBAC5B,WAAW,EAAE,IAAI,CAAC,QAAQ;oBAC1B,cAAc,EAAE,MAAM,GAAG,CAAC;oBAC1B,UAAU,EAAE,SAAS;oBACrB,eAAe,EAAE,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAC;oBACjD,UAAU,EAAE,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC;oBACvC,eAAe,EAAE,QAAQ,CAAC,OAAO;iBAClC,CAAC,CAAC;YACL,CAAC,CACF,CAAC;YACF,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;YACxB,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC3B,CAAC;QAED,gBAAgB;QAChB,MAAM,iBAAiB,CAAC,KAAK,EAAE;YAC7B,eAAe,EAAE,sBAAsB,WAAW,EAAE;YACpD,WAAW,EAAE,SAAS;YACtB,cAAc,EAAE,YAAY;YAC5B,WAAW,EAAE,WAAW;YACxB,cAAc,EAAE,MAAM,GAAG,CAAC;YAC1B,UAAU,EAAE,SAAS;YACrB,eAAe,EAAE,IAAI;YACrB,UAAU,EAAE,IAAI;YAChB,eAAe,EAAE,IAAI;SACtB,CAAC,CAAC;QAEH,MAAM,WAAW,GAAG,MAAM,SAAS,CAAC,SAAS,CAAC,CAAC;QAE/C,sBAAsB;QACtB,MAAM,iBAAiB,CAAC,KAAK,EAAE;YAC7B,eAAe,EAAE,eAAe,WAAW,iBAAiB;YAC5D,WAAW,EAAE,eAAe;YAC5B,cAAc,EAAE,YAAY;YAC5B,WAAW,EAAE,WAAW;YACxB,cAAc,EAAE,MAAM,GAAG,CAAC;YAC1B,UAAU,EAAE,SAAS;SACtB,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC,WAAW,CAAC,CAAC;QACtD,IAAI,MAAM,EAAE,CAAC;YACX,QAAQ,CAAC,aAAa,EAAE,CAAC;YACzB,UAAU,CAAC,KAAK,CAAC,EAAE,WAAW,EAAE,EAAE,6BAA6B,CAAC,CAAC;YAEjE,MAAM,iBAAiB,CAAC,KAAK,EAAE;gBAC7B,eAAe,EAAE,WAAW,WAAW,cAAc;gBACrD,WAAW,EAAE,eAAe;gBAC5B,cAAc,EAAE,YAAY;gBAC5B,WAAW,EAAE,WAAW;gBACxB,cAAc,EAAE,MAAM,GAAG,CAAC;gBAC1B,UAAU,EAAE,SAAS;gBACrB,aAAa,EAAE,QAAQ,CAAC,aAAa;aACtC,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,yBAAyB;QACzB,MAAM,iBAAiB,CAAC,KAAK,EAAE;YAC7B,eAAe,EAAE,0BAA0B,WAAW,EAAE;YACxD,WAAW,EAAE,kBAAkB;YAC/B,cAAc,EAAE,YAAY;YAC5B,WAAW,EAAE,WAAW;YACxB,cAAc,EAAE,MAAM,GAAG,CAAC;YAC1B,UAAU,EAAE,SAAS;SACtB,CAAC,CAAC;QAEH,IAAI,OAAO,GAA2I,EAAE,CAAC;QACzJ,IAAI,CAAC;YACH,IAAI,UAAU,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;gBAC9B,OAAO,GAAG,MAAM,uBAAuB,CAAC,SAAS,CAAC,CAAC;YACrD,CAAC;iBAAM,CAAC;gBACN,OAAO,GAAG,MAAM,eAAe,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;YAChD,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,UAAU,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,UAAU,CAAC,QAAQ,EAAE,EAAE,8DAA8D,CAAC,CAAC;QAC1H,CAAC;QAED,0CAA0C;QAC1C,IAAI,WAAW,GAAG,CAAC,GAAG,SAAS,CAAC,CAAC;QACjC,MAAM,SAAS,GAAG,UAAU,CAAC,KAAK,CAAC,MAAM,CACvC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,EAC5B,EAAE,CACH,CAAC;QACF,MAAM,eAAe,GAAG,EAAE,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,CAAC;QACnD,MAAM,gBAAgB,GAAG,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,eAAe,CAAC,CAAC;QAEpF,IAAI,gBAAgB,EAAE,CAAC;YACrB,sFAAsF;YACtF,MAAM,iBAAiB,CAAC,KAAK,EAAE;gBAC7B,eAAe,EAAE,aAAa,WAAW,6CAA6C;gBACtF,WAAW,EAAE,WAAW;gBACxB,cAAc,EAAE,YAAY;gBAC5B,WAAW,EAAE,WAAW;gBACxB,cAAc,EAAE,MAAM,GAAG,CAAC;gBAC1B,UAAU,EAAE,SAAS;aACtB,CAAC,CAAC;YACH,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,UAAU,CAAC,QAAQ,SAAS,CAAC,CAAC;YACtE,MAAM,gBAAgB,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;YAC9C,UAAU,GAAG,MAAM,cAAc,CAAC,UAAU,CAAC,CAAC;YAC9C,WAAW,GAAG,UAAU,CAAC;YACzB,wCAAwC;YACxC,MAAM,MAAM,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAC3C,CAAC;aAAM,IAAI,CAAC,UAAU,CAAC,WAAW,IAAI,SAAS,GAAG,eAAe,EAAE,CAAC;YAClE,mCAAmC;YACnC,MAAM,iBAAiB,CAAC,KAAK,EAAE;gBAC7B,eAAe,EAAE,aAAa,WAAW,oBAAoB;gBAC7D,WAAW,EAAE,WAAW;gBACxB,cAAc,EAAE,YAAY;gBAC5B,WAAW,EAAE,WAAW;gBACxB,cAAc,EAAE,MAAM,GAAG,CAAC;gBAC1B,UAAU,EAAE,SAAS;aACtB,CAAC,CAAC;YACH,UAAU,GAAG,MAAM,cAAc,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;YAChD,WAAW,GAAG,UAAU,CAAC;QAC3B,CAAC;QAED,kBAAkB;QAClB,MAAM,WAAW,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC;YACxC,CAAC,CAAC,KAAK,WAAW,CAAC,MAAM,SAAS;YAClC,CAAC,CAAC,EAAE,CAAC;QACP,MAAM,iBAAiB,CAAC,KAAK,EAAE;YAC7B,eAAe,EAAE,aAAa,WAAW,sBAAsB,WAAW,EAAE;YAC5E,WAAW,EAAE,WAAW;YACxB,cAAc,EAAE,YAAY;YAC5B,WAAW,EAAE,WAAW;YACxB,cAAc,EAAE,MAAM,GAAG,CAAC;YAC1B,UAAU,EAAE,SAAS;SACtB,CAAC,CAAC;QAEH,MAAM,UAAU,GAAG,MAAM,eAAe,CACtC,MAAM,EACN,qBAAqB,EACrB,WAAW,CACZ,CAAC;QAEF,0BAA0B;QAC1B,IAAI,WAAW,GAAkB,IAAI,CAAC;QACtC,IAAI,YAAY,GAAkB,IAAI,CAAC;QACvC,MAAM,YAAY,GAAG,cAAc,CAAC,GAAG,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;QAC7D,IAAI,YAAY,EAAE,CAAC;YACjB,MAAM,iBAAiB,CAAC,KAAK,EAAE;gBAC7B,eAAe,EAAE,iCAAiC,WAAW,EAAE;gBAC/D,WAAW,EAAE,SAAS;gBACtB,cAAc,EAAE,YAAY;gBAC5B,WAAW,EAAE,WAAW;gBACxB,cAAc,EAAE,MAAM,GAAG,CAAC;gBAC1B,UAAU,EAAE,SAAS;aACtB,CAAC,CAAC;YACH,WAAW,GAAG,MAAM,sBAAsB,CAAC,MAAM,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC;YACxE,YAAY,GAAG,YAAY,CAAC,EAAE,CAAC;QACjC,CAAC;QAED,iEAAiE;QACjE,MAAM,OAAO,GAAG,YAAY,IAAI,0BAA0B,CAAC,WAAW,CAAC,IAAI,IAAI,CAAC;QAEhF,iBAAiB;QACjB,MAAM,iBAAiB,CAAC,KAAK,EAAE;YAC7B,eAAe,EAAE,uBAAuB,WAAW,KAAK,OAAO,CAAC,MAAM,SAAS;YAC/E,WAAW,EAAE,UAAU;YACvB,cAAc,EAAE,YAAY;YAC5B,WAAW,EAAE,WAAW;YACxB,cAAc,EAAE,MAAM,GAAG,CAAC;YAC1B,UAAU,EAAE,SAAS;SACtB,CAAC,CAAC;QAEH,8EAA8E;QAC9E,MAAM,2BAA2B,CAAC,WAAW,CAAC,CAAC;QAE/C,MAAM,sBAAsB,CAAC;YAC3B,WAAW;YACX,QAAQ,EAAE,WAAW;YACrB,QAAQ,EAAE,SAAS;YACnB,WAAW,EAAE,UAAU,CAAC,IAAI;YAC5B,eAAe,EAAE,OAAO,CAAC,EAAE;YAC3B,eAAe,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE;YACvC,aAAa;YACb,aAAa;YACb,aAAa,EAAE,UAAU,CAAC,SAAS;YACnC,WAAW,EACT,UAAU,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC;YACvD,SAAS,EAAE,WAAW,CAAC,MAAM;YAC7B,cAAc;YACd,OAAO;YACP,WAAW;YACX,YAAY;YACZ,KAAK,EAAE,OAAO;SACf,CAAC,CAAC;QAEH,QAAQ,CAAC,YAAY,EAAE,CAAC;QAExB,MAAM,iBAAiB,CAAC,KAAK,EAAE;YAC7B,eAAe,EAAE,YAAY,WAAW,KAAK,OAAO,CAAC,MAAM,iBAAiB;YAC5E,WAAW,EAAE,UAAU;YACvB,cAAc,EAAE,YAAY;YAC5B,WAAW,EAAE,WAAW;YACxB,cAAc,EAAE,MAAM,GAAG,CAAC;YAC1B,UAAU,EAAE,SAAS;YACrB,YAAY,EAAE,QAAQ,CAAC,YAAY;SACpC,CAAC,CAAC;QAEH,UAAU,CAAC,IAAI,CACb,EAAE,QAAQ,EAAE,WAAW,EAAE,WAAW,EAAE,SAAS,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,EAC1E,kBAAkB,CACnB,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,iDAAiD;QACjD,MAAM,WAAW,CAAC,CAAC,GAAG,SAAS,EAAE,GAAG,UAAU,CAAC,CAAC,CAAC;QACjD,MAAM,EAAE,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACrE,CAAC;AACH,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,KAAe;IACxC,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAAC,MAAM,CAAC;YACP,+CAA+C;QACjD,CAAC;IACH,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc;IAClC,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC9C,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAC/F,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,GAAG,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,EAAE,6BAA6B,CAAC,CAAC;QACrE,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,gCAAgC;IAClC,CAAC;AACH,CAAC"} \ No newline at end of file diff --git a/worker/src/scheduler.ts b/worker/src/scheduler.ts index edb02c1..6a5f6f7 100644 --- a/worker/src/scheduler.ts +++ b/worker/src/scheduler.ts @@ -10,6 +10,13 @@ let running = false; let timer: ReturnType | null = null; let cycleCount = 0; +/** + * Maximum time for a single ingestion cycle (ms). + * After this, new accounts won't be started (in-progress work finishes). + * Default: 4 hours. Configurable via WORKER_CYCLE_TIMEOUT_MINUTES. + */ +const CYCLE_TIMEOUT_MS = (parseInt(process.env.WORKER_CYCLE_TIMEOUT_MINUTES ?? "240", 10)) * 60 * 1000; + /** * Run one ingestion cycle: * 1. Authenticate any PENDING accounts (triggers SMS code flow + auto-fetch channels) @@ -17,6 +24,10 @@ let cycleCount = 0; * * All TDLib operations are wrapped in the mutex to ensure only one client * runs at a time (also shared with the fetch listener for on-demand requests). + * + * The cycle has a configurable timeout (WORKER_CYCLE_TIMEOUT_MINUTES, default 4h). + * Once the timeout elapses, no new accounts will be started but any in-progress + * account processing is allowed to finish its current archive set. */ async function runCycle(): Promise { if (running) { @@ -26,7 +37,8 @@ async function runCycle(): Promise { running = true; cycleCount++; - log.info({ cycle: cycleCount }, "Starting ingestion cycle"); + const cycleStart = Date.now(); + log.info({ cycle: cycleCount, timeoutMinutes: CYCLE_TIMEOUT_MS / 60_000 }, "Starting ingestion cycle"); try { // ── Phase 1: Authenticate pending accounts ── @@ -37,6 +49,10 @@ async function runCycle(): Promise { "Found pending accounts, starting authentication" ); for (const account of pendingAccounts) { + if (Date.now() - cycleStart > CYCLE_TIMEOUT_MS) { + log.warn("Cycle timeout reached during authentication phase, stopping"); + break; + } await withTdlibMutex(`auth:${account.phone}`, () => authenticateAccount(account) ); @@ -54,12 +70,22 @@ async function runCycle(): Promise { log.info({ accountCount: accounts.length }, "Processing accounts"); for (const account of accounts) { + if (Date.now() - cycleStart > CYCLE_TIMEOUT_MS) { + log.warn( + { elapsed: Math.round((Date.now() - cycleStart) / 60_000), timeoutMinutes: CYCLE_TIMEOUT_MS / 60_000 }, + "Cycle timeout reached, skipping remaining accounts" + ); + break; + } await withTdlibMutex(`ingest:${account.phone}`, () => runWorkerForAccount(account) ); } - log.info("Ingestion cycle complete"); + log.info( + { elapsed: Math.round((Date.now() - cycleStart) / 1000) }, + "Ingestion cycle complete" + ); } catch (err) { log.error({ err }, "Ingestion cycle failed"); } finally { diff --git a/worker/src/tdlib/download.ts b/worker/src/tdlib/download.ts index 0597ae3..d9c4a88 100644 --- a/worker/src/tdlib/download.ts +++ b/worker/src/tdlib/download.ts @@ -8,6 +8,12 @@ import type { TelegramPhoto } from "../preview/match.js"; const log = childLogger("download"); +/** Maximum number of pages to scan per channel/topic to prevent infinite loops */ +export const MAX_SCAN_PAGES = 5000; + +/** Timeout for a single TDLib API call (ms) */ +export const INVOKE_TIMEOUT_MS = 120_000; // 2 minutes + interface TdPhotoSize { type: string; photo: { @@ -71,6 +77,33 @@ export interface ChannelScanResult { export type ScanProgressCallback = (messagesScanned: number) => void; +/** + * Invoke a TDLib method with a timeout to prevent indefinite hangs. + * If TDLib does not respond within the timeout, the promise rejects. + */ +export async function invokeWithTimeout( + client: Client, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + request: Record, + timeoutMs = INVOKE_TIMEOUT_MS +): Promise { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + reject(new Error(`TDLib invoke timed out after ${timeoutMs}ms for ${request._}`)); + }, timeoutMs); + + (client.invoke(request) as Promise) + .then((result) => { + clearTimeout(timer); + resolve(result); + }) + .catch((err) => { + clearTimeout(timer); + reject(err); + }); + }); +} + /** * Fetch messages from a channel, stopping once we've scanned past the * last-processed boundary (with one page of lookback for multipart safety). @@ -80,6 +113,11 @@ export type ScanProgressCallback = (messagesScanned: number) => void; * When `lastProcessedMessageId` is null (first run), scans everything. * The worker applies a post-grouping filter to skip fully-processed sets, * and keeps `packageExistsBySourceMessage` as a safety net. + * + * Safety features: + * - Max page limit to prevent infinite loops + * - Stuck detection: breaks if from_message_id stops advancing + * - Timeout on each TDLib API call */ export async function getChannelMessages( client: Client, @@ -94,17 +132,29 @@ export async function getChannelMessages( let currentFromId = 0; let totalScanned = 0; + let pageCount = 0; // eslint-disable-next-line no-constant-condition while (true) { - const result = (await client.invoke({ + if (pageCount >= MAX_SCAN_PAGES) { + log.warn( + { chatId: chatId.toString(), pageCount, totalScanned }, + "Hit max page limit for channel scan, stopping" + ); + break; + } + pageCount++; + + const previousFromId = currentFromId; + + const result = await invokeWithTimeout<{ messages: TdMessage[] }>(client, { _: "getChatHistory", chat_id: Number(chatId), from_message_id: currentFromId, offset: 0, limit: Math.min(limit, 100), only_local: false, - })) as { messages: TdMessage[] }; + }); if (!result.messages || result.messages.length === 0) break; @@ -144,17 +194,26 @@ export async function getChannelMessages( currentFromId = result.messages[result.messages.length - 1].id; + // Stuck detection: if from_message_id didn't advance, break to prevent infinite loop + if (currentFromId === previousFromId) { + log.warn( + { chatId: chatId.toString(), currentFromId, totalScanned }, + "Pagination stuck (from_message_id not advancing), breaking" + ); + break; + } + // Stop scanning once we've gone past the boundary (this page is the lookback) if (boundary && currentFromId < boundary) break; - if (result.messages.length < 100) break; + if (result.messages.length < Math.min(limit, 100)) break; // Rate limit delay await sleep(config.apiDelayMs); } log.info( - { chatId: chatId.toString(), archives: archives.length, photos: photos.length, totalScanned }, + { chatId: chatId.toString(), archives: archives.length, photos: photos.length, totalScanned, pages: pageCount }, "Channel scan complete" ); diff --git a/worker/src/tdlib/topics.ts b/worker/src/tdlib/topics.ts index 107bf37..2f45a6b 100644 --- a/worker/src/tdlib/topics.ts +++ b/worker/src/tdlib/topics.ts @@ -5,6 +5,7 @@ import { isArchiveAttachment } from "../archive/detect.js"; import type { TelegramMessage } from "../archive/multipart.js"; import type { TelegramPhoto } from "../preview/match.js"; import type { ChannelScanResult, ScanProgressCallback } from "./download.js"; +import { invokeWithTimeout, MAX_SCAN_PAGES, INVOKE_TIMEOUT_MS } from "./download.js"; const log = childLogger("topics"); @@ -21,16 +22,16 @@ export async function isChatForum( chatId: bigint ): Promise { try { - const chat = (await client.invoke({ - _: "getChat", - chat_id: Number(chatId), - })) as { + const chat = await invokeWithTimeout<{ type?: { _: string; supergroup_id?: number; is_forum?: boolean; }; - }; + }>(client, { + _: "getChat", + chat_id: Number(chatId), + }); if (chat.type?._ === "chatTypeSupergroup" && chat.type.is_forum) { return true; @@ -38,10 +39,10 @@ export async function isChatForum( // Also check via getSupergroup for older TDLib versions if (chat.type?._ === "chatTypeSupergroup" && chat.type.supergroup_id) { - const sg = (await client.invoke({ + const sg = await invokeWithTimeout<{ is_forum?: boolean }>(client, { _: "getSupergroup", supergroup_id: chat.type.supergroup_id, - })) as { is_forum?: boolean }; + }); return sg.is_forum === true; } @@ -54,6 +55,7 @@ export async function isChatForum( /** * Get all forum topics in a supergroup. + * Includes stuck detection and timeout protection on API calls. */ export async function getForumTopicList( client: Client, @@ -63,18 +65,24 @@ export async function getForumTopicList( let offsetDate = 0; let offsetMessageId = 0; let offsetMessageThreadId = 0; + let pageCount = 0; // eslint-disable-next-line no-constant-condition while (true) { - const result = (await client.invoke({ - _: "getForumTopics", - chat_id: Number(chatId), - query: "", - offset_date: offsetDate, - offset_message_id: offsetMessageId, - offset_message_thread_id: offsetMessageThreadId, - limit: 100, - })) as { + if (pageCount >= MAX_SCAN_PAGES) { + log.warn( + { chatId: chatId.toString(), pageCount, topicCount: topics.length }, + "Hit max page limit for topic enumeration, stopping" + ); + break; + } + pageCount++; + + const prevOffsetDate = offsetDate; + const prevOffsetMessageId = offsetMessageId; + const prevOffsetMessageThreadId = offsetMessageThreadId; + + const result = await invokeWithTimeout<{ topics?: { info?: { message_thread_id?: number; @@ -85,7 +93,15 @@ export async function getForumTopicList( next_offset_date?: number; next_offset_message_id?: number; next_offset_message_thread_id?: number; - }; + }>(client, { + _: "getForumTopics", + chat_id: Number(chatId), + query: "", + offset_date: offsetDate, + offset_message_id: offsetMessageId, + offset_message_thread_id: offsetMessageThreadId, + limit: 100, + }); if (!result.topics || result.topics.length === 0) break; @@ -113,6 +129,19 @@ export async function getForumTopicList( offsetMessageId = result.next_offset_message_id ?? 0; offsetMessageThreadId = result.next_offset_message_thread_id ?? 0; + // Stuck detection: if offsets didn't advance, break + if ( + offsetDate === prevOffsetDate && + offsetMessageId === prevOffsetMessageId && + offsetMessageThreadId === prevOffsetMessageThreadId + ) { + log.warn( + { chatId: chatId.toString(), topicCount: topics.length }, + "Topic pagination stuck (offsets not advancing), breaking" + ); + break; + } + await sleep(config.apiDelayMs); } @@ -134,6 +163,11 @@ export async function getForumTopicList( * When `lastProcessedMessageId` is null (first run), scans everything. * The worker applies a post-grouping filter to skip fully-processed sets, * and keeps `packageExistsBySourceMessage` as a safety net. + * + * Safety features: + * - Max page limit to prevent infinite loops + * - Stuck detection: breaks if from_message_id stops advancing + * - Timeout on each TDLib API call */ export async function getTopicMessages( client: Client, @@ -149,22 +183,23 @@ export async function getTopicMessages( let currentFromId = 0; let totalScanned = 0; + let pageCount = 0; // eslint-disable-next-line no-constant-condition while (true) { + if (pageCount >= MAX_SCAN_PAGES) { + log.warn( + { chatId: chatId.toString(), topicId: topicId.toString(), pageCount, totalScanned }, + "Hit max page limit for topic scan, stopping" + ); + break; + } + pageCount++; + + const previousFromId = currentFromId; + // eslint-disable-next-line @typescript-eslint/no-explicit-any - const result = (await client.invoke({ - _: "searchChatMessages", - chat_id: Number(chatId), - query: "", - message_thread_id: Number(topicId), - from_message_id: currentFromId, - offset: 0, - limit: Math.min(limit, 100), - filter: null, - sender_id: null, - saved_messages_topic_id: 0, - })) as { + const result = await invokeWithTimeout<{ messages?: { id: number; date: number; @@ -188,7 +223,18 @@ export async function getTopicMessages( caption?: { text?: string }; }; }[]; - }; + }>(client, { + _: "searchChatMessages", + chat_id: Number(chatId), + query: "", + message_thread_id: Number(topicId), + from_message_id: currentFromId, + offset: 0, + limit: Math.min(limit, 100), + filter: null, + sender_id: null, + saved_messages_topic_id: 0, + }); if (!result.messages || result.messages.length === 0) break; @@ -228,16 +274,25 @@ export async function getTopicMessages( currentFromId = result.messages[result.messages.length - 1].id; + // Stuck detection: if from_message_id didn't advance, break to prevent infinite loop + if (currentFromId === previousFromId) { + log.warn( + { chatId: chatId.toString(), topicId: topicId.toString(), currentFromId, totalScanned }, + "Topic pagination stuck (from_message_id not advancing), breaking" + ); + break; + } + // Stop scanning once we've gone past the boundary (this page is the lookback) if (boundary && currentFromId < boundary) break; - if (result.messages.length < 100) break; + if (result.messages.length < Math.min(limit, 100)) break; await sleep(config.apiDelayMs); } log.info( - { chatId: chatId.toString(), topicId: topicId.toString(), archives: archives.length, photos: photos.length, totalScanned }, + { chatId: chatId.toString(), topicId: topicId.toString(), archives: archives.length, photos: photos.length, totalScanned, pages: pageCount }, "Topic scan complete" ); diff --git a/worker/src/util/mutex.ts b/worker/src/util/mutex.ts index f35f193..dcb95cf 100644 --- a/worker/src/util/mutex.ts +++ b/worker/src/util/mutex.ts @@ -4,12 +4,21 @@ const log = childLogger("mutex"); let locked = false; let holder = ""; -const queue: Array<{ resolve: () => void; label: string }> = []; +const queue: Array<{ resolve: () => void; reject: (err: Error) => void; label: string }> = []; + +/** + * Maximum time to wait for the TDLib mutex (ms). + * If the mutex is not available within this time, the operation is rejected. + * Default: 30 minutes (long enough for large downloads, short enough to detect hangs). + */ +const MUTEX_WAIT_TIMEOUT_MS = 30 * 60 * 1000; /** * Ensures only one TDLib client runs at a time across the entire worker process. * Both the scheduler (auth, ingestion) and the fetch listener acquire this * before creating any TDLib client. + * + * Includes a wait timeout to prevent indefinite blocking if the current holder hangs. */ export async function withTdlibMutex( label: string, @@ -17,7 +26,29 @@ export async function withTdlibMutex( ): Promise { if (locked) { log.info({ waiting: label, holder }, "Waiting for TDLib mutex"); - await new Promise((resolve) => queue.push({ resolve, label })); + await new Promise((resolve, reject) => { + const entry = { resolve, reject, label }; + queue.push(entry); + + // Timeout: reject if we've been waiting too long + const timer = setTimeout(() => { + const idx = queue.indexOf(entry); + if (idx !== -1) { + queue.splice(idx, 1); + reject(new Error( + `TDLib mutex wait timeout after ${MUTEX_WAIT_TIMEOUT_MS / 60_000}min ` + + `(waiting: ${label}, holder: ${holder})` + )); + } + }, MUTEX_WAIT_TIMEOUT_MS); + + // Wrap resolve to clear the timer + const origResolve = entry.resolve; + entry.resolve = () => { + clearTimeout(timer); + origResolve(); + }; + }); } locked = true; From 2e242912af311d995598ad4201e7b93c806fd92d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 5 Mar 2026 13:15:17 +0000 Subject: [PATCH 3/4] Remove worker/dist build artifacts from git, add to .gitignore Co-authored-by: xCyanGrizzly <53275238+xCyanGrizzly@users.noreply.github.com> --- .gitignore | 1 + worker/dist/archive/creator.d.ts | 12 - worker/dist/archive/creator.js | 21 - worker/dist/archive/creator.js.map | 1 - worker/dist/archive/detect.d.ts | 15 - worker/dist/archive/detect.js | 77 --- worker/dist/archive/detect.js.map | 1 - worker/dist/archive/hash.d.ts | 6 - worker/dist/archive/hash.js | 22 - worker/dist/archive/hash.js.map | 1 - worker/dist/archive/multipart.d.ts | 19 - worker/dist/archive/multipart.js | 74 --- worker/dist/archive/multipart.js.map | 1 - worker/dist/archive/rar-reader.d.ts | 6 - worker/dist/archive/rar-reader.js | 77 --- worker/dist/archive/rar-reader.js.map | 1 - worker/dist/archive/split.d.ts | 11 - worker/dist/archive/split.js | 55 -- worker/dist/archive/split.js.map | 1 - worker/dist/archive/zip-reader.d.ts | 15 - worker/dist/archive/zip-reader.js | 161 ------ worker/dist/archive/zip-reader.js.map | 1 - worker/dist/db/client.d.ts | 7 - worker/dist/db/client.js | 12 - worker/dist/db/client.js.map | 1 - worker/dist/db/locks.d.ts | 9 - worker/dist/db/locks.js | 53 -- worker/dist/db/locks.js.map | 1 - worker/dist/db/queries.d.ts | 356 ------------ worker/dist/db/queries.js | 319 ----------- worker/dist/db/queries.js.map | 1 - worker/dist/fetch-listener.d.ts | 11 - worker/dist/fetch-listener.js | 195 ------- worker/dist/fetch-listener.js.map | 1 - worker/dist/index.d.ts | 1 - worker/dist/index.js | 50 -- worker/dist/index.js.map | 1 - worker/dist/preview/match.d.ts | 22 - worker/dist/preview/match.js | 53 -- worker/dist/preview/match.js.map | 1 - worker/dist/scheduler.d.ts | 13 - worker/dist/scheduler.js | 121 ----- worker/dist/scheduler.js.map | 1 - worker/dist/tdlib/chats.d.ts | 31 -- worker/dist/tdlib/chats.js | 124 ----- worker/dist/tdlib/chats.js.map | 1 - worker/dist/tdlib/client.d.ts | 18 - worker/dist/tdlib/client.js | 96 ---- worker/dist/tdlib/client.js.map | 1 - worker/dist/tdlib/download.d.ts | 67 --- worker/dist/tdlib/download.js | 307 ----------- worker/dist/tdlib/download.js.map | 1 - worker/dist/tdlib/topics.d.ts | 32 -- worker/dist/tdlib/topics.js | 196 ------- worker/dist/tdlib/topics.js.map | 1 - worker/dist/upload/channel.d.ts | 16 - worker/dist/upload/channel.js | 137 ----- worker/dist/upload/channel.js.map | 1 - worker/dist/util/config.d.ts | 18 - worker/dist/util/config.js | 19 - worker/dist/util/config.js.map | 1 - worker/dist/util/logger.d.ts | 3 - worker/dist/util/logger.js | 12 - worker/dist/util/logger.js.map | 1 - worker/dist/util/mutex.d.ts | 8 - worker/dist/util/mutex.js | 61 --- worker/dist/util/mutex.js.map | 1 - worker/dist/worker.d.ts | 28 - worker/dist/worker.js | 745 -------------------------- worker/dist/worker.js.map | 1 - 70 files changed, 1 insertion(+), 3734 deletions(-) delete mode 100644 worker/dist/archive/creator.d.ts delete mode 100644 worker/dist/archive/creator.js delete mode 100644 worker/dist/archive/creator.js.map delete mode 100644 worker/dist/archive/detect.d.ts delete mode 100644 worker/dist/archive/detect.js delete mode 100644 worker/dist/archive/detect.js.map delete mode 100644 worker/dist/archive/hash.d.ts delete mode 100644 worker/dist/archive/hash.js delete mode 100644 worker/dist/archive/hash.js.map delete mode 100644 worker/dist/archive/multipart.d.ts delete mode 100644 worker/dist/archive/multipart.js delete mode 100644 worker/dist/archive/multipart.js.map delete mode 100644 worker/dist/archive/rar-reader.d.ts delete mode 100644 worker/dist/archive/rar-reader.js delete mode 100644 worker/dist/archive/rar-reader.js.map delete mode 100644 worker/dist/archive/split.d.ts delete mode 100644 worker/dist/archive/split.js delete mode 100644 worker/dist/archive/split.js.map delete mode 100644 worker/dist/archive/zip-reader.d.ts delete mode 100644 worker/dist/archive/zip-reader.js delete mode 100644 worker/dist/archive/zip-reader.js.map delete mode 100644 worker/dist/db/client.d.ts delete mode 100644 worker/dist/db/client.js delete mode 100644 worker/dist/db/client.js.map delete mode 100644 worker/dist/db/locks.d.ts delete mode 100644 worker/dist/db/locks.js delete mode 100644 worker/dist/db/locks.js.map delete mode 100644 worker/dist/db/queries.d.ts delete mode 100644 worker/dist/db/queries.js delete mode 100644 worker/dist/db/queries.js.map delete mode 100644 worker/dist/fetch-listener.d.ts delete mode 100644 worker/dist/fetch-listener.js delete mode 100644 worker/dist/fetch-listener.js.map delete mode 100644 worker/dist/index.d.ts delete mode 100644 worker/dist/index.js delete mode 100644 worker/dist/index.js.map delete mode 100644 worker/dist/preview/match.d.ts delete mode 100644 worker/dist/preview/match.js delete mode 100644 worker/dist/preview/match.js.map delete mode 100644 worker/dist/scheduler.d.ts delete mode 100644 worker/dist/scheduler.js delete mode 100644 worker/dist/scheduler.js.map delete mode 100644 worker/dist/tdlib/chats.d.ts delete mode 100644 worker/dist/tdlib/chats.js delete mode 100644 worker/dist/tdlib/chats.js.map delete mode 100644 worker/dist/tdlib/client.d.ts delete mode 100644 worker/dist/tdlib/client.js delete mode 100644 worker/dist/tdlib/client.js.map delete mode 100644 worker/dist/tdlib/download.d.ts delete mode 100644 worker/dist/tdlib/download.js delete mode 100644 worker/dist/tdlib/download.js.map delete mode 100644 worker/dist/tdlib/topics.d.ts delete mode 100644 worker/dist/tdlib/topics.js delete mode 100644 worker/dist/tdlib/topics.js.map delete mode 100644 worker/dist/upload/channel.d.ts delete mode 100644 worker/dist/upload/channel.js delete mode 100644 worker/dist/upload/channel.js.map delete mode 100644 worker/dist/util/config.d.ts delete mode 100644 worker/dist/util/config.js delete mode 100644 worker/dist/util/config.js.map delete mode 100644 worker/dist/util/logger.d.ts delete mode 100644 worker/dist/util/logger.js delete mode 100644 worker/dist/util/logger.js.map delete mode 100644 worker/dist/util/mutex.d.ts delete mode 100644 worker/dist/util/mutex.js delete mode 100644 worker/dist/util/mutex.js.map delete mode 100644 worker/dist/worker.d.ts delete mode 100644 worker/dist/worker.js delete mode 100644 worker/dist/worker.js.map diff --git a/.gitignore b/.gitignore index 0f0f125..7718c40 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ worker/node_modules # production /build +worker/dist # misc .DS_Store diff --git a/worker/dist/archive/creator.d.ts b/worker/dist/archive/creator.d.ts deleted file mode 100644 index 07eacd3..0000000 --- a/worker/dist/archive/creator.d.ts +++ /dev/null @@ -1,12 +0,0 @@ -/** - * Extract a creator name from common archive file naming patterns. - * - * Priority in the worker: topic name > filename extraction. - * This is the fallback when no forum topic name is available. - * - * Patterns handled (split on ` - `): - * "Mammoth Factory - 2026-01.zip" → "Mammoth Factory" - * "Artist Name - Pack Title.part01.rar" → "Artist Name" - * "some_random_file.zip" → null - */ -export declare function extractCreatorFromFileName(fileName: string): string | null; diff --git a/worker/dist/archive/creator.js b/worker/dist/archive/creator.js deleted file mode 100644 index 2ecf610..0000000 --- a/worker/dist/archive/creator.js +++ /dev/null @@ -1,21 +0,0 @@ -/** - * Extract a creator name from common archive file naming patterns. - * - * Priority in the worker: topic name > filename extraction. - * This is the fallback when no forum topic name is available. - * - * Patterns handled (split on ` - `): - * "Mammoth Factory - 2026-01.zip" → "Mammoth Factory" - * "Artist Name - Pack Title.part01.rar" → "Artist Name" - * "some_random_file.zip" → null - */ -export function extractCreatorFromFileName(fileName) { - // Strip archive extensions (.zip, .rar, .part01.rar, .z01, etc.) - const bare = fileName.replace(/(\.(part\d+\.rar|z\d{2}|zip|rar))+$/i, ""); - const idx = bare.indexOf(" - "); - if (idx <= 0) - return null; - const creator = bare.slice(0, idx).trim(); - return creator.length > 0 ? creator : null; -} -//# sourceMappingURL=creator.js.map \ No newline at end of file diff --git a/worker/dist/archive/creator.js.map b/worker/dist/archive/creator.js.map deleted file mode 100644 index 4c5cdab..0000000 --- a/worker/dist/archive/creator.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"creator.js","sourceRoot":"","sources":["../../src/archive/creator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AACH,MAAM,UAAU,0BAA0B,CAAC,QAAgB;IACzD,iEAAiE;IACjE,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,sCAAsC,EAAE,EAAE,CAAC,CAAC;IAE1E,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAChC,IAAI,GAAG,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IAE1B,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IAC1C,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;AAC7C,CAAC"} \ No newline at end of file diff --git a/worker/dist/archive/detect.d.ts b/worker/dist/archive/detect.d.ts deleted file mode 100644 index 9f0c21f..0000000 --- a/worker/dist/archive/detect.d.ts +++ /dev/null @@ -1,15 +0,0 @@ -export type ArchiveFormat = "ZIP" | "RAR"; -export interface MultipartInfo { - baseName: string; - partNumber: number; - format: ArchiveFormat; - pattern: "ZIP_NUMBERED" | "ZIP_LEGACY" | "RAR_PART" | "RAR_LEGACY" | "SINGLE"; -} -/** - * Detect if a filename is an archive and extract multipart info. - */ -export declare function detectArchive(fileName: string): MultipartInfo | null; -/** - * Check if a filename looks like any archive attachment we should process. - */ -export declare function isArchiveAttachment(fileName: string): boolean; diff --git a/worker/dist/archive/detect.js b/worker/dist/archive/detect.js deleted file mode 100644 index ab22108..0000000 --- a/worker/dist/archive/detect.js +++ /dev/null @@ -1,77 +0,0 @@ -const patterns = [ - // pack.zip.001, pack.zip.002 - { - regex: /^(.+\.zip)\.(\d{3,})$/i, - format: "ZIP", - pattern: "ZIP_NUMBERED", - getBaseName: (m) => m[1], - getPartNumber: (m) => parseInt(m[2], 10), - }, - // pack.z01, pack.z02 (legacy split — final part is pack.zip) - { - regex: /^(.+)\.z(\d{2,})$/i, - format: "ZIP", - pattern: "ZIP_LEGACY", - getBaseName: (m) => m[1], - getPartNumber: (m) => parseInt(m[2], 10), - }, - // pack.part1.rar, pack.part2.rar - { - regex: /^(.+)\.part(\d+)\.rar$/i, - format: "RAR", - pattern: "RAR_PART", - getBaseName: (m) => m[1], - getPartNumber: (m) => parseInt(m[2], 10), - }, - // pack.r00, pack.r01 (legacy split — final part is pack.rar) - { - regex: /^(.+)\.r(\d{2,})$/i, - format: "RAR", - pattern: "RAR_LEGACY", - getBaseName: (m) => m[1], - getPartNumber: (m) => parseInt(m[2], 10), - }, -]; -/** - * Detect if a filename is an archive and extract multipart info. - */ -export function detectArchive(fileName) { - // Check multipart patterns first - for (const p of patterns) { - const match = fileName.match(p.regex); - if (match) { - return { - baseName: p.getBaseName(match), - partNumber: p.getPartNumber(match), - format: p.format, - pattern: p.pattern, - }; - } - } - // Single .zip file — could be a standalone or the final part of a ZIP_LEGACY set - if (/\.zip$/i.test(fileName)) { - return { - baseName: fileName.replace(/\.zip$/i, ""), - partNumber: -1, // -1 signals "could be single or final legacy part" - format: "ZIP", - pattern: "SINGLE", - }; - } - // Single .rar file — could be standalone or final part of RAR_LEGACY set - if (/\.rar$/i.test(fileName)) { - return { - baseName: fileName.replace(/\.rar$/i, ""), - partNumber: -1, - format: "RAR", - pattern: "SINGLE", - }; - } - return null; -} -/** - * Check if a filename looks like any archive attachment we should process. - */ -export function isArchiveAttachment(fileName) { - return detectArchive(fileName) !== null; -} -//# sourceMappingURL=detect.js.map \ No newline at end of file diff --git a/worker/dist/archive/detect.js.map b/worker/dist/archive/detect.js.map deleted file mode 100644 index 96036b7..0000000 --- a/worker/dist/archive/detect.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"detect.js","sourceRoot":"","sources":["../../src/archive/detect.ts"],"names":[],"mappings":"AASA,MAAM,QAAQ,GAMR;IACJ,6BAA6B;IAC7B;QACE,KAAK,EAAE,wBAAwB;QAC/B,MAAM,EAAE,KAAK;QACb,OAAO,EAAE,cAAc;QACvB,WAAW,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACxB,aAAa,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;KACzC;IACD,6DAA6D;IAC7D;QACE,KAAK,EAAE,oBAAoB;QAC3B,MAAM,EAAE,KAAK;QACb,OAAO,EAAE,YAAY;QACrB,WAAW,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACxB,aAAa,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;KACzC;IACD,iCAAiC;IACjC;QACE,KAAK,EAAE,yBAAyB;QAChC,MAAM,EAAE,KAAK;QACb,OAAO,EAAE,UAAU;QACnB,WAAW,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACxB,aAAa,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;KACzC;IACD,6DAA6D;IAC7D;QACE,KAAK,EAAE,oBAAoB;QAC3B,MAAM,EAAE,KAAK;QACb,OAAO,EAAE,YAAY;QACrB,WAAW,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACxB,aAAa,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;KACzC;CACF,CAAC;AAEF;;GAEG;AACH,MAAM,UAAU,aAAa,CAAC,QAAgB;IAC5C,iCAAiC;IACjC,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QACtC,IAAI,KAAK,EAAE,CAAC;YACV,OAAO;gBACL,QAAQ,EAAE,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC;gBAC9B,UAAU,EAAE,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC;gBAClC,MAAM,EAAE,CAAC,CAAC,MAAM;gBAChB,OAAO,EAAE,CAAC,CAAC,OAAO;aACnB,CAAC;QACJ,CAAC;IACH,CAAC;IAED,iFAAiF;IACjF,IAAI,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC7B,OAAO;YACL,QAAQ,EAAE,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC;YACzC,UAAU,EAAE,CAAC,CAAC,EAAE,oDAAoD;YACpE,MAAM,EAAE,KAAK;YACb,OAAO,EAAE,QAAQ;SAClB,CAAC;IACJ,CAAC;IAED,yEAAyE;IACzE,IAAI,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC7B,OAAO;YACL,QAAQ,EAAE,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC;YACzC,UAAU,EAAE,CAAC,CAAC;YACd,MAAM,EAAE,KAAK;YACb,OAAO,EAAE,QAAQ;SAClB,CAAC;IACJ,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,QAAgB;IAClD,OAAO,aAAa,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC;AAC1C,CAAC"} \ No newline at end of file diff --git a/worker/dist/archive/hash.d.ts b/worker/dist/archive/hash.d.ts deleted file mode 100644 index 0f02b3f..0000000 --- a/worker/dist/archive/hash.d.ts +++ /dev/null @@ -1,6 +0,0 @@ -/** - * Compute SHA-256 hash of one or more files by streaming them in order. - * Memory usage: O(1) — reads in 64KB chunks regardless of total size. - * For multipart archives, pass all parts sorted by part number. - */ -export declare function hashParts(filePaths: string[]): Promise; diff --git a/worker/dist/archive/hash.js b/worker/dist/archive/hash.js deleted file mode 100644 index c13759c..0000000 --- a/worker/dist/archive/hash.js +++ /dev/null @@ -1,22 +0,0 @@ -import { createReadStream } from "fs"; -import { createHash } from "crypto"; -import { pipeline } from "stream/promises"; -import { PassThrough } from "stream"; -/** - * Compute SHA-256 hash of one or more files by streaming them in order. - * Memory usage: O(1) — reads in 64KB chunks regardless of total size. - * For multipart archives, pass all parts sorted by part number. - */ -export async function hashParts(filePaths) { - const hash = createHash("sha256"); - for (const filePath of filePaths) { - await pipeline(createReadStream(filePath), new PassThrough({ - transform(chunk, _encoding, callback) { - hash.update(chunk); - callback(); - }, - })); - } - return hash.digest("hex"); -} -//# sourceMappingURL=hash.js.map \ No newline at end of file diff --git a/worker/dist/archive/hash.js.map b/worker/dist/archive/hash.js.map deleted file mode 100644 index 52d1556..0000000 --- a/worker/dist/archive/hash.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"hash.js","sourceRoot":"","sources":["../../src/archive/hash.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,IAAI,CAAC;AACtC,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAC3C,OAAO,EAAE,WAAW,EAAE,MAAM,QAAQ,CAAC;AAErC;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,SAAmB;IACjD,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IAClC,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,MAAM,QAAQ,CACZ,gBAAgB,CAAC,QAAQ,CAAC,EAC1B,IAAI,WAAW,CAAC;YACd,SAAS,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ;gBAClC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACnB,QAAQ,EAAE,CAAC;YACb,CAAC;SACF,CAAC,CACH,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC5B,CAAC"} \ No newline at end of file diff --git a/worker/dist/archive/multipart.d.ts b/worker/dist/archive/multipart.d.ts deleted file mode 100644 index 2b46519..0000000 --- a/worker/dist/archive/multipart.d.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { type ArchiveFormat } from "./detect.js"; -export interface TelegramMessage { - id: bigint; - fileName: string; - fileId: string; - fileSize: bigint; - date: Date; -} -export interface ArchiveSet { - type: ArchiveFormat; - baseName: string; - parts: TelegramMessage[]; - isMultipart: boolean; -} -/** - * Group messages into archive sets (single files + multipart groups). - * Messages should be pre-filtered to only include archive attachments. - */ -export declare function groupArchiveSets(messages: TelegramMessage[]): ArchiveSet[]; diff --git a/worker/dist/archive/multipart.js b/worker/dist/archive/multipart.js deleted file mode 100644 index efecfd1..0000000 --- a/worker/dist/archive/multipart.js +++ /dev/null @@ -1,74 +0,0 @@ -import { detectArchive } from "./detect.js"; -import { config } from "../util/config.js"; -import { childLogger } from "../util/logger.js"; -const log = childLogger("multipart"); -/** - * Group messages into archive sets (single files + multipart groups). - * Messages should be pre-filtered to only include archive attachments. - */ -export function groupArchiveSets(messages) { - // Detect and annotate each message - const annotated = []; - for (const msg of messages) { - const info = detectArchive(msg.fileName); - if (info) { - annotated.push({ msg, info }); - } - } - // Group by baseName + format - const groups = new Map(); - for (const item of annotated) { - const key = `${item.info.format}:${item.info.baseName.toLowerCase()}`; - const group = groups.get(key) ?? []; - group.push(item); - groups.set(key, group); - } - const results = []; - for (const [, group] of groups) { - const format = group[0].info.format; - const baseName = group[0].info.baseName; - // Separate explicit multipart entries from potential singles - const multipartEntries = group.filter((g) => g.info.pattern !== "SINGLE"); - const singleEntries = group.filter((g) => g.info.pattern === "SINGLE"); - if (multipartEntries.length > 0) { - // This is a multipart set - // Check if any single entry is the "final part" of a legacy split - const allEntries = [...multipartEntries, ...singleEntries]; - // Check time span — skip if parts span too long (0 = no limit) - if (config.multipartTimeoutHours > 0) { - const dates = allEntries.map((e) => e.msg.date.getTime()); - const span = Math.max(...dates) - Math.min(...dates); - const maxSpanMs = config.multipartTimeoutHours * 60 * 60 * 1000; - if (span > maxSpanMs) { - log.warn({ baseName, format, span: span / 3600000 }, "Multipart set spans too long, skipping"); - continue; - } - } - // Sort by part number (singles get a very high number so they come last — they're the final part) - allEntries.sort((a, b) => { - const aNum = a.info.partNumber === -1 ? 999999 : a.info.partNumber; - const bNum = b.info.partNumber === -1 ? 999999 : b.info.partNumber; - return aNum - bNum; - }); - results.push({ - type: format, - baseName, - parts: allEntries.map((e) => e.msg), - isMultipart: true, - }); - } - else { - // All entries are singles — each is its own archive set - for (const entry of singleEntries) { - results.push({ - type: format, - baseName: entry.info.baseName, - parts: [entry.msg], - isMultipart: false, - }); - } - } - } - return results; -} -//# sourceMappingURL=multipart.js.map \ No newline at end of file diff --git a/worker/dist/archive/multipart.js.map b/worker/dist/archive/multipart.js.map deleted file mode 100644 index 4708bf7..0000000 --- a/worker/dist/archive/multipart.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"multipart.js","sourceRoot":"","sources":["../../src/archive/multipart.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAA0C,MAAM,aAAa,CAAC;AACpF,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,MAAM,GAAG,GAAG,WAAW,CAAC,WAAW,CAAC,CAAC;AAiBrC;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,QAA2B;IAC1D,mCAAmC;IACnC,MAAM,SAAS,GAAoD,EAAE,CAAC;IACtE,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACzC,IAAI,IAAI,EAAE,CAAC;YACT,SAAS,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;IAED,6BAA6B;IAC7B,MAAM,MAAM,GAAG,IAAI,GAAG,EAA2D,CAAC;IAClF,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,EAAE,CAAC;QACtE,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACpC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjB,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACzB,CAAC;IAED,MAAM,OAAO,GAAiB,EAAE,CAAC;IAEjC,KAAK,MAAM,CAAC,EAAE,KAAK,CAAC,IAAI,MAAM,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;QACpC,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC;QAExC,6DAA6D;QAC7D,MAAM,gBAAgB,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC;QAC1E,MAAM,aAAa,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC;QAEvE,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChC,0BAA0B;YAC1B,kEAAkE;YAClE,MAAM,UAAU,GAAG,CAAC,GAAG,gBAAgB,EAAE,GAAG,aAAa,CAAC,CAAC;YAE3D,+DAA+D;YAC/D,IAAI,MAAM,CAAC,qBAAqB,GAAG,CAAC,EAAE,CAAC;gBACrC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;gBAC1D,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC;gBACrD,MAAM,SAAS,GAAG,MAAM,CAAC,qBAAqB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;gBAEhE,IAAI,IAAI,GAAG,SAAS,EAAE,CAAC;oBACrB,GAAG,CAAC,IAAI,CACN,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,GAAG,OAAO,EAAE,EAC1C,wCAAwC,CACzC,CAAC;oBACF,SAAS;gBACX,CAAC;YACH,CAAC;YAED,kGAAkG;YAClG,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;gBACvB,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,UAAU,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC;gBACnE,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,UAAU,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC;gBACnE,OAAO,IAAI,GAAG,IAAI,CAAC;YACrB,CAAC,CAAC,CAAC;YAEH,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,MAAM;gBACZ,QAAQ;gBACR,KAAK,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;gBACnC,WAAW,EAAE,IAAI;aAClB,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,wDAAwD;YACxD,KAAK,MAAM,KAAK,IAAI,aAAa,EAAE,CAAC;gBAClC,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,MAAM;oBACZ,QAAQ,EAAE,KAAK,CAAC,IAAI,CAAC,QAAQ;oBAC7B,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC;oBAClB,WAAW,EAAE,KAAK;iBACnB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"} \ No newline at end of file diff --git a/worker/dist/archive/rar-reader.d.ts b/worker/dist/archive/rar-reader.d.ts deleted file mode 100644 index 3053cdb..0000000 --- a/worker/dist/archive/rar-reader.d.ts +++ /dev/null @@ -1,6 +0,0 @@ -import type { FileEntry } from "./zip-reader.js"; -/** - * Parse output of `unrar l -v ` to extract file metadata. - * unrar automatically discovers sibling parts when they're co-located. - */ -export declare function readRarContents(firstPartPath: string): Promise; diff --git a/worker/dist/archive/rar-reader.js b/worker/dist/archive/rar-reader.js deleted file mode 100644 index d541f5d..0000000 --- a/worker/dist/archive/rar-reader.js +++ /dev/null @@ -1,77 +0,0 @@ -import { execFile } from "child_process"; -import { promisify } from "util"; -import path from "path"; -import { childLogger } from "../util/logger.js"; -const execFileAsync = promisify(execFile); -const log = childLogger("rar-reader"); -/** - * Parse output of `unrar l -v ` to extract file metadata. - * unrar automatically discovers sibling parts when they're co-located. - */ -export async function readRarContents(firstPartPath) { - try { - const { stdout } = await execFileAsync("unrar", ["l", "-v", firstPartPath], { - timeout: 30000, - maxBuffer: 10 * 1024 * 1024, // 10MB for very large archives - }); - return parseUnrarOutput(stdout); - } - catch (err) { - log.warn({ err, file: firstPartPath }, "Failed to read RAR contents"); - return []; // Fallback: return empty on error - } -} -/** - * Parse the tabular output of `unrar l -v`. - * - * Example output format: - * Archive: test.rar - * Details: RAR 5 - * - * Attributes Size Packed Ratio Date Time CRC-32 Name - * ----------- --------- --------- ----- -------- ----- -------- ---- - * ...A.... 12345 10234 83% 2024-01-15 10:30 DEADBEEF folder/file.stl - * ----------- --------- --------- ----- -------- ----- -------- ---- - */ -function parseUnrarOutput(output) { - const entries = []; - const lines = output.split("\n"); - let inFileList = false; - let separatorCount = 0; - for (const line of lines) { - const trimmed = line.trim(); - // Detect separator lines (------- pattern) - if (/^-{5,}/.test(trimmed)) { - separatorCount++; - if (separatorCount === 1) { - inFileList = true; - } - else if (separatorCount >= 2) { - inFileList = false; - } - continue; - } - if (!inFileList) - continue; - // Parse file entry line - // Format: Attributes Size Packed Ratio Date Time CRC Name - const match = trimmed.match(/^\S+\s+(\d+)\s+(\d+)\s+\d+%\s+\S+\s+\S+\s+([0-9A-Fa-f]+)\s+(.+)$/); - if (match) { - const [, uncompressedStr, compressedStr, crc32, filePath] = match; - // Skip directory entries (typically end with / or have size 0 with dir attributes) - if (filePath.endsWith("/") || filePath.endsWith("\\")) - continue; - const ext = path.extname(filePath).toLowerCase(); - entries.push({ - path: filePath, - fileName: path.basename(filePath), - extension: ext ? ext.slice(1) : null, - compressedSize: BigInt(compressedStr), - uncompressedSize: BigInt(uncompressedStr), - crc32: crc32.toLowerCase(), - }); - } - } - return entries; -} -//# sourceMappingURL=rar-reader.js.map \ No newline at end of file diff --git a/worker/dist/archive/rar-reader.js.map b/worker/dist/archive/rar-reader.js.map deleted file mode 100644 index 35d94a4..0000000 --- a/worker/dist/archive/rar-reader.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"rar-reader.js","sourceRoot":"","sources":["../../src/archive/rar-reader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,MAAM,CAAC;AACjC,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAGhD,MAAM,aAAa,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;AAC1C,MAAM,GAAG,GAAG,WAAW,CAAC,YAAY,CAAC,CAAC;AAEtC;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,aAAqB;IAErB,IAAI,CAAC;QACH,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,aAAa,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,IAAI,EAAE,aAAa,CAAC,EAAE;YAC1E,OAAO,EAAE,KAAK;YACd,SAAS,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI,EAAE,+BAA+B;SAC7D,CAAC,CAAC;QAEH,OAAO,gBAAgB,CAAC,MAAM,CAAC,CAAC;IAClC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,aAAa,EAAE,EAAE,6BAA6B,CAAC,CAAC;QACtE,OAAO,EAAE,CAAC,CAAC,kCAAkC;IAC/C,CAAC;AACH,CAAC;AAED;;;;;;;;;;;GAWG;AACH,SAAS,gBAAgB,CAAC,MAAc;IACtC,MAAM,OAAO,GAAgB,EAAE,CAAC;IAChC,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAEjC,IAAI,UAAU,GAAG,KAAK,CAAC;IACvB,IAAI,cAAc,GAAG,CAAC,CAAC;IAEvB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAE5B,2CAA2C;QAC3C,IAAI,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAC3B,cAAc,EAAE,CAAC;YACjB,IAAI,cAAc,KAAK,CAAC,EAAE,CAAC;gBACzB,UAAU,GAAG,IAAI,CAAC;YACpB,CAAC;iBAAM,IAAI,cAAc,IAAI,CAAC,EAAE,CAAC;gBAC/B,UAAU,GAAG,KAAK,CAAC;YACrB,CAAC;YACD,SAAS;QACX,CAAC;QAED,IAAI,CAAC,UAAU;YAAE,SAAS;QAE1B,wBAAwB;QACxB,0DAA0D;QAC1D,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CACzB,kEAAkE,CACnE,CAAC;QAEF,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,CAAC,EAAE,eAAe,EAAE,aAAa,EAAE,KAAK,EAAE,QAAQ,CAAC,GAAG,KAAK,CAAC;YAElE,mFAAmF;YACnF,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC;gBAAE,SAAS;YAEhE,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;YACjD,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,QAAQ;gBACd,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBACjC,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI;gBACpC,cAAc,EAAE,MAAM,CAAC,aAAa,CAAC;gBACrC,gBAAgB,EAAE,MAAM,CAAC,eAAe,CAAC;gBACzC,KAAK,EAAE,KAAK,CAAC,WAAW,EAAE;aAC3B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"} \ No newline at end of file diff --git a/worker/dist/archive/split.d.ts b/worker/dist/archive/split.d.ts deleted file mode 100644 index 67a31ae..0000000 --- a/worker/dist/archive/split.d.ts +++ /dev/null @@ -1,11 +0,0 @@ -/** - * Split a file into ≤2GB parts using byte-level splitting. - * Returns paths to the split parts. If the file is already ≤2GB, returns the original path. - */ -export declare function byteLevelSplit(filePath: string): Promise; -/** - * Concatenate multiple files into a single output file by streaming - * each input sequentially. Used for repacking multipart archives - * that have oversized parts (>2GB) before re-splitting. - */ -export declare function concatenateFiles(inputPaths: string[], outputPath: string): Promise; diff --git a/worker/dist/archive/split.js b/worker/dist/archive/split.js deleted file mode 100644 index ec67450..0000000 --- a/worker/dist/archive/split.js +++ /dev/null @@ -1,55 +0,0 @@ -import { createReadStream, createWriteStream } from "fs"; -import { stat } from "fs/promises"; -import path from "path"; -import { pipeline } from "stream/promises"; -import { childLogger } from "../util/logger.js"; -const log = childLogger("split"); -/** 2GB in bytes — Telegram's file size limit */ -const MAX_PART_SIZE = 2n * 1024n * 1024n * 1024n; -/** - * Split a file into ≤2GB parts using byte-level splitting. - * Returns paths to the split parts. If the file is already ≤2GB, returns the original path. - */ -export async function byteLevelSplit(filePath) { - const stats = await stat(filePath); - const fileSize = BigInt(stats.size); - if (fileSize <= MAX_PART_SIZE) { - return [filePath]; - } - const dir = path.dirname(filePath); - const baseName = path.basename(filePath); - const partSize = Number(MAX_PART_SIZE); - const totalParts = Math.ceil(Number(fileSize) / partSize); - const parts = []; - log.info({ filePath, fileSize: Number(fileSize), totalParts }, "Splitting file"); - for (let i = 0; i < totalParts; i++) { - const partNum = String(i + 1).padStart(3, "0"); - const partPath = path.join(dir, `${baseName}.${partNum}`); - const start = i * partSize; - const end = Math.min(start + partSize - 1, Number(fileSize) - 1); - await pipeline(createReadStream(filePath, { start, end }), createWriteStream(partPath)); - parts.push(partPath); - } - log.info({ filePath, parts: parts.length }, "File split complete"); - return parts; -} -/** - * Concatenate multiple files into a single output file by streaming - * each input sequentially. Used for repacking multipart archives - * that have oversized parts (>2GB) before re-splitting. - */ -export async function concatenateFiles(inputPaths, outputPath) { - const out = createWriteStream(outputPath); - for (let i = 0; i < inputPaths.length; i++) { - log.info({ part: i + 1, total: inputPaths.length, file: path.basename(inputPaths[i]) }, "Concatenating part"); - await pipeline(createReadStream(inputPaths[i]), out, { end: false }); - } - // Close the output stream - await new Promise((resolve, reject) => { - out.end(() => resolve()); - out.on("error", reject); - }); - const stats = await stat(outputPath); - log.info({ outputPath, totalBytes: stats.size, parts: inputPaths.length }, "Concatenation complete"); -} -//# sourceMappingURL=split.js.map \ No newline at end of file diff --git a/worker/dist/archive/split.js.map b/worker/dist/archive/split.js.map deleted file mode 100644 index 79031bb..0000000 --- a/worker/dist/archive/split.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"split.js","sourceRoot":"","sources":["../../src/archive/split.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,IAAI,CAAC;AACzD,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AACnC,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAC3C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,MAAM,GAAG,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;AAEjC,gDAAgD;AAChD,MAAM,aAAa,GAAG,EAAE,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,CAAC;AAEjD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,QAAgB;IACnD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;IACnC,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAEpC,IAAI,QAAQ,IAAI,aAAa,EAAE,CAAC;QAC9B,OAAO,CAAC,QAAQ,CAAC,CAAC;IACpB,CAAC;IAED,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACzC,MAAM,QAAQ,GAAG,MAAM,CAAC,aAAa,CAAC,CAAC;IACvC,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,QAAQ,CAAC,CAAC;IAC1D,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,GAAG,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC,EAAE,UAAU,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAEjF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,MAAM,OAAO,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAC/C,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,IAAI,OAAO,EAAE,CAAC,CAAC;QAC1D,MAAM,KAAK,GAAG,CAAC,GAAG,QAAQ,CAAC;QAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,QAAQ,GAAG,CAAC,EAAE,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;QAEjE,MAAM,QAAQ,CACZ,gBAAgB,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,EAC1C,iBAAiB,CAAC,QAAQ,CAAC,CAC5B,CAAC;QAEF,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACvB,CAAC;IAED,GAAG,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,EAAE,qBAAqB,CAAC,CAAC;IACnE,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,UAAoB,EACpB,UAAkB;IAElB,MAAM,GAAG,GAAG,iBAAiB,CAAC,UAAU,CAAC,CAAC;IAE1C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3C,GAAG,CAAC,IAAI,CACN,EAAE,IAAI,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,UAAU,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,EAC7E,oBAAoB,CACrB,CAAC;QACF,MAAM,QAAQ,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC;IACvE,CAAC;IAED,0BAA0B;IAC1B,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC1C,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;QACzB,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAC1B,CAAC,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC;IACrC,GAAG,CAAC,IAAI,CACN,EAAE,UAAU,EAAE,UAAU,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,UAAU,CAAC,MAAM,EAAE,EAChE,wBAAwB,CACzB,CAAC;AACJ,CAAC"} \ No newline at end of file diff --git a/worker/dist/archive/zip-reader.d.ts b/worker/dist/archive/zip-reader.d.ts deleted file mode 100644 index f721913..0000000 --- a/worker/dist/archive/zip-reader.d.ts +++ /dev/null @@ -1,15 +0,0 @@ -export interface FileEntry { - path: string; - fileName: string; - extension: string | null; - compressedSize: bigint; - uncompressedSize: bigint; - crc32: string | null; -} -/** - * Read the central directory of a ZIP file without extracting any contents. - * For multipart ZIPs (.zip.001, .zip.002 etc.), uses a custom random-access - * reader that spans all parts seamlessly so yauzl can find the central - * directory at the end of the combined data. - */ -export declare function readZipCentralDirectory(filePaths: string[]): Promise; diff --git a/worker/dist/archive/zip-reader.js b/worker/dist/archive/zip-reader.js deleted file mode 100644 index 6e70b3f..0000000 --- a/worker/dist/archive/zip-reader.js +++ /dev/null @@ -1,161 +0,0 @@ -import yauzl from "yauzl"; -import { open as fsOpen, stat as fsStat } from "fs/promises"; -import path from "path"; -import { Readable } from "stream"; -import { childLogger } from "../util/logger.js"; -const log = childLogger("zip-reader"); -/** - * Read the central directory of a ZIP file without extracting any contents. - * For multipart ZIPs (.zip.001, .zip.002 etc.), uses a custom random-access - * reader that spans all parts seamlessly so yauzl can find the central - * directory at the end of the combined data. - */ -export async function readZipCentralDirectory(filePaths) { - if (filePaths.length === 1) { - return readSingleZip(filePaths[0]); - } - // Multipart: use a spanning random-access reader - return readMultipartZip(filePaths); -} -/** Read a single (non-split) ZIP file. */ -function readSingleZip(targetFile) { - return new Promise((resolve) => { - yauzl.open(targetFile, { lazyEntries: true, autoClose: true }, (err, zipFile) => { - if (err) { - log.warn({ err, file: targetFile }, "Failed to open ZIP for reading"); - resolve([]); - return; - } - const entries = []; - zipFile.readEntry(); - zipFile.on("entry", (entry) => { - if (!entry.fileName.endsWith("/")) { - const ext = path.extname(entry.fileName).toLowerCase(); - entries.push({ - path: entry.fileName, - fileName: path.basename(entry.fileName), - extension: ext ? ext.slice(1) : null, - compressedSize: BigInt(entry.compressedSize), - uncompressedSize: BigInt(entry.uncompressedSize), - crc32: entry.crc32 !== 0 ? entry.crc32.toString(16).padStart(8, "0") : null, - }); - } - zipFile.readEntry(); - }); - zipFile.on("end", () => resolve(entries)); - zipFile.on("error", (error) => { - log.warn({ error, file: targetFile }, "Error reading ZIP entries"); - resolve(entries); - }); - }); - }); -} -/** - * Read a multipart split ZIP using yauzl's RandomAccessReader API. - * This creates a virtual "file" that spans all parts so yauzl can - * seek freely across the entire archive to read the central directory. - */ -async function readMultipartZip(filePaths) { - // Get sizes of all parts - const partSizes = []; - for (const fp of filePaths) { - const s = await fsStat(fp); - partSizes.push(s.size); - } - const totalSize = partSizes.reduce((a, b) => a + b, 0); - log.debug({ parts: filePaths.length, totalSize }, "Reading multipart ZIP via spanning reader"); - return new Promise((resolve) => { - const reader = createMultiPartReader(filePaths, partSizes); - yauzl.fromRandomAccessReader(reader, totalSize, { lazyEntries: true, autoClose: true }, (err, zipFile) => { - if (err) { - log.warn({ err }, "Failed to open multipart ZIP for reading"); - reader.close(() => { }); - resolve([]); - return; - } - const entries = []; - zipFile.readEntry(); - zipFile.on("entry", (entry) => { - if (!entry.fileName.endsWith("/")) { - const ext = path.extname(entry.fileName).toLowerCase(); - entries.push({ - path: entry.fileName, - fileName: path.basename(entry.fileName), - extension: ext ? ext.slice(1) : null, - compressedSize: BigInt(entry.compressedSize), - uncompressedSize: BigInt(entry.uncompressedSize), - crc32: entry.crc32 !== 0 ? entry.crc32.toString(16).padStart(8, "0") : null, - }); - } - zipFile.readEntry(); - }); - zipFile.on("end", () => { - log.info({ entries: entries.length }, "Multipart ZIP entries read"); - resolve(entries); - }); - zipFile.on("error", (error) => { - log.warn({ error }, "Error reading multipart ZIP entries"); - resolve(entries); - }); - }); - }); -} -/** - * Create a yauzl RandomAccessReader that reads across multiple split part files. - * Maps a global offset to the correct part file and local offset. - * - * Uses Object.create to properly inherit from yauzl.RandomAccessReader - * (whose constructor + prototype is defined at runtime, not as a TS class). - */ -function createMultiPartReader(filePaths, partSizes) { - // Build cumulative offset table - const partOffsets = []; - let offset = 0; - for (const size of partSizes) { - partOffsets.push(offset); - offset += size; - } - // Create an instance by calling the parent constructor - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const reader = new yauzl.RandomAccessReader(); - // Override _readStreamForRange — yauzl calls this to read a range of bytes - // eslint-disable-next-line @typescript-eslint/no-explicit-any - reader._readStreamForRange = function (start, end) { - const readable = new Readable({ read() { } }); - readRange(start, end, readable).catch((err) => { - readable.destroy(err); - }); - return readable; - }; - async function readRange(start, end, readable) { - let remaining = end - start; - let globalOffset = start; - while (remaining > 0) { - // Find which part this offset falls in - let partIdx = partOffsets.length - 1; - for (let i = 0; i < partOffsets.length; i++) { - if (i + 1 < partOffsets.length && globalOffset < partOffsets[i + 1]) { - partIdx = i; - break; - } - } - const localOffset = globalOffset - partOffsets[partIdx]; - const partRemaining = partSizes[partIdx] - localOffset; - const toRead = Math.min(remaining, partRemaining); - const fh = await fsOpen(filePaths[partIdx], "r"); - try { - const buf = Buffer.alloc(toRead); - const { bytesRead } = await fh.read(buf, 0, toRead, localOffset); - readable.push(buf.subarray(0, bytesRead)); - remaining -= bytesRead; - globalOffset += bytesRead; - } - finally { - await fh.close(); - } - } - readable.push(null); // Signal end of stream - } - return reader; -} -//# sourceMappingURL=zip-reader.js.map \ No newline at end of file diff --git a/worker/dist/archive/zip-reader.js.map b/worker/dist/archive/zip-reader.js.map deleted file mode 100644 index 7af95cd..0000000 --- a/worker/dist/archive/zip-reader.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"zip-reader.js","sourceRoot":"","sources":["../../src/archive/zip-reader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,IAAI,IAAI,MAAM,EAAE,IAAI,IAAI,MAAM,EAAE,MAAM,aAAa,CAAC;AAC7D,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,MAAM,GAAG,GAAG,WAAW,CAAC,YAAY,CAAC,CAAC;AAWtC;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,SAAmB;IAEnB,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,OAAO,aAAa,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;IACrC,CAAC;IAED,iDAAiD;IACjD,OAAO,gBAAgB,CAAC,SAAS,CAAC,CAAC;AACrC,CAAC;AAED,0CAA0C;AAC1C,SAAS,aAAa,CAAC,UAAkB;IACvC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE;YAC9E,IAAI,GAAG,EAAE,CAAC;gBACR,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,gCAAgC,CAAC,CAAC;gBACtE,OAAO,CAAC,EAAE,CAAC,CAAC;gBACZ,OAAO;YACT,CAAC;YAED,MAAM,OAAO,GAAgB,EAAE,CAAC;YAEhC,OAAO,CAAC,SAAS,EAAE,CAAC;YACpB,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAkB,EAAE,EAAE;gBACzC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;oBAClC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;oBACvD,OAAO,CAAC,IAAI,CAAC;wBACX,IAAI,EAAE,KAAK,CAAC,QAAQ;wBACpB,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC;wBACvC,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI;wBACpC,cAAc,EAAE,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC;wBAC5C,gBAAgB,EAAE,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC;wBAChD,KAAK,EAAE,KAAK,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI;qBAC5E,CAAC,CAAC;gBACL,CAAC;gBACD,OAAO,CAAC,SAAS,EAAE,CAAC;YACtB,CAAC,CAAC,CAAC;YAEH,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;YAC1C,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;gBAC5B,GAAG,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,2BAA2B,CAAC,CAAC;gBACnE,OAAO,CAAC,OAAO,CAAC,CAAC;YACnB,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,gBAAgB,CAAC,SAAmB;IACjD,yBAAyB;IACzB,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,KAAK,MAAM,EAAE,IAAI,SAAS,EAAE,CAAC;QAC3B,MAAM,CAAC,GAAG,MAAM,MAAM,CAAC,EAAE,CAAC,CAAC;QAC3B,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACzB,CAAC;IACD,MAAM,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IAEvD,GAAG,CAAC,KAAK,CACP,EAAE,KAAK,EAAE,SAAS,CAAC,MAAM,EAAE,SAAS,EAAE,EACtC,2CAA2C,CAC5C,CAAC;IAEF,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,MAAM,MAAM,GAAG,qBAAqB,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QAE3D,KAAK,CAAC,sBAAsB,CAC1B,MAAM,EACN,SAAS,EACT,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,EACtC,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE;YACf,IAAI,GAAG,EAAE,CAAC;gBACR,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,EAAE,0CAA0C,CAAC,CAAC;gBAC9D,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;gBACvB,OAAO,CAAC,EAAE,CAAC,CAAC;gBACZ,OAAO;YACT,CAAC;YAED,MAAM,OAAO,GAAgB,EAAE,CAAC;YAEhC,OAAO,CAAC,SAAS,EAAE,CAAC;YACpB,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAkB,EAAE,EAAE;gBACzC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;oBAClC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;oBACvD,OAAO,CAAC,IAAI,CAAC;wBACX,IAAI,EAAE,KAAK,CAAC,QAAQ;wBACpB,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC;wBACvC,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI;wBACpC,cAAc,EAAE,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC;wBAC5C,gBAAgB,EAAE,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC;wBAChD,KAAK,EAAE,KAAK,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI;qBAC5E,CAAC,CAAC;gBACL,CAAC;gBACD,OAAO,CAAC,SAAS,EAAE,CAAC;YACtB,CAAC,CAAC,CAAC;YAEH,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBACrB,GAAG,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,EAAE,4BAA4B,CAAC,CAAC;gBACpE,OAAO,CAAC,OAAO,CAAC,CAAC;YACnB,CAAC,CAAC,CAAC;YACH,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;gBAC5B,GAAG,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,EAAE,qCAAqC,CAAC,CAAC;gBAC3D,OAAO,CAAC,OAAO,CAAC,CAAC;YACnB,CAAC,CAAC,CAAC;QACL,CAAC,CACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;;;GAMG;AACH,SAAS,qBAAqB,CAC5B,SAAmB,EACnB,SAAmB;IAEnB,gCAAgC;IAChC,MAAM,WAAW,GAAa,EAAE,CAAC;IACjC,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACzB,MAAM,IAAI,IAAI,CAAC;IACjB,CAAC;IAED,uDAAuD;IACvD,8DAA8D;IAC9D,MAAM,MAAM,GAAG,IAAK,KAAK,CAAC,kBAA0B,EAA8B,CAAC;IAEnF,2EAA2E;IAC3E,8DAA8D;IAC7D,MAAc,CAAC,mBAAmB,GAAG,UAAU,KAAa,EAAE,GAAW;QACxE,MAAM,QAAQ,GAAG,IAAI,QAAQ,CAAC,EAAE,IAAI,KAAI,CAAC,EAAE,CAAC,CAAC;QAE7C,SAAS,CAAC,KAAK,EAAE,GAAG,EAAE,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YAC5C,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACxB,CAAC,CAAC,CAAC;QAEH,OAAO,QAAQ,CAAC;IAClB,CAAC,CAAC;IAEF,KAAK,UAAU,SAAS,CAAC,KAAa,EAAE,GAAW,EAAE,QAAkB;QACrE,IAAI,SAAS,GAAG,GAAG,GAAG,KAAK,CAAC;QAC5B,IAAI,YAAY,GAAG,KAAK,CAAC;QAEzB,OAAO,SAAS,GAAG,CAAC,EAAE,CAAC;YACrB,uCAAuC;YACvC,IAAI,OAAO,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC;YACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC5C,IAAI,CAAC,GAAG,CAAC,GAAG,WAAW,CAAC,MAAM,IAAI,YAAY,GAAG,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;oBACpE,OAAO,GAAG,CAAC,CAAC;oBACZ,MAAM;gBACR,CAAC;YACH,CAAC;YAED,MAAM,WAAW,GAAG,YAAY,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;YACxD,MAAM,aAAa,GAAG,SAAS,CAAC,OAAO,CAAC,GAAG,WAAW,CAAC;YACvD,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;YAElD,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,GAAG,CAAC,CAAC;YACjD,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBACjC,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;gBACjE,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC;gBAC1C,SAAS,IAAI,SAAS,CAAC;gBACvB,YAAY,IAAI,SAAS,CAAC;YAC5B,CAAC;oBAAS,CAAC;gBACT,MAAM,EAAE,CAAC,KAAK,EAAE,CAAC;YACnB,CAAC;QACH,CAAC;QAED,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,uBAAuB;IAC9C,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"} \ No newline at end of file diff --git a/worker/dist/db/client.d.ts b/worker/dist/db/client.d.ts deleted file mode 100644 index f99ad7c..0000000 --- a/worker/dist/db/client.d.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { PrismaClient } from "@prisma/client"; -import { PrismaPg } from "@prisma/adapter-pg"; -declare const pool: import("pg").Pool; -export declare const db: PrismaClient<{ - adapter: PrismaPg; -}, never, import("@prisma/client/runtime/client").DefaultArgs>; -export { pool }; diff --git a/worker/dist/db/client.js b/worker/dist/db/client.js deleted file mode 100644 index 2307f7c..0000000 --- a/worker/dist/db/client.js +++ /dev/null @@ -1,12 +0,0 @@ -import { PrismaClient } from "@prisma/client"; -import { PrismaPg } from "@prisma/adapter-pg"; -import pg from "pg"; -import { config } from "../util/config.js"; -const pool = new pg.Pool({ - connectionString: config.databaseUrl, - max: 5, -}); -const adapter = new PrismaPg(pool); -export const db = new PrismaClient({ adapter }); -export { pool }; -//# sourceMappingURL=client.js.map \ No newline at end of file diff --git a/worker/dist/db/client.js.map b/worker/dist/db/client.js.map deleted file mode 100644 index 5508b5b..0000000 --- a/worker/dist/db/client.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"client.js","sourceRoot":"","sources":["../../src/db/client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,MAAM,IAAI,GAAG,IAAI,EAAE,CAAC,IAAI,CAAC;IACvB,gBAAgB,EAAE,MAAM,CAAC,WAAW;IACpC,GAAG,EAAE,CAAC;CACP,CAAC,CAAC;AAEH,MAAM,OAAO,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC;AACnC,MAAM,CAAC,MAAM,EAAE,GAAG,IAAI,YAAY,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC;AAEhD,OAAO,EAAE,IAAI,EAAE,CAAC"} \ No newline at end of file diff --git a/worker/dist/db/locks.d.ts b/worker/dist/db/locks.d.ts deleted file mode 100644 index 646c184..0000000 --- a/worker/dist/db/locks.d.ts +++ /dev/null @@ -1,9 +0,0 @@ -/** - * Try to acquire a PostgreSQL advisory lock for an account. - * Returns true if acquired, false if already held by another session. - */ -export declare function tryAcquireLock(accountId: string): Promise; -/** - * Release the advisory lock for an account. - */ -export declare function releaseLock(accountId: string): Promise; diff --git a/worker/dist/db/locks.js b/worker/dist/db/locks.js deleted file mode 100644 index 926011d..0000000 --- a/worker/dist/db/locks.js +++ /dev/null @@ -1,53 +0,0 @@ -import { pool } from "./client.js"; -import { childLogger } from "../util/logger.js"; -const log = childLogger("locks"); -/** - * Derive a stable 32-bit integer lock ID from an account ID string. - * PostgreSQL advisory locks use bigint, but we use 32-bit for safety. - */ -function hashToLockId(accountId) { - let hash = 0; - for (let i = 0; i < accountId.length; i++) { - const char = accountId.charCodeAt(i); - hash = (hash << 5) - hash + char; - hash |= 0; // Convert to 32-bit integer - } - return Math.abs(hash); -} -/** - * Try to acquire a PostgreSQL advisory lock for an account. - * Returns true if acquired, false if already held by another session. - */ -export async function tryAcquireLock(accountId) { - const lockId = hashToLockId(accountId); - const client = await pool.connect(); - try { - const result = await client.query("SELECT pg_try_advisory_lock($1)", [lockId]); - const acquired = result.rows[0]?.pg_try_advisory_lock ?? false; - if (acquired) { - log.debug({ accountId, lockId }, "Advisory lock acquired"); - } - else { - log.debug({ accountId, lockId }, "Advisory lock already held"); - } - return acquired; - } - finally { - client.release(); - } -} -/** - * Release the advisory lock for an account. - */ -export async function releaseLock(accountId) { - const lockId = hashToLockId(accountId); - const client = await pool.connect(); - try { - await client.query("SELECT pg_advisory_unlock($1)", [lockId]); - log.debug({ accountId, lockId }, "Advisory lock released"); - } - finally { - client.release(); - } -} -//# sourceMappingURL=locks.js.map \ No newline at end of file diff --git a/worker/dist/db/locks.js.map b/worker/dist/db/locks.js.map deleted file mode 100644 index 2cade3d..0000000 --- a/worker/dist/db/locks.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"locks.js","sourceRoot":"","sources":["../../src/db/locks.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AACnC,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,MAAM,GAAG,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;AAEjC;;;GAGG;AACH,SAAS,YAAY,CAAC,SAAiB;IACrC,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,MAAM,IAAI,GAAG,SAAS,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QACrC,IAAI,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;QACjC,IAAI,IAAI,CAAC,CAAC,CAAC,4BAA4B;IACzC,CAAC;IACD,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;AACxB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,SAAiB;IACpD,MAAM,MAAM,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC;IACvC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;IACpC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,KAAK,CAC/B,iCAAiC,EACjC,CAAC,MAAM,CAAC,CACT,CAAC;QACF,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,oBAAoB,IAAI,KAAK,CAAC;QAC/D,IAAI,QAAQ,EAAE,CAAC;YACb,GAAG,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,wBAAwB,CAAC,CAAC;QAC7D,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,4BAA4B,CAAC,CAAC;QACjE,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;YAAS,CAAC;QACT,MAAM,CAAC,OAAO,EAAE,CAAC;IACnB,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,SAAiB;IACjD,MAAM,MAAM,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC;IACvC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;IACpC,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,KAAK,CAAC,+BAA+B,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC;QAC9D,GAAG,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,wBAAwB,CAAC,CAAC;IAC7D,CAAC;YAAS,CAAC;QACT,MAAM,CAAC,OAAO,EAAE,CAAC;IACnB,CAAC;AACH,CAAC"} \ No newline at end of file diff --git a/worker/dist/db/queries.d.ts b/worker/dist/db/queries.d.ts deleted file mode 100644 index 7e6f292..0000000 --- a/worker/dist/db/queries.d.ts +++ /dev/null @@ -1,356 +0,0 @@ -import type { ArchiveType, FetchStatus } from "@prisma/client"; -export declare function getActiveAccounts(): Promise<{ - id: string; - phone: string; - displayName: string | null; - isActive: boolean; - authState: import("@prisma/client").$Enums.AuthState; - authCode: string | null; - lastSeenAt: Date | null; - createdAt: Date; - updatedAt: Date; -}[]>; -export declare function getPendingAccounts(): Promise<{ - id: string; - phone: string; - displayName: string | null; - isActive: boolean; - authState: import("@prisma/client").$Enums.AuthState; - authCode: string | null; - lastSeenAt: Date | null; - createdAt: Date; - updatedAt: Date; -}[]>; -export declare function hasAnyChannels(): Promise; -export declare function getSourceChannelMappings(accountId: string): Promise<({ - channel: { - id: string; - isActive: boolean; - createdAt: Date; - updatedAt: Date; - telegramId: bigint; - title: string; - type: import("@prisma/client").$Enums.ChannelType; - isForum: boolean; - }; -} & { - accountId: string; - id: string; - createdAt: Date; - channelId: string; - role: import("@prisma/client").$Enums.ChannelRole; - lastProcessedMessageId: bigint | null; -})[]>; -export declare function getGlobalDestinationChannel(): Promise<{ - id: string; - isActive: boolean; - createdAt: Date; - updatedAt: Date; - telegramId: bigint; - title: string; - type: import("@prisma/client").$Enums.ChannelType; - isForum: boolean; -} | null>; -export declare function getGlobalSetting(key: string): Promise; -export declare function setGlobalSetting(key: string, value: string): Promise<{ - updatedAt: Date; - key: string; - value: string; -}>; -export declare function packageExistsByHash(contentHash: string): Promise; -/** - * Check if a package already exists for a given source message ID - * AND was successfully uploaded to the destination (destMessageId is set). - * Used as an early skip before downloading. - */ -export declare function packageExistsBySourceMessage(sourceChannelId: string, sourceMessageId: bigint): Promise; -/** - * Delete orphaned Package rows that have the same content hash but never - * completed the upload (destMessageId is null). Called before creating a - * new complete record to avoid unique constraint violations. - */ -export declare function deleteOrphanedPackageByHash(contentHash: string): Promise; -export interface CreatePackageInput { - contentHash: string; - fileName: string; - fileSize: bigint; - archiveType: ArchiveType; - sourceChannelId: string; - sourceMessageId: bigint; - sourceTopicId?: bigint | null; - destChannelId?: string; - destMessageId?: bigint; - isMultipart: boolean; - partCount: number; - ingestionRunId: string; - creator?: string | null; - previewData?: Buffer | null; - previewMsgId?: bigint | null; - files: { - path: string; - fileName: string; - extension: string | null; - compressedSize: bigint; - uncompressedSize: bigint; - crc32: string | null; - }[]; -} -export declare function createPackageWithFiles(input: CreatePackageInput): Promise<{ - id: string; - createdAt: Date; - contentHash: string; - fileName: string; - fileSize: bigint; - archiveType: import("@prisma/client").$Enums.ArchiveType; - creator: string | null; - sourceChannelId: string; - sourceMessageId: bigint; - sourceTopicId: bigint | null; - destChannelId: string | null; - destMessageId: bigint | null; - isMultipart: boolean; - partCount: number; - fileCount: number; - previewData: import("@prisma/client/runtime/client").Bytes | null; - previewMsgId: bigint | null; - indexedAt: Date; - ingestionRunId: string | null; -}>; -export declare function createIngestionRun(accountId: string): Promise<{ - accountId: string; - id: string; - status: import("@prisma/client").$Enums.IngestionStatus; - startedAt: Date; - finishedAt: Date | null; - messagesScanned: number; - zipsFound: number; - zipsDuplicate: number; - zipsIngested: number; - errorMessage: string | null; - currentActivity: string | null; - currentStep: string | null; - currentChannel: string | null; - currentFile: string | null; - currentFileNum: number | null; - totalFiles: number | null; - downloadedBytes: bigint | null; - totalBytes: bigint | null; - downloadPercent: number | null; - lastActivityAt: Date | null; -}>; -export interface ActivityUpdate { - currentActivity: string; - currentStep: string; - currentChannel?: string | null; - currentFile?: string | null; - currentFileNum?: number | null; - totalFiles?: number | null; - downloadedBytes?: bigint | null; - totalBytes?: bigint | null; - downloadPercent?: number | null; - messagesScanned?: number; - zipsFound?: number; - zipsDuplicate?: number; - zipsIngested?: number; -} -export declare function updateRunActivity(runId: string, activity: ActivityUpdate): Promise<{ - accountId: string; - id: string; - status: import("@prisma/client").$Enums.IngestionStatus; - startedAt: Date; - finishedAt: Date | null; - messagesScanned: number; - zipsFound: number; - zipsDuplicate: number; - zipsIngested: number; - errorMessage: string | null; - currentActivity: string | null; - currentStep: string | null; - currentChannel: string | null; - currentFile: string | null; - currentFileNum: number | null; - totalFiles: number | null; - downloadedBytes: bigint | null; - totalBytes: bigint | null; - downloadPercent: number | null; - lastActivityAt: Date | null; -}>; -export declare function completeIngestionRun(runId: string, counters: { - messagesScanned: number; - zipsFound: number; - zipsDuplicate: number; - zipsIngested: number; -}): Promise<{ - accountId: string; - id: string; - status: import("@prisma/client").$Enums.IngestionStatus; - startedAt: Date; - finishedAt: Date | null; - messagesScanned: number; - zipsFound: number; - zipsDuplicate: number; - zipsIngested: number; - errorMessage: string | null; - currentActivity: string | null; - currentStep: string | null; - currentChannel: string | null; - currentFile: string | null; - currentFileNum: number | null; - totalFiles: number | null; - downloadedBytes: bigint | null; - totalBytes: bigint | null; - downloadPercent: number | null; - lastActivityAt: Date | null; -}>; -export declare function failIngestionRun(runId: string, errorMessage: string): Promise<{ - accountId: string; - id: string; - status: import("@prisma/client").$Enums.IngestionStatus; - startedAt: Date; - finishedAt: Date | null; - messagesScanned: number; - zipsFound: number; - zipsDuplicate: number; - zipsIngested: number; - errorMessage: string | null; - currentActivity: string | null; - currentStep: string | null; - currentChannel: string | null; - currentFile: string | null; - currentFileNum: number | null; - totalFiles: number | null; - downloadedBytes: bigint | null; - totalBytes: bigint | null; - downloadPercent: number | null; - lastActivityAt: Date | null; -}>; -export declare function updateLastProcessedMessage(mappingId: string, messageId: bigint): Promise<{ - accountId: string; - id: string; - createdAt: Date; - channelId: string; - role: import("@prisma/client").$Enums.ChannelRole; - lastProcessedMessageId: bigint | null; -}>; -export declare function markStaleRunsAsFailed(): Promise; -export declare function updateAccountAuthState(accountId: string, authState: "PENDING" | "AWAITING_CODE" | "AWAITING_PASSWORD" | "AUTHENTICATED" | "EXPIRED", authCode?: string | null): Promise<{ - id: string; - phone: string; - displayName: string | null; - isActive: boolean; - authState: import("@prisma/client").$Enums.AuthState; - authCode: string | null; - lastSeenAt: Date | null; - createdAt: Date; - updatedAt: Date; -}>; -export declare function getAccountAuthCode(accountId: string): Promise<{ - authState: import("@prisma/client").$Enums.AuthState; - authCode: string | null; -} | null>; -export interface UpsertChannelInput { - telegramId: bigint; - title: string; - type: "SOURCE" | "DESTINATION"; - isForum: boolean; - isActive?: boolean; -} -/** - * Upsert a channel by telegramId. Returns the channel record. - * If it already exists, update title and forum status. - * New channels default to disabled (isActive: false) so the admin must - * explicitly enable them before the worker processes them. - * Pass isActive: true for DESTINATION channels that must be active immediately. - */ -export declare function upsertChannel(input: UpsertChannelInput): Promise<{ - id: string; - isActive: boolean; - createdAt: Date; - updatedAt: Date; - telegramId: bigint; - title: string; - type: import("@prisma/client").$Enums.ChannelType; - isForum: boolean; -}>; -/** - * Link an account to a channel if not already linked. - * Uses a try/catch on unique constraint to make it idempotent. - */ -export declare function ensureAccountChannelLink(accountId: string, channelId: string, role: "READER" | "WRITER"): Promise<{ - accountId: string; - id: string; - createdAt: Date; - channelId: string; - role: import("@prisma/client").$Enums.ChannelRole; - lastProcessedMessageId: bigint | null; -} | null>; -export declare function setChannelForum(channelId: string, isForum: boolean): Promise<{ - id: string; - isActive: boolean; - createdAt: Date; - updatedAt: Date; - telegramId: bigint; - title: string; - type: import("@prisma/client").$Enums.ChannelType; - isForum: boolean; -}>; -export declare function getTopicProgress(mappingId: string): Promise<{ - id: string; - lastProcessedMessageId: bigint | null; - accountChannelMapId: string; - topicId: bigint; - topicName: string | null; -}[]>; -export declare function upsertTopicProgress(mappingId: string, topicId: bigint, topicName: string | null, lastProcessedMessageId: bigint): Promise<{ - id: string; - lastProcessedMessageId: bigint | null; - accountChannelMapId: string; - topicId: bigint; - topicName: string | null; -}>; -export declare function getChannelFetchRequest(requestId: string): Promise<({ - account: { - id: string; - phone: string; - displayName: string | null; - isActive: boolean; - authState: import("@prisma/client").$Enums.AuthState; - authCode: string | null; - lastSeenAt: Date | null; - createdAt: Date; - updatedAt: Date; - }; -} & { - error: string | null; - accountId: string; - id: string; - createdAt: Date; - updatedAt: Date; - status: import("@prisma/client").$Enums.FetchStatus; - resultJson: string | null; -}) | null>; -export declare function updateFetchRequestStatus(requestId: string, status: FetchStatus, extra?: { - resultJson?: string; - error?: string; -}): Promise<{ - error: string | null; - accountId: string; - id: string; - createdAt: Date; - updatedAt: Date; - status: import("@prisma/client").$Enums.FetchStatus; - resultJson: string | null; -}>; -export declare function getAccountLinkedChannelIds(accountId: string): Promise>; -export declare function getExistingChannelsByTelegramId(): Promise>; -export declare function getAccountById(accountId: string): Promise<{ - id: string; - phone: string; - displayName: string | null; - isActive: boolean; - authState: import("@prisma/client").$Enums.AuthState; - authCode: string | null; - lastSeenAt: Date | null; - createdAt: Date; - updatedAt: Date; -} | null>; diff --git a/worker/dist/db/queries.js b/worker/dist/db/queries.js deleted file mode 100644 index b6e4850..0000000 --- a/worker/dist/db/queries.js +++ /dev/null @@ -1,319 +0,0 @@ -import { db } from "./client.js"; -export async function getActiveAccounts() { - return db.telegramAccount.findMany({ - where: { isActive: true, authState: "AUTHENTICATED" }, - }); -} -export async function getPendingAccounts() { - return db.telegramAccount.findMany({ - where: { isActive: true, authState: "PENDING" }, - }); -} -export async function hasAnyChannels() { - const count = await db.telegramChannel.count(); - return count > 0; -} -export async function getSourceChannelMappings(accountId) { - return db.accountChannelMap.findMany({ - where: { - accountId, - role: "READER", - channel: { type: "SOURCE", isActive: true }, - }, - include: { channel: true }, - }); -} -// ── Global destination channel ── -export async function getGlobalDestinationChannel() { - const setting = await db.globalSetting.findUnique({ - where: { key: "destination_channel_id" }, - }); - if (!setting) - return null; - return db.telegramChannel.findFirst({ - where: { id: setting.value, type: "DESTINATION", isActive: true }, - }); -} -export async function getGlobalSetting(key) { - const setting = await db.globalSetting.findUnique({ where: { key } }); - return setting?.value ?? null; -} -export async function setGlobalSetting(key, value) { - return db.globalSetting.upsert({ - where: { key }, - create: { key, value }, - update: { value }, - }); -} -export async function packageExistsByHash(contentHash) { - const pkg = await db.package.findFirst({ - where: { contentHash, destMessageId: { not: null } }, - select: { id: true }, - }); - return pkg !== null; -} -/** - * Check if a package already exists for a given source message ID - * AND was successfully uploaded to the destination (destMessageId is set). - * Used as an early skip before downloading. - */ -export async function packageExistsBySourceMessage(sourceChannelId, sourceMessageId) { - const pkg = await db.package.findFirst({ - where: { sourceChannelId, sourceMessageId, destMessageId: { not: null } }, - select: { id: true }, - }); - return pkg !== null; -} -/** - * Delete orphaned Package rows that have the same content hash but never - * completed the upload (destMessageId is null). Called before creating a - * new complete record to avoid unique constraint violations. - */ -export async function deleteOrphanedPackageByHash(contentHash) { - await db.package.deleteMany({ - where: { contentHash, destMessageId: null }, - }); -} -export async function createPackageWithFiles(input) { - const pkg = await db.package.create({ - data: { - contentHash: input.contentHash, - fileName: input.fileName, - fileSize: input.fileSize, - archiveType: input.archiveType, - sourceChannelId: input.sourceChannelId, - sourceMessageId: input.sourceMessageId, - sourceTopicId: input.sourceTopicId ?? undefined, - destChannelId: input.destChannelId, - destMessageId: input.destMessageId, - isMultipart: input.isMultipart, - partCount: input.partCount, - fileCount: input.files.length, - ingestionRunId: input.ingestionRunId, - creator: input.creator ?? undefined, - previewData: input.previewData ? new Uint8Array(input.previewData) : undefined, - previewMsgId: input.previewMsgId ?? undefined, - files: { - create: input.files, - }, - }, - }); - // Notify the bot service about the new package (for subscription alerts) - try { - await db.$queryRawUnsafe(`SELECT pg_notify('new_package', $1)`, JSON.stringify({ - packageId: pkg.id, - fileName: input.fileName, - creator: input.creator ?? null, - })); - } - catch { - // Best-effort — don't fail the ingestion if notification fails - } - return pkg; -} -export async function createIngestionRun(accountId) { - return db.ingestionRun.create({ - data: { - accountId, - status: "RUNNING", - currentActivity: "Starting ingestion run", - currentStep: "initializing", - lastActivityAt: new Date(), - }, - }); -} -export async function updateRunActivity(runId, activity) { - return db.ingestionRun.update({ - where: { id: runId }, - data: { - currentActivity: activity.currentActivity, - currentStep: activity.currentStep, - currentChannel: activity.currentChannel ?? undefined, - currentFile: activity.currentFile ?? undefined, - currentFileNum: activity.currentFileNum ?? undefined, - totalFiles: activity.totalFiles ?? undefined, - downloadedBytes: activity.downloadedBytes ?? undefined, - totalBytes: activity.totalBytes ?? undefined, - downloadPercent: activity.downloadPercent ?? undefined, - lastActivityAt: new Date(), - ...(activity.messagesScanned !== undefined && { messagesScanned: activity.messagesScanned }), - ...(activity.zipsFound !== undefined && { zipsFound: activity.zipsFound }), - ...(activity.zipsDuplicate !== undefined && { zipsDuplicate: activity.zipsDuplicate }), - ...(activity.zipsIngested !== undefined && { zipsIngested: activity.zipsIngested }), - }, - }); -} -const CLEAR_ACTIVITY = { - currentActivity: null, - currentStep: null, - currentChannel: null, - currentFile: null, - currentFileNum: null, - totalFiles: null, - downloadedBytes: null, - totalBytes: null, - downloadPercent: null, - lastActivityAt: new Date(), -}; -export async function completeIngestionRun(runId, counters) { - return db.ingestionRun.update({ - where: { id: runId }, - data: { - status: "COMPLETED", - finishedAt: new Date(), - ...counters, - ...CLEAR_ACTIVITY, - }, - }); -} -export async function failIngestionRun(runId, errorMessage) { - return db.ingestionRun.update({ - where: { id: runId }, - data: { - status: "FAILED", - finishedAt: new Date(), - errorMessage, - ...CLEAR_ACTIVITY, - }, - }); -} -export async function updateLastProcessedMessage(mappingId, messageId) { - return db.accountChannelMap.update({ - where: { id: mappingId }, - data: { lastProcessedMessageId: messageId }, - }); -} -export async function markStaleRunsAsFailed() { - return db.ingestionRun.updateMany({ - where: { status: "RUNNING" }, - data: { - status: "FAILED", - finishedAt: new Date(), - errorMessage: "Worker restarted — run was still marked as RUNNING", - }, - }); -} -export async function updateAccountAuthState(accountId, authState, authCode) { - return db.telegramAccount.update({ - where: { id: accountId }, - data: { authState, authCode, lastSeenAt: authState === "AUTHENTICATED" ? new Date() : undefined }, - }); -} -export async function getAccountAuthCode(accountId) { - const account = await db.telegramAccount.findUnique({ - where: { id: accountId }, - select: { authCode: true, authState: true }, - }); - return account; -} -/** - * Upsert a channel by telegramId. Returns the channel record. - * If it already exists, update title and forum status. - * New channels default to disabled (isActive: false) so the admin must - * explicitly enable them before the worker processes them. - * Pass isActive: true for DESTINATION channels that must be active immediately. - */ -export async function upsertChannel(input) { - return db.telegramChannel.upsert({ - where: { telegramId: input.telegramId }, - create: { - telegramId: input.telegramId, - title: input.title, - type: input.type, - isForum: input.isForum, - isActive: input.isActive ?? false, - }, - update: { - title: input.title, - isForum: input.isForum, - }, - }); -} -/** - * Link an account to a channel if not already linked. - * Uses a try/catch on unique constraint to make it idempotent. - */ -export async function ensureAccountChannelLink(accountId, channelId, role) { - try { - return await db.accountChannelMap.create({ - data: { accountId, channelId, role }, - }); - } - catch (err) { - // Already linked — ignore unique constraint violation - if (err instanceof Error && err.message.includes("Unique constraint")) { - return null; - } - throw err; - } -} -// ── Forum / Topic progress ── -export async function setChannelForum(channelId, isForum) { - return db.telegramChannel.update({ - where: { id: channelId }, - data: { isForum }, - }); -} -export async function getTopicProgress(mappingId) { - return db.topicProgress.findMany({ - where: { accountChannelMapId: mappingId }, - }); -} -export async function upsertTopicProgress(mappingId, topicId, topicName, lastProcessedMessageId) { - return db.topicProgress.upsert({ - where: { - accountChannelMapId_topicId: { - accountChannelMapId: mappingId, - topicId, - }, - }, - create: { - accountChannelMapId: mappingId, - topicId, - topicName, - lastProcessedMessageId, - }, - update: { - topicName, - lastProcessedMessageId, - }, - }); -} -// ── Channel fetch requests (DB-mediated communication with web app) ── -export async function getChannelFetchRequest(requestId) { - return db.channelFetchRequest.findUnique({ - where: { id: requestId }, - include: { account: true }, - }); -} -export async function updateFetchRequestStatus(requestId, status, extra) { - return db.channelFetchRequest.update({ - where: { id: requestId }, - data: { - status, - resultJson: extra?.resultJson ?? undefined, - error: extra?.error ?? undefined, - }, - }); -} -export async function getAccountLinkedChannelIds(accountId) { - const links = await db.accountChannelMap.findMany({ - where: { accountId }, - select: { channel: { select: { telegramId: true } } }, - }); - return new Set(links.map((l) => l.channel.telegramId.toString())); -} -export async function getExistingChannelsByTelegramId() { - const channels = await db.telegramChannel.findMany({ - select: { id: true, telegramId: true }, - }); - const map = new Map(); - for (const ch of channels) { - map.set(ch.telegramId.toString(), ch.id); - } - return map; -} -export async function getAccountById(accountId) { - return db.telegramAccount.findUnique({ where: { id: accountId } }); -} -//# sourceMappingURL=queries.js.map \ No newline at end of file diff --git a/worker/dist/db/queries.js.map b/worker/dist/db/queries.js.map deleted file mode 100644 index fddee22..0000000 --- a/worker/dist/db/queries.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"queries.js","sourceRoot":"","sources":["../../src/db/queries.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,EAAE,EAAE,MAAM,aAAa,CAAC;AAGjC,MAAM,CAAC,KAAK,UAAU,iBAAiB;IACrC,OAAO,EAAE,CAAC,eAAe,CAAC,QAAQ,CAAC;QACjC,KAAK,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAE,eAAe,EAAE;KACtD,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB;IACtC,OAAO,EAAE,CAAC,eAAe,CAAC,QAAQ,CAAC;QACjC,KAAK,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE;KAChD,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc;IAClC,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;IAC/C,OAAO,KAAK,GAAG,CAAC,CAAC;AACnB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAAC,SAAiB;IAC9D,OAAO,EAAE,CAAC,iBAAiB,CAAC,QAAQ,CAAC;QACnC,KAAK,EAAE;YACL,SAAS;YACT,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE;SAC5C;QACD,OAAO,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE;KAC3B,CAAC,CAAC;AACL,CAAC;AAED,mCAAmC;AAEnC,MAAM,CAAC,KAAK,UAAU,2BAA2B;IAC/C,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,aAAa,CAAC,UAAU,CAAC;QAChD,KAAK,EAAE,EAAE,GAAG,EAAE,wBAAwB,EAAE;KACzC,CAAC,CAAC;IACH,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IAC1B,OAAO,EAAE,CAAC,eAAe,CAAC,SAAS,CAAC;QAClC,KAAK,EAAE,EAAE,EAAE,EAAE,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE,aAAa,EAAE,QAAQ,EAAE,IAAI,EAAE;KAClE,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,GAAW;IAChD,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,aAAa,CAAC,UAAU,CAAC,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;IACtE,OAAO,OAAO,EAAE,KAAK,IAAI,IAAI,CAAC;AAChC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,GAAW,EAAE,KAAa;IAC/D,OAAO,EAAE,CAAC,aAAa,CAAC,MAAM,CAAC;QAC7B,KAAK,EAAE,EAAE,GAAG,EAAE;QACd,MAAM,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE;QACtB,MAAM,EAAE,EAAE,KAAK,EAAE;KAClB,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,WAAmB;IAC3D,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC;QACrC,KAAK,EAAE,EAAE,WAAW,EAAE,aAAa,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE;QACpD,MAAM,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE;KACrB,CAAC,CAAC;IACH,OAAO,GAAG,KAAK,IAAI,CAAC;AACtB,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,4BAA4B,CAChD,eAAuB,EACvB,eAAuB;IAEvB,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC;QACrC,KAAK,EAAE,EAAE,eAAe,EAAE,eAAe,EAAE,aAAa,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE;QACzE,MAAM,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE;KACrB,CAAC,CAAC;IACH,OAAO,GAAG,KAAK,IAAI,CAAC;AACtB,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAAC,WAAmB;IACnE,MAAM,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC;QAC1B,KAAK,EAAE,EAAE,WAAW,EAAE,aAAa,EAAE,IAAI,EAAE;KAC5C,CAAC,CAAC;AACL,CAAC;AA4BD,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAC,KAAyB;IACpE,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC;QAClC,IAAI,EAAE;YACJ,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,eAAe,EAAE,KAAK,CAAC,eAAe;YACtC,eAAe,EAAE,KAAK,CAAC,eAAe;YACtC,aAAa,EAAE,KAAK,CAAC,aAAa,IAAI,SAAS;YAC/C,aAAa,EAAE,KAAK,CAAC,aAAa;YAClC,aAAa,EAAE,KAAK,CAAC,aAAa;YAClC,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM;YAC7B,cAAc,EAAE,KAAK,CAAC,cAAc;YACpC,OAAO,EAAE,KAAK,CAAC,OAAO,IAAI,SAAS;YACnC,WAAW,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,UAAU,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,SAAS;YAC9E,YAAY,EAAE,KAAK,CAAC,YAAY,IAAI,SAAS;YAC7C,KAAK,EAAE;gBACL,MAAM,EAAE,KAAK,CAAC,KAAK;aACpB;SACF;KACF,CAAC,CAAC;IAEH,yEAAyE;IACzE,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,eAAe,CACtB,qCAAqC,EACrC,IAAI,CAAC,SAAS,CAAC;YACb,SAAS,EAAE,GAAG,CAAC,EAAE;YACjB,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,OAAO,EAAE,KAAK,CAAC,OAAO,IAAI,IAAI;SAC/B,CAAC,CACH,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,+DAA+D;IACjE,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,SAAiB;IACxD,OAAO,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC;QAC5B,IAAI,EAAE;YACJ,SAAS;YACT,MAAM,EAAE,SAAS;YACjB,eAAe,EAAE,wBAAwB;YACzC,WAAW,EAAE,cAAc;YAC3B,cAAc,EAAE,IAAI,IAAI,EAAE;SAC3B;KACF,CAAC,CAAC;AACL,CAAC;AAkBD,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,KAAa,EACb,QAAwB;IAExB,OAAO,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC;QAC5B,KAAK,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE;QACpB,IAAI,EAAE;YACJ,eAAe,EAAE,QAAQ,CAAC,eAAe;YACzC,WAAW,EAAE,QAAQ,CAAC,WAAW;YACjC,cAAc,EAAE,QAAQ,CAAC,cAAc,IAAI,SAAS;YACpD,WAAW,EAAE,QAAQ,CAAC,WAAW,IAAI,SAAS;YAC9C,cAAc,EAAE,QAAQ,CAAC,cAAc,IAAI,SAAS;YACpD,UAAU,EAAE,QAAQ,CAAC,UAAU,IAAI,SAAS;YAC5C,eAAe,EAAE,QAAQ,CAAC,eAAe,IAAI,SAAS;YACtD,UAAU,EAAE,QAAQ,CAAC,UAAU,IAAI,SAAS;YAC5C,eAAe,EAAE,QAAQ,CAAC,eAAe,IAAI,SAAS;YACtD,cAAc,EAAE,IAAI,IAAI,EAAE;YAC1B,GAAG,CAAC,QAAQ,CAAC,eAAe,KAAK,SAAS,IAAI,EAAE,eAAe,EAAE,QAAQ,CAAC,eAAe,EAAE,CAAC;YAC5F,GAAG,CAAC,QAAQ,CAAC,SAAS,KAAK,SAAS,IAAI,EAAE,SAAS,EAAE,QAAQ,CAAC,SAAS,EAAE,CAAC;YAC1E,GAAG,CAAC,QAAQ,CAAC,aAAa,KAAK,SAAS,IAAI,EAAE,aAAa,EAAE,QAAQ,CAAC,aAAa,EAAE,CAAC;YACtF,GAAG,CAAC,QAAQ,CAAC,YAAY,KAAK,SAAS,IAAI,EAAE,YAAY,EAAE,QAAQ,CAAC,YAAY,EAAE,CAAC;SACpF;KACF,CAAC,CAAC;AACL,CAAC;AAED,MAAM,cAAc,GAAG;IACrB,eAAe,EAAE,IAAI;IACrB,WAAW,EAAE,IAAI;IACjB,cAAc,EAAE,IAAI;IACpB,WAAW,EAAE,IAAI;IACjB,cAAc,EAAE,IAAI;IACpB,UAAU,EAAE,IAAI;IAChB,eAAe,EAAE,IAAI;IACrB,UAAU,EAAE,IAAI;IAChB,eAAe,EAAE,IAAI;IACrB,cAAc,EAAE,IAAI,IAAI,EAAE;CAC3B,CAAC;AAEF,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,KAAa,EACb,QAKC;IAED,OAAO,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC;QAC5B,KAAK,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE;QACpB,IAAI,EAAE;YACJ,MAAM,EAAE,WAAW;YACnB,UAAU,EAAE,IAAI,IAAI,EAAE;YACtB,GAAG,QAAQ;YACX,GAAG,cAAc;SAClB;KACF,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,KAAa,EAAE,YAAoB;IACxE,OAAO,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC;QAC5B,KAAK,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE;QACpB,IAAI,EAAE;YACJ,MAAM,EAAE,QAAQ;YAChB,UAAU,EAAE,IAAI,IAAI,EAAE;YACtB,YAAY;YACZ,GAAG,cAAc;SAClB;KACF,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAC9C,SAAiB,EACjB,SAAiB;IAEjB,OAAO,EAAE,CAAC,iBAAiB,CAAC,MAAM,CAAC;QACjC,KAAK,EAAE,EAAE,EAAE,EAAE,SAAS,EAAE;QACxB,IAAI,EAAE,EAAE,sBAAsB,EAAE,SAAS,EAAE;KAC5C,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB;IACzC,OAAO,EAAE,CAAC,YAAY,CAAC,UAAU,CAAC;QAChC,KAAK,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE;QAC5B,IAAI,EAAE;YACJ,MAAM,EAAE,QAAQ;YAChB,UAAU,EAAE,IAAI,IAAI,EAAE;YACtB,YAAY,EAAE,oDAAoD;SACnE;KACF,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,SAAiB,EACjB,SAA0F,EAC1F,QAAwB;IAExB,OAAO,EAAE,CAAC,eAAe,CAAC,MAAM,CAAC;QAC/B,KAAK,EAAE,EAAE,EAAE,EAAE,SAAS,EAAE;QACxB,IAAI,EAAE,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU,EAAE,SAAS,KAAK,eAAe,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC,SAAS,EAAE;KAClG,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,SAAiB;IACxD,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,eAAe,CAAC,UAAU,CAAC;QAClD,KAAK,EAAE,EAAE,EAAE,EAAE,SAAS,EAAE;QACxB,MAAM,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE;KAC5C,CAAC,CAAC;IACH,OAAO,OAAO,CAAC;AACjB,CAAC;AAYD;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,KAAyB;IAC3D,OAAO,EAAE,CAAC,eAAe,CAAC,MAAM,CAAC;QAC/B,KAAK,EAAE,EAAE,UAAU,EAAE,KAAK,CAAC,UAAU,EAAE;QACvC,MAAM,EAAE;YACN,UAAU,EAAE,KAAK,CAAC,UAAU;YAC5B,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,QAAQ,EAAE,KAAK,CAAC,QAAQ,IAAI,KAAK;SAClC;QACD,MAAM,EAAE;YACN,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,OAAO,EAAE,KAAK,CAAC,OAAO;SACvB;KACF,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,SAAiB,EACjB,SAAiB,EACjB,IAAyB;IAEzB,IAAI,CAAC;QACH,OAAO,MAAM,EAAE,CAAC,iBAAiB,CAAC,MAAM,CAAC;YACvC,IAAI,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,IAAI,EAAE;SACrC,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,sDAAsD;QACtD,IAAI,GAAG,YAAY,KAAK,IAAI,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAC,EAAE,CAAC;YACtE,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,GAAG,CAAC;IACZ,CAAC;AACH,CAAC;AAED,+BAA+B;AAE/B,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,SAAiB,EAAE,OAAgB;IACvE,OAAO,EAAE,CAAC,eAAe,CAAC,MAAM,CAAC;QAC/B,KAAK,EAAE,EAAE,EAAE,EAAE,SAAS,EAAE;QACxB,IAAI,EAAE,EAAE,OAAO,EAAE;KAClB,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,SAAiB;IACtD,OAAO,EAAE,CAAC,aAAa,CAAC,QAAQ,CAAC;QAC/B,KAAK,EAAE,EAAE,mBAAmB,EAAE,SAAS,EAAE;KAC1C,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,SAAiB,EACjB,OAAe,EACf,SAAwB,EACxB,sBAA8B;IAE9B,OAAO,EAAE,CAAC,aAAa,CAAC,MAAM,CAAC;QAC7B,KAAK,EAAE;YACL,2BAA2B,EAAE;gBAC3B,mBAAmB,EAAE,SAAS;gBAC9B,OAAO;aACR;SACF;QACD,MAAM,EAAE;YACN,mBAAmB,EAAE,SAAS;YAC9B,OAAO;YACP,SAAS;YACT,sBAAsB;SACvB;QACD,MAAM,EAAE;YACN,SAAS;YACT,sBAAsB;SACvB;KACF,CAAC,CAAC;AACL,CAAC;AAED,wEAAwE;AAExE,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAC,SAAiB;IAC5D,OAAO,EAAE,CAAC,mBAAmB,CAAC,UAAU,CAAC;QACvC,KAAK,EAAE,EAAE,EAAE,EAAE,SAAS,EAAE;QACxB,OAAO,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE;KAC3B,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,SAAiB,EACjB,MAAmB,EACnB,KAA+C;IAE/C,OAAO,EAAE,CAAC,mBAAmB,CAAC,MAAM,CAAC;QACnC,KAAK,EAAE,EAAE,EAAE,EAAE,SAAS,EAAE;QACxB,IAAI,EAAE;YACJ,MAAM;YACN,UAAU,EAAE,KAAK,EAAE,UAAU,IAAI,SAAS;YAC1C,KAAK,EAAE,KAAK,EAAE,KAAK,IAAI,SAAS;SACjC;KACF,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAAC,SAAiB;IAChE,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,iBAAiB,CAAC,QAAQ,CAAC;QAChD,KAAK,EAAE,EAAE,SAAS,EAAE;QACpB,MAAM,EAAE,EAAE,OAAO,EAAE,EAAE,MAAM,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,EAAE,EAAE;KACtD,CAAC,CAAC;IACH,OAAO,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;AACpE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,+BAA+B;IACnD,MAAM,QAAQ,GAAG,MAAM,EAAE,CAAC,eAAe,CAAC,QAAQ,CAAC;QACjD,MAAM,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE;KACvC,CAAC,CAAC;IACH,MAAM,GAAG,GAAG,IAAI,GAAG,EAAkB,CAAC;IACtC,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1B,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;IAC3C,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,SAAiB;IACpD,OAAO,EAAE,CAAC,eAAe,CAAC,UAAU,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;AACrE,CAAC"} \ No newline at end of file diff --git a/worker/dist/fetch-listener.d.ts b/worker/dist/fetch-listener.d.ts deleted file mode 100644 index 462baf1..0000000 --- a/worker/dist/fetch-listener.d.ts +++ /dev/null @@ -1,11 +0,0 @@ -/** - * Start listening for pg_notify signals from the web app. - * - * Channels: - * - `channel_fetch` — payload = requestId → fetch channels for an account - * - `generate_invite` — payload = channelId → generate invite link for destination - * - `create_destination` — payload = JSON { requestId, title } → create supergroup via TDLib - * - `ingestion_trigger` — trigger an immediate ingestion cycle - */ -export declare function startFetchListener(): Promise; -export declare function stopFetchListener(): void; diff --git a/worker/dist/fetch-listener.js b/worker/dist/fetch-listener.js deleted file mode 100644 index 60dd305..0000000 --- a/worker/dist/fetch-listener.js +++ /dev/null @@ -1,195 +0,0 @@ -import { pool } from "./db/client.js"; -import { childLogger } from "./util/logger.js"; -import { withTdlibMutex } from "./util/mutex.js"; -import { processFetchRequest } from "./worker.js"; -import { generateInviteLink, createSupergroup } from "./tdlib/chats.js"; -import { createTdlibClient, closeTdlibClient } from "./tdlib/client.js"; -import { triggerImmediateCycle } from "./scheduler.js"; -import { getGlobalDestinationChannel, setGlobalSetting, getActiveAccounts, upsertChannel, ensureAccountChannelLink, } from "./db/queries.js"; -const log = childLogger("fetch-listener"); -let pgClient = null; -/** - * Start listening for pg_notify signals from the web app. - * - * Channels: - * - `channel_fetch` — payload = requestId → fetch channels for an account - * - `generate_invite` — payload = channelId → generate invite link for destination - * - `create_destination` — payload = JSON { requestId, title } → create supergroup via TDLib - * - `ingestion_trigger` — trigger an immediate ingestion cycle - */ -export async function startFetchListener() { - pgClient = await pool.connect(); - await pgClient.query("LISTEN channel_fetch"); - await pgClient.query("LISTEN generate_invite"); - await pgClient.query("LISTEN create_destination"); - await pgClient.query("LISTEN ingestion_trigger"); - pgClient.on("notification", (msg) => { - if (msg.channel === "channel_fetch" && msg.payload) { - handleChannelFetch(msg.payload); - } - else if (msg.channel === "generate_invite" && msg.payload) { - handleGenerateInvite(msg.payload); - } - else if (msg.channel === "create_destination" && msg.payload) { - handleCreateDestination(msg.payload); - } - else if (msg.channel === "ingestion_trigger") { - handleIngestionTrigger(); - } - }); - log.info("Fetch listener started (channel_fetch, generate_invite, create_destination, ingestion_trigger)"); -} -export function stopFetchListener() { - if (pgClient) { - pgClient.release(); - pgClient = null; - } - log.info("Fetch listener stopped"); -} -// ── Channel fetch handler ── -// Chain promises to ensure sequential execution -let fetchQueue = Promise.resolve(); -function handleChannelFetch(requestId) { - fetchQueue = fetchQueue.then(async () => { - try { - await withTdlibMutex("fetch-channels", () => processFetchRequest(requestId)); - } - catch (err) { - log.error({ err, requestId }, "Failed to process fetch request"); - } - }); -} -// ── Invite link generation handler ── -function handleGenerateInvite(channelId) { - fetchQueue = fetchQueue.then(async () => { - try { - await withTdlibMutex("generate-invite", async () => { - const destChannel = await getGlobalDestinationChannel(); - if (!destChannel || destChannel.id !== channelId) { - log.warn({ channelId }, "Destination channel mismatch, skipping invite generation"); - return; - } - // Use the first available authenticated account to generate the link - const accounts = await getActiveAccounts(); - if (accounts.length === 0) { - log.warn("No authenticated accounts to generate invite link"); - return; - } - const account = accounts[0]; - const client = await createTdlibClient({ id: account.id, phone: account.phone }); - try { - const link = await generateInviteLink(client, destChannel.telegramId); - await setGlobalSetting("destination_invite_link", link); - log.info({ link }, "Invite link generated and saved"); - } - finally { - await closeTdlibClient(client); - } - }); - } - catch (err) { - log.error({ err, channelId }, "Failed to generate invite link"); - } - }); -} -// ── Create destination supergroup handler ── -function handleCreateDestination(payload) { - fetchQueue = fetchQueue.then(async () => { - let requestId; - try { - const parsed = JSON.parse(payload); - requestId = parsed.requestId; - await withTdlibMutex("create-destination", async () => { - const { db } = await import("./db/client.js"); - // Mark the request as in-progress - await db.channelFetchRequest.update({ - where: { id: parsed.requestId }, - data: { status: "IN_PROGRESS" }, - }); - // Use the first available authenticated account - const accounts = await getActiveAccounts(); - if (accounts.length === 0) { - throw new Error("No authenticated accounts available to create the group"); - } - const account = accounts[0]; - const client = await createTdlibClient({ id: account.id, phone: account.phone }); - try { - // Create the supergroup via TDLib - const result = await createSupergroup(client, parsed.title); - log.info({ chatId: result.chatId.toString(), title: result.title }, "Supergroup created"); - // Upsert it as a DESTINATION channel in the DB (active by default) - const channel = await upsertChannel({ - telegramId: result.chatId, - title: result.title, - type: "DESTINATION", - isForum: false, - isActive: true, - }); - // Set as global destination - await setGlobalSetting("destination_channel_id", channel.id); - // Generate an invite link - const link = await generateInviteLink(client, result.chatId); - await setGlobalSetting("destination_invite_link", link); - log.info({ link }, "Invite link generated for new destination"); - // Link all authenticated accounts as WRITER - for (const acc of accounts) { - try { - await ensureAccountChannelLink(acc.id, channel.id, "WRITER"); - } - catch { - // Already linked - } - } - // Mark fetch request as completed with the channel info - await db.channelFetchRequest.update({ - where: { id: parsed.requestId }, - data: { - status: "COMPLETED", - resultJson: JSON.stringify({ - channelId: channel.id, - telegramId: result.chatId.toString(), - title: result.title, - inviteLink: link, - }), - }, - }); - log.info({ channelId: channel.id, telegramId: result.chatId.toString() }, "Destination channel created and configured"); - } - finally { - await closeTdlibClient(client); - } - }); - } - catch (err) { - log.error({ err, payload }, "Failed to create destination channel"); - if (requestId) { - try { - const { db } = await import("./db/client.js"); - await db.channelFetchRequest.update({ - where: { id: requestId }, - data: { - status: "FAILED", - error: err instanceof Error ? err.message : String(err), - }, - }); - } - catch { - // Best-effort - } - } - } - }); -} -// ── Ingestion trigger handler ── -function handleIngestionTrigger() { - fetchQueue = fetchQueue.then(async () => { - try { - log.info("Ingestion trigger received from UI"); - await triggerImmediateCycle(); - } - catch (err) { - log.error({ err }, "Failed to trigger immediate ingestion cycle"); - } - }); -} -//# sourceMappingURL=fetch-listener.js.map \ No newline at end of file diff --git a/worker/dist/fetch-listener.js.map b/worker/dist/fetch-listener.js.map deleted file mode 100644 index cb513a9..0000000 --- a/worker/dist/fetch-listener.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"fetch-listener.js","sourceRoot":"","sources":["../src/fetch-listener.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAC;AACtC,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AACxE,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACxE,OAAO,EAAE,qBAAqB,EAAE,MAAM,gBAAgB,CAAC;AACvD,OAAO,EACL,2BAA2B,EAE3B,gBAAgB,EAChB,iBAAiB,EACjB,aAAa,EACb,wBAAwB,GACzB,MAAM,iBAAiB,CAAC;AAEzB,MAAM,GAAG,GAAG,WAAW,CAAC,gBAAgB,CAAC,CAAC;AAE1C,IAAI,QAAQ,GAAyB,IAAI,CAAC;AAE1C;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB;IACtC,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;IAChC,MAAM,QAAQ,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;IAC7C,MAAM,QAAQ,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC;IAC/C,MAAM,QAAQ,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAClD,MAAM,QAAQ,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;IAEjD,QAAQ,CAAC,EAAE,CAAC,cAAc,EAAE,CAAC,GAAG,EAAE,EAAE;QAClC,IAAI,GAAG,CAAC,OAAO,KAAK,eAAe,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;YACnD,kBAAkB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAClC,CAAC;aAAM,IAAI,GAAG,CAAC,OAAO,KAAK,iBAAiB,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;YAC5D,oBAAoB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACpC,CAAC;aAAM,IAAI,GAAG,CAAC,OAAO,KAAK,oBAAoB,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;YAC/D,uBAAuB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACvC,CAAC;aAAM,IAAI,GAAG,CAAC,OAAO,KAAK,mBAAmB,EAAE,CAAC;YAC/C,sBAAsB,EAAE,CAAC;QAC3B,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,GAAG,CAAC,IAAI,CAAC,gGAAgG,CAAC,CAAC;AAC7G,CAAC;AAED,MAAM,UAAU,iBAAiB;IAC/B,IAAI,QAAQ,EAAE,CAAC;QACb,QAAQ,CAAC,OAAO,EAAE,CAAC;QACnB,QAAQ,GAAG,IAAI,CAAC;IAClB,CAAC;IACD,GAAG,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;AACrC,CAAC;AAED,8BAA8B;AAE9B,gDAAgD;AAChD,IAAI,UAAU,GAAkB,OAAO,CAAC,OAAO,EAAE,CAAC;AAElD,SAAS,kBAAkB,CAAC,SAAiB;IAC3C,UAAU,GAAG,UAAU,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;QACtC,IAAI,CAAC;YACH,MAAM,cAAc,CAAC,gBAAgB,EAAE,GAAG,EAAE,CAC1C,mBAAmB,CAAC,SAAS,CAAC,CAC/B,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,EAAE,iCAAiC,CAAC,CAAC;QACnE,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AAED,uCAAuC;AAEvC,SAAS,oBAAoB,CAAC,SAAiB;IAC7C,UAAU,GAAG,UAAU,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;QACtC,IAAI,CAAC;YACH,MAAM,cAAc,CAAC,iBAAiB,EAAE,KAAK,IAAI,EAAE;gBACjD,MAAM,WAAW,GAAG,MAAM,2BAA2B,EAAE,CAAC;gBACxD,IAAI,CAAC,WAAW,IAAI,WAAW,CAAC,EAAE,KAAK,SAAS,EAAE,CAAC;oBACjD,GAAG,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,EAAE,0DAA0D,CAAC,CAAC;oBACpF,OAAO;gBACT,CAAC;gBAED,qEAAqE;gBACrE,MAAM,QAAQ,GAAG,MAAM,iBAAiB,EAAE,CAAC;gBAC3C,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAC1B,GAAG,CAAC,IAAI,CAAC,mDAAmD,CAAC,CAAC;oBAC9D,OAAO;gBACT,CAAC;gBAED,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;gBAC5B,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,EAAE,EAAE,EAAE,OAAO,CAAC,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;gBAEjF,IAAI,CAAC;oBACH,MAAM,IAAI,GAAG,MAAM,kBAAkB,CAAC,MAAM,EAAE,WAAW,CAAC,UAAU,CAAC,CAAC;oBACtE,MAAM,gBAAgB,CAAC,yBAAyB,EAAE,IAAI,CAAC,CAAC;oBACxD,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,EAAE,iCAAiC,CAAC,CAAC;gBACxD,CAAC;wBAAS,CAAC;oBACT,MAAM,gBAAgB,CAAC,MAAM,CAAC,CAAC;gBACjC,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,EAAE,gCAAgC,CAAC,CAAC;QAClE,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AAED,8CAA8C;AAE9C,SAAS,uBAAuB,CAAC,OAAe;IAC9C,UAAU,GAAG,UAAU,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;QACtC,IAAI,SAA6B,CAAC;QAClC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAyC,CAAC;YAC3E,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;YAE7B,MAAM,cAAc,CAAC,oBAAoB,EAAE,KAAK,IAAI,EAAE;gBACpD,MAAM,EAAE,EAAE,EAAE,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,CAAC;gBAE9C,kCAAkC;gBAClC,MAAM,EAAE,CAAC,mBAAmB,CAAC,MAAM,CAAC;oBAClC,KAAK,EAAE,EAAE,EAAE,EAAE,MAAM,CAAC,SAAS,EAAE;oBAC/B,IAAI,EAAE,EAAE,MAAM,EAAE,aAAa,EAAE;iBAChC,CAAC,CAAC;gBAEH,gDAAgD;gBAChD,MAAM,QAAQ,GAAG,MAAM,iBAAiB,EAAE,CAAC;gBAC3C,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAC1B,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;gBAC7E,CAAC;gBAED,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;gBAC5B,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,EAAE,EAAE,EAAE,OAAO,CAAC,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;gBAEjF,IAAI,CAAC;oBACH,kCAAkC;oBAClC,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;oBAC5D,GAAG,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,EAAE,oBAAoB,CAAC,CAAC;oBAE1F,mEAAmE;oBACnE,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC;wBAClC,UAAU,EAAE,MAAM,CAAC,MAAM;wBACzB,KAAK,EAAE,MAAM,CAAC,KAAK;wBACnB,IAAI,EAAE,aAAa;wBACnB,OAAO,EAAE,KAAK;wBACd,QAAQ,EAAE,IAAI;qBACf,CAAC,CAAC;oBAEH,4BAA4B;oBAC5B,MAAM,gBAAgB,CAAC,wBAAwB,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC;oBAE7D,0BAA0B;oBAC1B,MAAM,IAAI,GAAG,MAAM,kBAAkB,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;oBAC7D,MAAM,gBAAgB,CAAC,yBAAyB,EAAE,IAAI,CAAC,CAAC;oBACxD,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,EAAE,2CAA2C,CAAC,CAAC;oBAEhE,4CAA4C;oBAC5C,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;wBAC3B,IAAI,CAAC;4BACH,MAAM,wBAAwB,CAAC,GAAG,CAAC,EAAE,EAAE,OAAO,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;wBAC/D,CAAC;wBAAC,MAAM,CAAC;4BACP,iBAAiB;wBACnB,CAAC;oBACH,CAAC;oBAED,wDAAwD;oBACxD,MAAM,EAAE,CAAC,mBAAmB,CAAC,MAAM,CAAC;wBAClC,KAAK,EAAE,EAAE,EAAE,EAAE,MAAM,CAAC,SAAS,EAAE;wBAC/B,IAAI,EAAE;4BACJ,MAAM,EAAE,WAAW;4BACnB,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC;gCACzB,SAAS,EAAE,OAAO,CAAC,EAAE;gCACrB,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE;gCACpC,KAAK,EAAE,MAAM,CAAC,KAAK;gCACnB,UAAU,EAAE,IAAI;6BACjB,CAAC;yBACH;qBACF,CAAC,CAAC;oBAEH,GAAG,CAAC,IAAI,CACN,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,EAC/D,4CAA4C,CAC7C,CAAC;gBACJ,CAAC;wBAAS,CAAC;oBACT,MAAM,gBAAgB,CAAC,MAAM,CAAC,CAAC;gBACjC,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,sCAAsC,CAAC,CAAC;YACpE,IAAI,SAAS,EAAE,CAAC;gBACd,IAAI,CAAC;oBACH,MAAM,EAAE,EAAE,EAAE,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,CAAC;oBAC9C,MAAM,EAAE,CAAC,mBAAmB,CAAC,MAAM,CAAC;wBAClC,KAAK,EAAE,EAAE,EAAE,EAAE,SAAS,EAAE;wBACxB,IAAI,EAAE;4BACJ,MAAM,EAAE,QAAQ;4BAChB,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;yBACxD;qBACF,CAAC,CAAC;gBACL,CAAC;gBAAC,MAAM,CAAC;oBACP,cAAc;gBAChB,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AAED,kCAAkC;AAElC,SAAS,sBAAsB;IAC7B,UAAU,GAAG,UAAU,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;QACtC,IAAI,CAAC;YACH,GAAG,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;YAC/C,MAAM,qBAAqB,EAAE,CAAC;QAChC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,EAAE,6CAA6C,CAAC,CAAC;QACpE,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC"} \ No newline at end of file diff --git a/worker/dist/index.d.ts b/worker/dist/index.d.ts deleted file mode 100644 index cb0ff5c..0000000 --- a/worker/dist/index.d.ts +++ /dev/null @@ -1 +0,0 @@ -export {}; diff --git a/worker/dist/index.js b/worker/dist/index.js deleted file mode 100644 index 4b59786..0000000 --- a/worker/dist/index.js +++ /dev/null @@ -1,50 +0,0 @@ -import { mkdir } from "fs/promises"; -import { config } from "./util/config.js"; -import { logger } from "./util/logger.js"; -import { markStaleRunsAsFailed } from "./db/queries.js"; -import { cleanupTempDir } from "./worker.js"; -import { startScheduler, stopScheduler } from "./scheduler.js"; -import { startFetchListener, stopFetchListener } from "./fetch-listener.js"; -import { db, pool } from "./db/client.js"; -const log = logger.child({ module: "main" }); -async function main() { - log.info("DragonsStash Telegram Worker starting"); - log.info({ config: { ...config, databaseUrl: "***" } }, "Configuration loaded"); - if (!config.telegramApiId || !config.telegramApiHash) { - log.fatal("TELEGRAM_API_ID and TELEGRAM_API_HASH are both required"); - process.exit(1); - } - // Ensure temp directory exists - await mkdir(config.tempDir, { recursive: true }); - await mkdir(config.tdlibStateDir, { recursive: true }); - // Clean up stale state - await cleanupTempDir(); - await markStaleRunsAsFailed(); - // Start the fetch listener (pg_notify for on-demand channel fetching) - await startFetchListener(); - // Start the scheduler - await startScheduler(); -} -// Graceful shutdown -function shutdown(signal) { - log.info({ signal }, "Shutdown signal received"); - stopScheduler(); - stopFetchListener(); - // Close DB connections - Promise.all([db.$disconnect(), pool.end()]) - .then(() => { - log.info("Shutdown complete"); - process.exit(0); - }) - .catch((err) => { - log.error({ err }, "Error during shutdown"); - process.exit(1); - }); -} -process.on("SIGTERM", () => shutdown("SIGTERM")); -process.on("SIGINT", () => shutdown("SIGINT")); -main().catch((err) => { - log.fatal({ err }, "Worker failed to start"); - process.exit(1); -}); -//# sourceMappingURL=index.js.map \ No newline at end of file diff --git a/worker/dist/index.js.map b/worker/dist/index.js.map deleted file mode 100644 index ceccbac..0000000 --- a/worker/dist/index.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,OAAO,EAAE,qBAAqB,EAAE,MAAM,iBAAiB,CAAC;AACxD,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/D,OAAO,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAC5E,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAC;AAE1C,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;AAE7C,KAAK,UAAU,IAAI;IACjB,GAAG,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;IAClD,GAAG,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,GAAG,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,EAAE,EAAE,sBAAsB,CAAC,CAAC;IAEhF,IAAI,CAAC,MAAM,CAAC,aAAa,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC;QACrD,GAAG,CAAC,KAAK,CAAC,yDAAyD,CAAC,CAAC;QACrE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,+BAA+B;IAC/B,MAAM,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACjD,MAAM,KAAK,CAAC,MAAM,CAAC,aAAa,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAEvD,uBAAuB;IACvB,MAAM,cAAc,EAAE,CAAC;IACvB,MAAM,qBAAqB,EAAE,CAAC;IAE9B,sEAAsE;IACtE,MAAM,kBAAkB,EAAE,CAAC;IAE3B,sBAAsB;IACtB,MAAM,cAAc,EAAE,CAAC;AACzB,CAAC;AAED,oBAAoB;AACpB,SAAS,QAAQ,CAAC,MAAc;IAC9B,GAAG,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,0BAA0B,CAAC,CAAC;IACjD,aAAa,EAAE,CAAC;IAChB,iBAAiB,EAAE,CAAC;IAEpB,uBAAuB;IACvB,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,WAAW,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;SACxC,IAAI,CAAC,GAAG,EAAE;QACT,GAAG,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;QAC9B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC;SACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;QACb,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,EAAE,uBAAuB,CAAC,CAAC;QAC5C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;AACP,CAAC;AAED,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;AACjD,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;AAE/C,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IACnB,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,EAAE,wBAAwB,CAAC,CAAC;IAC7C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"} \ No newline at end of file diff --git a/worker/dist/preview/match.d.ts b/worker/dist/preview/match.d.ts deleted file mode 100644 index 50201fd..0000000 --- a/worker/dist/preview/match.d.ts +++ /dev/null @@ -1,22 +0,0 @@ -export interface TelegramPhoto { - id: bigint; - date: Date; - /** Caption text on the photo message (if any). */ - caption: string; - /** The smallest photo size available — used as thumbnail. */ - fileId: string; - fileSize: number; -} -export interface ArchiveRef { - baseName: string; - firstMessageId: bigint; - firstMessageDate: Date; -} -/** - * Try to match a photo message to an archive by: - * 1. Caption contains the archive baseName (without extension) - * 2. Photo was posted within ±10 messages (time-window: ±6 hours) - * - * Returns the best match (closest in time), or null. - */ -export declare function matchPreviewToArchive(photos: TelegramPhoto[], archives: ArchiveRef[]): Map; diff --git a/worker/dist/preview/match.js b/worker/dist/preview/match.js deleted file mode 100644 index 02dd716..0000000 --- a/worker/dist/preview/match.js +++ /dev/null @@ -1,53 +0,0 @@ -import { childLogger } from "../util/logger.js"; -const log = childLogger("preview-match"); -/** - * Try to match a photo message to an archive by: - * 1. Caption contains the archive baseName (without extension) - * 2. Photo was posted within ±10 messages (time-window: ±6 hours) - * - * Returns the best match (closest in time), or null. - */ -export function matchPreviewToArchive(photos, archives) { - const results = new Map(); - const TIME_WINDOW_MS = 6 * 60 * 60 * 1000; // 6 hours - for (const archive of archives) { - // Normalize the archive base name for matching - const normalizedBase = normalizeForMatch(archive.baseName); - if (!normalizedBase) - continue; - let bestMatch = null; - let bestTimeDiff = Infinity; - for (const photo of photos) { - const timeDiff = Math.abs(photo.date.getTime() - archive.firstMessageDate.getTime()); - // Must be within time window - if (timeDiff > TIME_WINDOW_MS) - continue; - // Check if the photo caption contains the archive base name - const normalizedCaption = normalizeForMatch(photo.caption); - if (!normalizedCaption) - continue; - const matches = normalizedCaption.includes(normalizedBase) || - normalizedBase.includes(normalizedCaption); - if (matches && timeDiff < bestTimeDiff) { - bestMatch = photo; - bestTimeDiff = timeDiff; - } - } - if (bestMatch) { - log.debug({ baseName: archive.baseName, photoId: bestMatch.id.toString() }, "Matched preview photo to archive"); - results.set(archive.baseName, bestMatch); - } - } - return results; -} -/** - * Strip extension, punctuation, and normalize for fuzzy matching. - */ -function normalizeForMatch(input) { - return input - .toLowerCase() - .replace(/\.[a-z0-9]{1,5}$/i, "") // strip extension - .replace(/[_\-.\s]+/g, " ") // normalize separators - .trim(); -} -//# sourceMappingURL=match.js.map \ No newline at end of file diff --git a/worker/dist/preview/match.js.map b/worker/dist/preview/match.js.map deleted file mode 100644 index 6e0d388..0000000 --- a/worker/dist/preview/match.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"match.js","sourceRoot":"","sources":["../../src/preview/match.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,MAAM,GAAG,GAAG,WAAW,CAAC,eAAe,CAAC,CAAC;AAkBzC;;;;;;GAMG;AACH,MAAM,UAAU,qBAAqB,CACnC,MAAuB,EACvB,QAAsB;IAEtB,MAAM,OAAO,GAAG,IAAI,GAAG,EAAyB,CAAC;IACjD,MAAM,cAAc,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,UAAU;IAErD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,+CAA+C;QAC/C,MAAM,cAAc,GAAG,iBAAiB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC3D,IAAI,CAAC,cAAc;YAAE,SAAS;QAE9B,IAAI,SAAS,GAAyB,IAAI,CAAC;QAC3C,IAAI,YAAY,GAAG,QAAQ,CAAC;QAE5B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CACvB,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,OAAO,CAAC,gBAAgB,CAAC,OAAO,EAAE,CAC1D,CAAC;YAEF,6BAA6B;YAC7B,IAAI,QAAQ,GAAG,cAAc;gBAAE,SAAS;YAExC,4DAA4D;YAC5D,MAAM,iBAAiB,GAAG,iBAAiB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAC3D,IAAI,CAAC,iBAAiB;gBAAE,SAAS;YAEjC,MAAM,OAAO,GACX,iBAAiB,CAAC,QAAQ,CAAC,cAAc,CAAC;gBAC1C,cAAc,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC;YAE7C,IAAI,OAAO,IAAI,QAAQ,GAAG,YAAY,EAAE,CAAC;gBACvC,SAAS,GAAG,KAAK,CAAC;gBAClB,YAAY,GAAG,QAAQ,CAAC;YAC1B,CAAC;QACH,CAAC;QAED,IAAI,SAAS,EAAE,CAAC;YACd,GAAG,CAAC,KAAK,CACP,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,OAAO,EAAE,SAAS,CAAC,EAAE,CAAC,QAAQ,EAAE,EAAE,EAChE,kCAAkC,CACnC,CAAC;YACF,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;QAC3C,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB,CAAC,KAAa;IACtC,OAAO,KAAK;SACT,WAAW,EAAE;SACb,OAAO,CAAC,mBAAmB,EAAE,EAAE,CAAC,CAAC,kBAAkB;SACnD,OAAO,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC,uBAAuB;SAClD,IAAI,EAAE,CAAC;AACZ,CAAC"} \ No newline at end of file diff --git a/worker/dist/scheduler.d.ts b/worker/dist/scheduler.d.ts deleted file mode 100644 index 1825681..0000000 --- a/worker/dist/scheduler.d.ts +++ /dev/null @@ -1,13 +0,0 @@ -/** - * Start the scheduler. Runs an immediate first cycle, then schedules subsequent ones. - */ -export declare function startScheduler(): Promise; -/** - * Trigger an immediate ingestion cycle (e.g. from the admin UI). - * If a cycle is already running, this is a no-op. - */ -export declare function triggerImmediateCycle(): Promise; -/** - * Stop the scheduler gracefully. - */ -export declare function stopScheduler(): void; diff --git a/worker/dist/scheduler.js b/worker/dist/scheduler.js deleted file mode 100644 index 58f5585..0000000 --- a/worker/dist/scheduler.js +++ /dev/null @@ -1,121 +0,0 @@ -import { config } from "./util/config.js"; -import { childLogger } from "./util/logger.js"; -import { withTdlibMutex } from "./util/mutex.js"; -import { getActiveAccounts, getPendingAccounts } from "./db/queries.js"; -import { runWorkerForAccount, authenticateAccount } from "./worker.js"; -const log = childLogger("scheduler"); -let running = false; -let timer = null; -let cycleCount = 0; -/** - * Maximum time for a single ingestion cycle (ms). - * After this, new accounts won't be started (in-progress work finishes). - * Default: 4 hours. Configurable via WORKER_CYCLE_TIMEOUT_MINUTES. - */ -const CYCLE_TIMEOUT_MS = (parseInt(process.env.WORKER_CYCLE_TIMEOUT_MINUTES ?? "240", 10)) * 60 * 1000; -/** - * Run one ingestion cycle: - * 1. Authenticate any PENDING accounts (triggers SMS code flow + auto-fetch channels) - * 2. Process all active AUTHENTICATED accounts for ingestion - * - * All TDLib operations are wrapped in the mutex to ensure only one client - * runs at a time (also shared with the fetch listener for on-demand requests). - * - * The cycle has a configurable timeout (WORKER_CYCLE_TIMEOUT_MINUTES, default 4h). - * Once the timeout elapses, no new accounts will be started but any in-progress - * account processing is allowed to finish its current archive set. - */ -async function runCycle() { - if (running) { - log.warn("Previous cycle still running, skipping"); - return; - } - running = true; - cycleCount++; - const cycleStart = Date.now(); - log.info({ cycle: cycleCount, timeoutMinutes: CYCLE_TIMEOUT_MS / 60_000 }, "Starting ingestion cycle"); - try { - // ── Phase 1: Authenticate pending accounts ── - const pendingAccounts = await getPendingAccounts(); - if (pendingAccounts.length > 0) { - log.info({ count: pendingAccounts.length }, "Found pending accounts, starting authentication"); - for (const account of pendingAccounts) { - if (Date.now() - cycleStart > CYCLE_TIMEOUT_MS) { - log.warn("Cycle timeout reached during authentication phase, stopping"); - break; - } - await withTdlibMutex(`auth:${account.phone}`, () => authenticateAccount(account)); - } - } - // ── Phase 2: Ingest for authenticated accounts ── - const accounts = await getActiveAccounts(); - if (accounts.length === 0) { - log.info("No active authenticated accounts, nothing to ingest"); - return; - } - log.info({ accountCount: accounts.length }, "Processing accounts"); - for (const account of accounts) { - if (Date.now() - cycleStart > CYCLE_TIMEOUT_MS) { - log.warn({ elapsed: Math.round((Date.now() - cycleStart) / 60_000), timeoutMinutes: CYCLE_TIMEOUT_MS / 60_000 }, "Cycle timeout reached, skipping remaining accounts"); - break; - } - await withTdlibMutex(`ingest:${account.phone}`, () => runWorkerForAccount(account)); - } - log.info({ elapsed: Math.round((Date.now() - cycleStart) / 1000) }, "Ingestion cycle complete"); - } - catch (err) { - log.error({ err }, "Ingestion cycle failed"); - } - finally { - running = false; - } -} -/** - * Schedule the next cycle with jitter. - */ -function scheduleNext() { - const intervalMs = config.workerIntervalMinutes * 60 * 1000; - const jitterMs = Math.random() * config.jitterMinutes * 60 * 1000; - const delay = intervalMs + jitterMs; - log.info({ nextRunInMinutes: Math.round(delay / 60000) }, "Next cycle scheduled"); - timer = setTimeout(async () => { - await runCycle(); - scheduleNext(); - }, delay); -} -/** - * Start the scheduler. Runs an immediate first cycle, then schedules subsequent ones. - */ -export async function startScheduler() { - log.info({ - intervalMinutes: config.workerIntervalMinutes, - jitterMinutes: config.jitterMinutes, - }, "Scheduler starting"); - // Run immediately on start - await runCycle(); - // Then schedule recurring cycles - scheduleNext(); -} -/** - * Trigger an immediate ingestion cycle (e.g. from the admin UI). - * If a cycle is already running, this is a no-op. - */ -export async function triggerImmediateCycle() { - if (running) { - log.info("Cycle already running, ignoring trigger"); - return; - } - log.info("Immediate cycle triggered via UI"); - await runCycle(); -} -/** - * Stop the scheduler gracefully. - */ -export function stopScheduler() { - if (timer) { - clearTimeout(timer); - timer = null; - } - log.info("Scheduler stopped"); -} -//# sourceMappingURL=scheduler.js.map \ No newline at end of file diff --git a/worker/dist/scheduler.js.map b/worker/dist/scheduler.js.map deleted file mode 100644 index c43fa77..0000000 --- a/worker/dist/scheduler.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"scheduler.js","sourceRoot":"","sources":["../src/scheduler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AACxE,OAAO,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAEvE,MAAM,GAAG,GAAG,WAAW,CAAC,WAAW,CAAC,CAAC;AAErC,IAAI,OAAO,GAAG,KAAK,CAAC;AACpB,IAAI,KAAK,GAAyC,IAAI,CAAC;AACvD,IAAI,UAAU,GAAG,CAAC,CAAC;AAEnB;;;;GAIG;AACH,MAAM,gBAAgB,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,4BAA4B,IAAI,KAAK,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;AAEvG;;;;;;;;;;;GAWG;AACH,KAAK,UAAU,QAAQ;IACrB,IAAI,OAAO,EAAE,CAAC;QACZ,GAAG,CAAC,IAAI,CAAC,wCAAwC,CAAC,CAAC;QACnD,OAAO;IACT,CAAC;IAED,OAAO,GAAG,IAAI,CAAC;IACf,UAAU,EAAE,CAAC;IACb,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC9B,GAAG,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,cAAc,EAAE,gBAAgB,GAAG,MAAM,EAAE,EAAE,0BAA0B,CAAC,CAAC;IAEvG,IAAI,CAAC;QACH,+CAA+C;QAC/C,MAAM,eAAe,GAAG,MAAM,kBAAkB,EAAE,CAAC;QACnD,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/B,GAAG,CAAC,IAAI,CACN,EAAE,KAAK,EAAE,eAAe,CAAC,MAAM,EAAE,EACjC,iDAAiD,CAClD,CAAC;YACF,KAAK,MAAM,OAAO,IAAI,eAAe,EAAE,CAAC;gBACtC,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,GAAG,gBAAgB,EAAE,CAAC;oBAC/C,GAAG,CAAC,IAAI,CAAC,6DAA6D,CAAC,CAAC;oBACxE,MAAM;gBACR,CAAC;gBACD,MAAM,cAAc,CAAC,QAAQ,OAAO,CAAC,KAAK,EAAE,EAAE,GAAG,EAAE,CACjD,mBAAmB,CAAC,OAAO,CAAC,CAC7B,CAAC;YACJ,CAAC;QACH,CAAC;QAED,mDAAmD;QACnD,MAAM,QAAQ,GAAG,MAAM,iBAAiB,EAAE,CAAC;QAE3C,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,GAAG,CAAC,IAAI,CAAC,qDAAqD,CAAC,CAAC;YAChE,OAAO;QACT,CAAC;QAED,GAAG,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,QAAQ,CAAC,MAAM,EAAE,EAAE,qBAAqB,CAAC,CAAC;QAEnE,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,GAAG,gBAAgB,EAAE,CAAC;gBAC/C,GAAG,CAAC,IAAI,CACN,EAAE,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC,GAAG,MAAM,CAAC,EAAE,cAAc,EAAE,gBAAgB,GAAG,MAAM,EAAE,EACtG,oDAAoD,CACrD,CAAC;gBACF,MAAM;YACR,CAAC;YACD,MAAM,cAAc,CAAC,UAAU,OAAO,CAAC,KAAK,EAAE,EAAE,GAAG,EAAE,CACnD,mBAAmB,CAAC,OAAO,CAAC,CAC7B,CAAC;QACJ,CAAC;QAED,GAAG,CAAC,IAAI,CACN,EAAE,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC,GAAG,IAAI,CAAC,EAAE,EACzD,0BAA0B,CAC3B,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,EAAE,wBAAwB,CAAC,CAAC;IAC/C,CAAC;YAAS,CAAC;QACT,OAAO,GAAG,KAAK,CAAC;IAClB,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,YAAY;IACnB,MAAM,UAAU,GAAG,MAAM,CAAC,qBAAqB,GAAG,EAAE,GAAG,IAAI,CAAC;IAC5D,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC,aAAa,GAAG,EAAE,GAAG,IAAI,CAAC;IAClE,MAAM,KAAK,GAAG,UAAU,GAAG,QAAQ,CAAC;IAEpC,GAAG,CAAC,IAAI,CACN,EAAE,gBAAgB,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,EAAE,EAC/C,sBAAsB,CACvB,CAAC;IAEF,KAAK,GAAG,UAAU,CAAC,KAAK,IAAI,EAAE;QAC5B,MAAM,QAAQ,EAAE,CAAC;QACjB,YAAY,EAAE,CAAC;IACjB,CAAC,EAAE,KAAK,CAAC,CAAC;AACZ,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc;IAClC,GAAG,CAAC,IAAI,CACN;QACE,eAAe,EAAE,MAAM,CAAC,qBAAqB;QAC7C,aAAa,EAAE,MAAM,CAAC,aAAa;KACpC,EACD,oBAAoB,CACrB,CAAC;IAEF,2BAA2B;IAC3B,MAAM,QAAQ,EAAE,CAAC;IAEjB,iCAAiC;IACjC,YAAY,EAAE,CAAC;AACjB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB;IACzC,IAAI,OAAO,EAAE,CAAC;QACZ,GAAG,CAAC,IAAI,CAAC,yCAAyC,CAAC,CAAC;QACpD,OAAO;IACT,CAAC;IACD,GAAG,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;IAC7C,MAAM,QAAQ,EAAE,CAAC;AACnB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,aAAa;IAC3B,IAAI,KAAK,EAAE,CAAC;QACV,YAAY,CAAC,KAAK,CAAC,CAAC;QACpB,KAAK,GAAG,IAAI,CAAC;IACf,CAAC;IACD,GAAG,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;AAChC,CAAC"} \ No newline at end of file diff --git a/worker/dist/tdlib/chats.d.ts b/worker/dist/tdlib/chats.d.ts deleted file mode 100644 index cf9b4e6..0000000 --- a/worker/dist/tdlib/chats.d.ts +++ /dev/null @@ -1,31 +0,0 @@ -import type { Client } from "tdl"; -export interface TelegramChatInfo { - chatId: bigint; - title: string; - type: "channel" | "supergroup" | "group" | "private" | "other"; - isForum: boolean; - memberCount?: number; -} -/** - * Fetch all chats the account is a member of. - * Uses TDLib's getChats to load the chat list, then getChat for details. - * Filters to channels and supergroups only (groups/privates are not useful for ingestion). - */ -export declare function getAccountChats(client: Client): Promise; -/** - * Generate an invite link for a chat. The account must be an admin or have - * invite link permissions. - */ -export declare function generateInviteLink(client: Client, chatId: bigint): Promise; -/** - * Create a new supergroup (private group) via TDLib. - * Returns the chat ID and title. - */ -export declare function createSupergroup(client: Client, title: string): Promise<{ - chatId: bigint; - title: string; -}>; -/** - * Join a chat using an invite link. - */ -export declare function joinChatByInviteLink(client: Client, inviteLink: string): Promise; diff --git a/worker/dist/tdlib/chats.js b/worker/dist/tdlib/chats.js deleted file mode 100644 index 4f8a305..0000000 --- a/worker/dist/tdlib/chats.js +++ /dev/null @@ -1,124 +0,0 @@ -import { childLogger } from "../util/logger.js"; -import { config } from "../util/config.js"; -const log = childLogger("chats"); -/** - * Fetch all chats the account is a member of. - * Uses TDLib's getChats to load the chat list, then getChat for details. - * Filters to channels and supergroups only (groups/privates are not useful for ingestion). - */ -export async function getAccountChats(client) { - const chats = []; - // Load main chat list — TDLib loads in batches - let offsetOrder = "9223372036854775807"; // max int64 as string - let offsetChatId = 0; - let hasMore = true; - while (hasMore) { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const result = (await client.invoke({ - _: "getChats", - chat_list: { _: "chatListMain" }, - limit: 100, - })); - if (!result.chat_ids || result.chat_ids.length === 0) { - break; - } - for (const chatId of result.chat_ids) { - try { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const chat = (await client.invoke({ - _: "getChat", - chat_id: chatId, - })); - const chatType = chat.type?._; - let type = "other"; - let isForum = false; - if (chatType === "chatTypeSupergroup") { - // Get supergroup details to check if it's a channel or group - try { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const sg = (await client.invoke({ - _: "getSupergroup", - supergroup_id: chat.type.supergroup_id, - })); - type = sg.is_channel ? "channel" : "supergroup"; - isForum = sg.is_forum ?? false; - } - catch { - type = "supergroup"; - } - } - else if (chatType === "chatTypeBasicGroup") { - type = "group"; - } - else if (chatType === "chatTypePrivate" || chatType === "chatTypeSecret") { - type = "private"; - } - // Only include channels and supergroups - if (type === "channel" || type === "supergroup") { - chats.push({ - chatId: BigInt(chatId), - title: chat.title ?? `Chat ${chatId}`, - type, - isForum, - }); - } - } - catch (err) { - log.warn({ chatId, err }, "Failed to get chat details, skipping"); - } - } - // getChats with chatListMain returns all chats at once in newer TDLib versions - // So we break after the first batch - hasMore = false; - await sleep(config.apiDelayMs); - } - log.info({ total: chats.length }, "Fetched channels/supergroups from Telegram"); - return chats; -} -/** - * Generate an invite link for a chat. The account must be an admin or have - * invite link permissions. - */ -export async function generateInviteLink(client, chatId) { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const result = (await client.invoke({ - _: "createChatInviteLink", - chat_id: Number(chatId), - name: "DragonsStash Auto-Join", - creates_join_request: false, - })); - const link = result.invite_link; - log.info({ chatId: chatId.toString(), link }, "Generated invite link"); - return link; -} -/** - * Create a new supergroup (private group) via TDLib. - * Returns the chat ID and title. - */ -export async function createSupergroup(client, title) { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const result = (await client.invoke({ - _: "createNewSupergroupChat", - title, - is_forum: false, - is_channel: false, - description: "DragonsStash archive destination — all accounts write here", - })); - const chatId = BigInt(result.id); - log.info({ chatId: chatId.toString(), title }, "Created new supergroup"); - return { chatId, title: result.title ?? title }; -} -/** - * Join a chat using an invite link. - */ -export async function joinChatByInviteLink(client, inviteLink) { - await client.invoke({ - _: "joinChatByInviteLink", - invite_link: inviteLink, - }); - log.info({ inviteLink }, "Joined chat by invite link"); -} -function sleep(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} -//# sourceMappingURL=chats.js.map \ No newline at end of file diff --git a/worker/dist/tdlib/chats.js.map b/worker/dist/tdlib/chats.js.map deleted file mode 100644 index a5b7c30..0000000 --- a/worker/dist/tdlib/chats.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"chats.js","sourceRoot":"","sources":["../../src/tdlib/chats.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,MAAM,GAAG,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;AAUjC;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,MAAc;IAEd,MAAM,KAAK,GAAuB,EAAE,CAAC;IAErC,+CAA+C;IAC/C,IAAI,WAAW,GAAG,qBAAqB,CAAC,CAAC,sBAAsB;IAC/D,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,OAAO,GAAG,IAAI,CAAC;IAEnB,OAAO,OAAO,EAAE,CAAC;QACf,8DAA8D;QAC9D,MAAM,MAAM,GAAG,CAAC,MAAM,MAAM,CAAC,MAAM,CAAC;YAClC,CAAC,EAAE,UAAU;YACb,SAAS,EAAE,EAAE,CAAC,EAAE,cAAc,EAAE;YAChC,KAAK,EAAE,GAAG;SACX,CAAC,CAA2B,CAAC;QAE9B,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrD,MAAM;QACR,CAAC;QAED,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YACrC,IAAI,CAAC;gBACH,8DAA8D;gBAC9D,MAAM,IAAI,GAAG,CAAC,MAAM,MAAM,CAAC,MAAM,CAAC;oBAChC,CAAC,EAAE,SAAS;oBACZ,OAAO,EAAE,MAAM;iBAChB,CAAC,CAAQ,CAAC;gBAEX,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC9B,IAAI,IAAI,GAA6B,OAAO,CAAC;gBAC7C,IAAI,OAAO,GAAG,KAAK,CAAC;gBAEpB,IAAI,QAAQ,KAAK,oBAAoB,EAAE,CAAC;oBACtC,6DAA6D;oBAC7D,IAAI,CAAC;wBACH,8DAA8D;wBAC9D,MAAM,EAAE,GAAG,CAAC,MAAM,MAAM,CAAC,MAAM,CAAC;4BAC9B,CAAC,EAAE,eAAe;4BAClB,aAAa,EAAE,IAAI,CAAC,IAAI,CAAC,aAAa;yBACvC,CAAC,CAAQ,CAAC;wBAEX,IAAI,GAAG,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC;wBAChD,OAAO,GAAG,EAAE,CAAC,QAAQ,IAAI,KAAK,CAAC;oBACjC,CAAC;oBAAC,MAAM,CAAC;wBACP,IAAI,GAAG,YAAY,CAAC;oBACtB,CAAC;gBACH,CAAC;qBAAM,IAAI,QAAQ,KAAK,oBAAoB,EAAE,CAAC;oBAC7C,IAAI,GAAG,OAAO,CAAC;gBACjB,CAAC;qBAAM,IAAI,QAAQ,KAAK,iBAAiB,IAAI,QAAQ,KAAK,gBAAgB,EAAE,CAAC;oBAC3E,IAAI,GAAG,SAAS,CAAC;gBACnB,CAAC;gBAED,wCAAwC;gBACxC,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,YAAY,EAAE,CAAC;oBAChD,KAAK,CAAC,IAAI,CAAC;wBACT,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC;wBACtB,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,QAAQ,MAAM,EAAE;wBACrC,IAAI;wBACJ,OAAO;qBACR,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,GAAG,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,sCAAsC,CAAC,CAAC;YACpE,CAAC;QACH,CAAC;QAED,+EAA+E;QAC/E,oCAAoC;QACpC,OAAO,GAAG,KAAK,CAAC;QAEhB,MAAM,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IACjC,CAAC;IAED,GAAG,CAAC,IAAI,CACN,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,EACvB,4CAA4C,CAC7C,CAAC;IAEF,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,MAAc,EACd,MAAc;IAEd,8DAA8D;IAC9D,MAAM,MAAM,GAAG,CAAC,MAAM,MAAM,CAAC,MAAM,CAAC;QAClC,CAAC,EAAE,sBAAsB;QACzB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC;QACvB,IAAI,EAAE,wBAAwB;QAC9B,oBAAoB,EAAE,KAAK;KAC5B,CAAC,CAAQ,CAAC;IAEX,MAAM,IAAI,GAAG,MAAM,CAAC,WAAqB,CAAC;IAC1C,GAAG,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,EAAE,uBAAuB,CAAC,CAAC;IACvE,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,MAAc,EACd,KAAa;IAEb,8DAA8D;IAC9D,MAAM,MAAM,GAAG,CAAC,MAAM,MAAM,CAAC,MAAM,CAAC;QAClC,CAAC,EAAE,yBAAyB;QAC5B,KAAK;QACL,QAAQ,EAAE,KAAK;QACf,UAAU,EAAE,KAAK;QACjB,WAAW,EAAE,4DAA4D;KAC1E,CAAC,CAAQ,CAAC;IAEX,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACjC,GAAG,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,EAAE,wBAAwB,CAAC,CAAC;IACzE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,KAAK,EAAE,CAAC;AAClD,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,MAAc,EACd,UAAkB;IAElB,MAAM,MAAM,CAAC,MAAM,CAAC;QAClB,CAAC,EAAE,sBAAsB;QACzB,WAAW,EAAE,UAAU;KACxB,CAAC,CAAC;IACH,GAAG,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,EAAE,4BAA4B,CAAC,CAAC;AACzD,CAAC;AAED,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC"} \ No newline at end of file diff --git a/worker/dist/tdlib/client.d.ts b/worker/dist/tdlib/client.d.ts deleted file mode 100644 index 477bd64..0000000 --- a/worker/dist/tdlib/client.d.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { type Client } from "tdl"; -interface AccountConfig { - id: string; - phone: string; -} -/** - * Create and authenticate a TDLib client for a Telegram account. - * Authentication flow communicates with the admin UI via the database: - * - Worker sets authState to AWAITING_CODE when TDLib asks for phone code - * - Admin enters the code via UI, which writes it to authCode field - * - Worker polls DB for the code and feeds it to TDLib - */ -export declare function createTdlibClient(account: AccountConfig): Promise; -/** - * Close a TDLib client gracefully. - */ -export declare function closeTdlibClient(client: Client): Promise; -export {}; diff --git a/worker/dist/tdlib/client.js b/worker/dist/tdlib/client.js deleted file mode 100644 index 9d64701..0000000 --- a/worker/dist/tdlib/client.js +++ /dev/null @@ -1,96 +0,0 @@ -import tdl, { createClient } from "tdl"; -import { getTdjson } from "prebuilt-tdlib"; -import path from "path"; -import { config } from "../util/config.js"; -import { childLogger } from "../util/logger.js"; -import { updateAccountAuthState, getAccountAuthCode, } from "../db/queries.js"; -const log = childLogger("tdlib-client"); -// Configure tdl to use the prebuilt tdjson shared library -tdl.configure({ tdjson: getTdjson() }); -/** - * Create and authenticate a TDLib client for a Telegram account. - * Authentication flow communicates with the admin UI via the database: - * - Worker sets authState to AWAITING_CODE when TDLib asks for phone code - * - Admin enters the code via UI, which writes it to authCode field - * - Worker polls DB for the code and feeds it to TDLib - */ -export async function createTdlibClient(account) { - const dbPath = path.join(config.tdlibStateDir, account.id); - const client = createClient({ - apiId: config.telegramApiId, - apiHash: config.telegramApiHash, - databaseDirectory: dbPath, - filesDirectory: path.join(dbPath, "files"), - }); - client.on("error", (err) => { - log.error({ err, accountId: account.id }, "TDLib client error"); - }); - try { - await client.login(() => ({ - getPhoneNumber: async () => { - log.info({ accountId: account.id }, "TDLib requesting phone number"); - return account.phone; - }, - getAuthCode: async () => { - log.info({ accountId: account.id }, "TDLib requesting auth code"); - await updateAccountAuthState(account.id, "AWAITING_CODE"); - // Poll database for the code entered via admin UI - const code = await pollForAuthCode(account.id); - if (!code) { - throw new Error("Auth code not provided within timeout"); - } - // Clear the code after reading - await updateAccountAuthState(account.id, "AUTHENTICATED", null); - return code; - }, - getPassword: async () => { - log.info({ accountId: account.id }, "TDLib requesting 2FA password"); - await updateAccountAuthState(account.id, "AWAITING_PASSWORD"); - // Poll database for the password entered via admin UI - const code = await pollForAuthCode(account.id); - if (!code) { - throw new Error("2FA password not provided within timeout"); - } - await updateAccountAuthState(account.id, "AUTHENTICATED", null); - return code; - }, - })); - await updateAccountAuthState(account.id, "AUTHENTICATED"); - log.info({ accountId: account.id }, "TDLib client authenticated"); - return client; - } - catch (err) { - log.error({ err, accountId: account.id }, "TDLib authentication failed"); - await updateAccountAuthState(account.id, "EXPIRED"); - throw err; - } -} -/** - * Poll the database every 5 seconds for an auth code, up to 5 minutes. - */ -async function pollForAuthCode(accountId, timeoutMs = 300_000) { - const start = Date.now(); - while (Date.now() - start < timeoutMs) { - const result = await getAccountAuthCode(accountId); - if (result?.authCode) { - return result.authCode; - } - await sleep(5000); - } - return null; -} -function sleep(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} -/** - * Close a TDLib client gracefully. - */ -export async function closeTdlibClient(client) { - try { - await client.close(); - } - catch (err) { - log.warn({ err }, "Error closing TDLib client"); - } -} -//# sourceMappingURL=client.js.map \ No newline at end of file diff --git a/worker/dist/tdlib/client.js.map b/worker/dist/tdlib/client.js.map deleted file mode 100644 index 3b185cb..0000000 --- a/worker/dist/tdlib/client.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"client.js","sourceRoot":"","sources":["../../src/tdlib/client.ts"],"names":[],"mappings":"AAAA,OAAO,GAAG,EAAE,EAAE,YAAY,EAAe,MAAM,KAAK,CAAC;AACrD,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EACL,sBAAsB,EACtB,kBAAkB,GACnB,MAAM,kBAAkB,CAAC;AAE1B,MAAM,GAAG,GAAG,WAAW,CAAC,cAAc,CAAC,CAAC;AAExC,0DAA0D;AAC1D,GAAG,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;AAOvC;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,OAAsB;IAEtB,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC;IAE3D,MAAM,MAAM,GAAG,YAAY,CAAC;QAC1B,KAAK,EAAE,MAAM,CAAC,aAAa;QAC3B,OAAO,EAAE,MAAM,CAAC,eAAe;QAC/B,iBAAiB,EAAE,MAAM;QACzB,cAAc,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC;KAC3C,CAAC,CAAC;IAEH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;QACzB,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,EAAE,oBAAoB,CAAC,CAAC;IAClE,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC;YACxB,cAAc,EAAE,KAAK,IAAI,EAAE;gBACzB,GAAG,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,EAAE,+BAA+B,CAAC,CAAC;gBACrE,OAAO,OAAO,CAAC,KAAK,CAAC;YACvB,CAAC;YACD,WAAW,EAAE,KAAK,IAAI,EAAE;gBACtB,GAAG,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,EAAE,4BAA4B,CAAC,CAAC;gBAClE,MAAM,sBAAsB,CAAC,OAAO,CAAC,EAAE,EAAE,eAAe,CAAC,CAAC;gBAE1D,kDAAkD;gBAClD,MAAM,IAAI,GAAG,MAAM,eAAe,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;gBAC/C,IAAI,CAAC,IAAI,EAAE,CAAC;oBACV,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;gBAC3D,CAAC;gBAED,+BAA+B;gBAC/B,MAAM,sBAAsB,CAAC,OAAO,CAAC,EAAE,EAAE,eAAe,EAAE,IAAI,CAAC,CAAC;gBAChE,OAAO,IAAI,CAAC;YACd,CAAC;YACD,WAAW,EAAE,KAAK,IAAI,EAAE;gBACtB,GAAG,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,EAAE,+BAA+B,CAAC,CAAC;gBACrE,MAAM,sBAAsB,CAAC,OAAO,CAAC,EAAE,EAAE,mBAAmB,CAAC,CAAC;gBAE9D,sDAAsD;gBACtD,MAAM,IAAI,GAAG,MAAM,eAAe,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;gBAC/C,IAAI,CAAC,IAAI,EAAE,CAAC;oBACV,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;gBAC9D,CAAC;gBAED,MAAM,sBAAsB,CAAC,OAAO,CAAC,EAAE,EAAE,eAAe,EAAE,IAAI,CAAC,CAAC;gBAChE,OAAO,IAAI,CAAC;YACd,CAAC;SACF,CAAC,CAAC,CAAC;QAEJ,MAAM,sBAAsB,CAAC,OAAO,CAAC,EAAE,EAAE,eAAe,CAAC,CAAC;QAC1D,GAAG,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,EAAE,4BAA4B,CAAC,CAAC;QAClE,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,EAAE,6BAA6B,CAAC,CAAC;QACzE,MAAM,sBAAsB,CAAC,OAAO,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;QACpD,MAAM,GAAG,CAAC;IACZ,CAAC;AACH,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,eAAe,CAC5B,SAAiB,EACjB,SAAS,GAAG,OAAO;IAEnB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,GAAG,SAAS,EAAE,CAAC;QACtC,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,SAAS,CAAC,CAAC;QACnD,IAAI,MAAM,EAAE,QAAQ,EAAE,CAAC;YACrB,OAAO,MAAM,CAAC,QAAQ,CAAC;QACzB,CAAC;QACD,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,MAAc;IACnD,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,EAAE,4BAA4B,CAAC,CAAC;IAClD,CAAC;AACH,CAAC"} \ No newline at end of file diff --git a/worker/dist/tdlib/download.d.ts b/worker/dist/tdlib/download.d.ts deleted file mode 100644 index 4813078..0000000 --- a/worker/dist/tdlib/download.d.ts +++ /dev/null @@ -1,67 +0,0 @@ -import type { Client } from "tdl"; -import type { TelegramMessage } from "../archive/multipart.js"; -import type { TelegramPhoto } from "../preview/match.js"; -/** Maximum number of pages to scan per channel/topic to prevent infinite loops */ -export declare const MAX_SCAN_PAGES = 5000; -/** Timeout for a single TDLib API call (ms) */ -export declare const INVOKE_TIMEOUT_MS = 120000; -export interface ChannelScanResult { - archives: TelegramMessage[]; - photos: TelegramPhoto[]; - totalScanned: number; -} -export type ScanProgressCallback = (messagesScanned: number) => void; -/** - * Invoke a TDLib method with a timeout to prevent indefinite hangs. - * If TDLib does not respond within the timeout, the promise rejects. - */ -export declare function invokeWithTimeout(client: Client, request: Record, timeoutMs?: number): Promise; -/** - * Fetch messages from a channel, stopping once we've scanned past the - * last-processed boundary (with one page of lookback for multipart safety). - * Collects both archive attachments AND photo messages (for preview matching). - * Returns messages in chronological order (oldest first). - * - * When `lastProcessedMessageId` is null (first run), scans everything. - * The worker applies a post-grouping filter to skip fully-processed sets, - * and keeps `packageExistsBySourceMessage` as a safety net. - * - * Safety features: - * - Max page limit to prevent infinite loops - * - Stuck detection: breaks if from_message_id stops advancing - * - Timeout on each TDLib API call - */ -export declare function getChannelMessages(client: Client, chatId: bigint, lastProcessedMessageId?: bigint | null, limit?: number, onProgress?: ScanProgressCallback): Promise; -/** - * Download a photo thumbnail from Telegram and return its raw bytes. - * Uses synchronous download (photos are small, typically < 100KB). - * Returns null if download fails (non-critical). - */ -export declare function downloadPhotoThumbnail(client: Client, fileId: string): Promise; -export interface DownloadProgress { - fileId: string; - fileName: string; - downloadedBytes: number; - totalBytes: number; - percent: number; - isComplete: boolean; -} -export type ProgressCallback = (progress: DownloadProgress) => void; -/** - * Download a file from Telegram to a local path with progress tracking - * and integrity verification. - * - * Progress flow: - * 1. Starts async download via TDLib - * 2. Listens for `updateFile` events to track download progress - * 3. Logs progress at every 10% increment - * 4. Once complete, verifies the local file size matches the expected size - * 5. Moves the file from TDLib's cache to the destination path - * - * Verification: - * - Compares actual file size on disk to the expected size from Telegram - * - Throws on mismatch (partial/corrupt download) - * - Throws on timeout (configurable, scales with file size) - * - Throws if download stops without completing (network error, etc.) - */ -export declare function downloadFile(client: Client, fileId: string, destPath: string, expectedSize: bigint, fileName: string, onProgress?: ProgressCallback): Promise; diff --git a/worker/dist/tdlib/download.js b/worker/dist/tdlib/download.js deleted file mode 100644 index 0b56c0f..0000000 --- a/worker/dist/tdlib/download.js +++ /dev/null @@ -1,307 +0,0 @@ -import { readFile, rename, copyFile, unlink, stat } from "fs/promises"; -import { config } from "../util/config.js"; -import { childLogger } from "../util/logger.js"; -import { isArchiveAttachment } from "../archive/detect.js"; -const log = childLogger("download"); -/** Maximum number of pages to scan per channel/topic to prevent infinite loops */ -export const MAX_SCAN_PAGES = 5000; -/** Timeout for a single TDLib API call (ms) */ -export const INVOKE_TIMEOUT_MS = 120_000; // 2 minutes -/** - * Invoke a TDLib method with a timeout to prevent indefinite hangs. - * If TDLib does not respond within the timeout, the promise rejects. - */ -export async function invokeWithTimeout(client, -// eslint-disable-next-line @typescript-eslint/no-explicit-any -request, timeoutMs = INVOKE_TIMEOUT_MS) { - return new Promise((resolve, reject) => { - const timer = setTimeout(() => { - reject(new Error(`TDLib invoke timed out after ${timeoutMs}ms for ${request._}`)); - }, timeoutMs); - client.invoke(request) - .then((result) => { - clearTimeout(timer); - resolve(result); - }) - .catch((err) => { - clearTimeout(timer); - reject(err); - }); - }); -} -/** - * Fetch messages from a channel, stopping once we've scanned past the - * last-processed boundary (with one page of lookback for multipart safety). - * Collects both archive attachments AND photo messages (for preview matching). - * Returns messages in chronological order (oldest first). - * - * When `lastProcessedMessageId` is null (first run), scans everything. - * The worker applies a post-grouping filter to skip fully-processed sets, - * and keeps `packageExistsBySourceMessage` as a safety net. - * - * Safety features: - * - Max page limit to prevent infinite loops - * - Stuck detection: breaks if from_message_id stops advancing - * - Timeout on each TDLib API call - */ -export async function getChannelMessages(client, chatId, lastProcessedMessageId, limit = 100, onProgress) { - const archives = []; - const photos = []; - const boundary = lastProcessedMessageId ? Number(lastProcessedMessageId) : null; - let currentFromId = 0; - let totalScanned = 0; - let pageCount = 0; - // eslint-disable-next-line no-constant-condition - while (true) { - if (pageCount >= MAX_SCAN_PAGES) { - log.warn({ chatId: chatId.toString(), pageCount, totalScanned }, "Hit max page limit for channel scan, stopping"); - break; - } - pageCount++; - const previousFromId = currentFromId; - const result = await invokeWithTimeout(client, { - _: "getChatHistory", - chat_id: Number(chatId), - from_message_id: currentFromId, - offset: 0, - limit: Math.min(limit, 100), - only_local: false, - }); - if (!result.messages || result.messages.length === 0) - break; - totalScanned += result.messages.length; - for (const msg of result.messages) { - // Check for archive documents - const doc = msg.content?.document; - if (doc?.file_name && doc.document && isArchiveAttachment(doc.file_name)) { - archives.push({ - id: BigInt(msg.id), - fileName: doc.file_name, - fileId: String(doc.document.id), - fileSize: BigInt(doc.document.size), - date: new Date(msg.date * 1000), - }); - continue; - } - // Check for photo messages (potential previews) - const photo = msg.content?.photo; - const caption = msg.content?.caption?.text ?? ""; - if (photo?.sizes && photo.sizes.length > 0) { - const smallest = photo.sizes[0]; - photos.push({ - id: BigInt(msg.id), - date: new Date(msg.date * 1000), - caption, - fileId: String(smallest.photo.id), - fileSize: smallest.photo.size || smallest.photo.expected_size, - }); - } - } - // Report scanning progress after each page - onProgress?.(totalScanned); - currentFromId = result.messages[result.messages.length - 1].id; - // Stuck detection: if from_message_id didn't advance, break to prevent infinite loop - if (currentFromId === previousFromId) { - log.warn({ chatId: chatId.toString(), currentFromId, totalScanned }, "Pagination stuck (from_message_id not advancing), breaking"); - break; - } - // Stop scanning once we've gone past the boundary (this page is the lookback) - if (boundary && currentFromId < boundary) - break; - if (result.messages.length < Math.min(limit, 100)) - break; - // Rate limit delay - await sleep(config.apiDelayMs); - } - log.info({ chatId: chatId.toString(), archives: archives.length, photos: photos.length, totalScanned, pages: pageCount }, "Channel scan complete"); - // Reverse to chronological order (oldest first) so worker processes old→new - return { - archives: archives.reverse(), - photos: photos.reverse(), - totalScanned, - }; -} -/** - * Download a photo thumbnail from Telegram and return its raw bytes. - * Uses synchronous download (photos are small, typically < 100KB). - * Returns null if download fails (non-critical). - */ -export async function downloadPhotoThumbnail(client, fileId) { - const numericId = parseInt(fileId, 10); - try { - const result = (await client.invoke({ - _: "downloadFile", - file_id: numericId, - priority: 1, // Low priority — thumbnails are nice-to-have - offset: 0, - limit: 0, - synchronous: true, // Small file — wait for it - })); - if (result?.local?.is_downloading_completed && result.local.path) { - const data = await readFile(result.local.path); - log.debug({ fileId, bytes: data.length }, "Downloaded photo thumbnail"); - return data; - } - } - catch (err) { - log.warn({ fileId, err }, "Failed to download photo thumbnail"); - } - return null; -} -/** - * Download a file from Telegram to a local path with progress tracking - * and integrity verification. - * - * Progress flow: - * 1. Starts async download via TDLib - * 2. Listens for `updateFile` events to track download progress - * 3. Logs progress at every 10% increment - * 4. Once complete, verifies the local file size matches the expected size - * 5. Moves the file from TDLib's cache to the destination path - * - * Verification: - * - Compares actual file size on disk to the expected size from Telegram - * - Throws on mismatch (partial/corrupt download) - * - Throws on timeout (configurable, scales with file size) - * - Throws if download stops without completing (network error, etc.) - */ -export async function downloadFile(client, fileId, destPath, expectedSize, fileName, onProgress) { - const numericId = parseInt(fileId, 10); - const totalBytes = Number(expectedSize); - log.info({ fileId, fileName, destPath, totalBytes }, "Starting file download"); - // Report initial progress - onProgress?.({ - fileId, - fileName, - downloadedBytes: 0, - totalBytes, - percent: 0, - isComplete: false, - }); - return new Promise((resolve, reject) => { - let lastLoggedPercent = 0; - let settled = false; - // Timeout: 10 minutes per GB, minimum 5 minutes - const timeoutMs = Math.max(5 * 60_000, (totalBytes / (1024 * 1024 * 1024)) * 10 * 60_000); - const timer = setTimeout(() => { - if (!settled) { - settled = true; - cleanup(); - reject(new Error(`Download timed out after ${Math.round(timeoutMs / 60_000)}min for ${fileName}`)); - } - }, timeoutMs); - // Listen for file update events to track progress - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const handleUpdate = (update) => { - if (update?._ !== "updateFile") - return; - const file = update.file; - if (!file || file.id !== numericId) - return; - const downloaded = file.local.downloaded_size; - const percent = totalBytes > 0 ? Math.round((downloaded / totalBytes) * 100) : 0; - // Log at every 10% increment - if (percent >= lastLoggedPercent + 10) { - lastLoggedPercent = percent - (percent % 10); - log.info({ fileId, fileName, downloaded, totalBytes, percent: `${percent}%` }, "Download progress"); - } - // Report to callback - onProgress?.({ - fileId, - fileName, - downloadedBytes: downloaded, - totalBytes, - percent, - isComplete: file.local.is_downloading_completed, - }); - // Download finished - if (file.local.is_downloading_completed) { - if (!settled) { - settled = true; - cleanup(); - verifyAndMove(file.local.path, destPath, totalBytes, fileName, fileId) - .then(resolve) - .catch(reject); - } - } - // Download stopped without completing (network error, cancelled, etc.) - if (!file.local.is_downloading_active && - !file.local.is_downloading_completed) { - if (!settled) { - settled = true; - cleanup(); - reject(new Error(`Download stopped unexpectedly for ${fileName} ` + - `(${downloaded}/${totalBytes} bytes, ${percent}%)`)); - } - } - }; - const cleanup = () => { - clearTimeout(timer); - client.off("update", handleUpdate); - }; - // Subscribe to updates BEFORE starting download - client.on("update", handleUpdate); - // Start async download (non-blocking — progress via updateFile events) - client - .invoke({ - _: "downloadFile", - file_id: numericId, - priority: 32, - offset: 0, - limit: 0, - synchronous: false, - }) - .then((result) => { - // If the file was already cached locally, invoke returns immediately - const file = result; - if (file?.local?.is_downloading_completed && !settled) { - settled = true; - cleanup(); - verifyAndMove(file.local.path, destPath, totalBytes, fileName, fileId) - .then(resolve) - .catch(reject); - } - }) - .catch((err) => { - if (!settled) { - settled = true; - cleanup(); - reject(err); - } - }); - }); -} -/** - * Verify the downloaded file's size matches the expected size, - * then move it to the destination path. - */ -async function verifyAndMove(localPath, destPath, expectedBytes, fileName, fileId) { - const stats = await stat(localPath); - const actualBytes = stats.size; - if (expectedBytes > 0 && actualBytes !== expectedBytes) { - log.error({ fileId, fileName, expectedBytes, actualBytes }, "Download size mismatch — file is incomplete or corrupted"); - throw new Error(`Download verification failed for ${fileName}: ` + - `expected ${expectedBytes} bytes, got ${actualBytes} bytes`); - } - log.info({ fileId, fileName, bytes: actualBytes, destPath }, "File verified and complete"); - // Move from TDLib's cache to our temp directory. - // Use rename first (fast, same filesystem), fall back to copy+delete - // when source and destination are on different filesystems (EXDEV). - try { - await rename(localPath, destPath); - } - catch (err) { - if (err.code === "EXDEV") { - log.debug({ fileId, fileName }, "Cross-device rename — falling back to copy + unlink"); - await copyFile(localPath, destPath); - await unlink(localPath); - } - else { - throw err; - } - } -} -function sleep(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} -//# sourceMappingURL=download.js.map \ No newline at end of file diff --git a/worker/dist/tdlib/download.js.map b/worker/dist/tdlib/download.js.map deleted file mode 100644 index 5fc9f0b..0000000 --- a/worker/dist/tdlib/download.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"download.js","sourceRoot":"","sources":["../../src/tdlib/download.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AACvE,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAI3D,MAAM,GAAG,GAAG,WAAW,CAAC,UAAU,CAAC,CAAC;AAEpC,kFAAkF;AAClF,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,CAAC;AAEnC,+CAA+C;AAC/C,MAAM,CAAC,MAAM,iBAAiB,GAAG,OAAO,CAAC,CAAC,YAAY;AAiEtD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,MAAc;AACd,8DAA8D;AAC9D,OAA4B,EAC5B,SAAS,GAAG,iBAAiB;IAE7B,OAAO,IAAI,OAAO,CAAI,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACxC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,MAAM,CAAC,IAAI,KAAK,CAAC,gCAAgC,SAAS,UAAU,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACpF,CAAC,EAAE,SAAS,CAAC,CAAC;QAEb,MAAM,CAAC,MAAM,CAAC,OAAO,CAAgB;aACnC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE;YACf,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,OAAO,CAAC,MAAM,CAAC,CAAC;QAClB,CAAC,CAAC;aACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACb,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,CAAC,GAAG,CAAC,CAAC;QACd,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,MAAc,EACd,MAAc,EACd,sBAAsC,EACtC,KAAK,GAAG,GAAG,EACX,UAAiC;IAEjC,MAAM,QAAQ,GAAsB,EAAE,CAAC;IACvC,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,MAAM,QAAQ,GAAG,sBAAsB,CAAC,CAAC,CAAC,MAAM,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAEhF,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,iDAAiD;IACjD,OAAO,IAAI,EAAE,CAAC;QACZ,IAAI,SAAS,IAAI,cAAc,EAAE,CAAC;YAChC,GAAG,CAAC,IAAI,CACN,EAAE,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,YAAY,EAAE,EACtD,+CAA+C,CAChD,CAAC;YACF,MAAM;QACR,CAAC;QACD,SAAS,EAAE,CAAC;QAEZ,MAAM,cAAc,GAAG,aAAa,CAAC;QAErC,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAA4B,MAAM,EAAE;YACxE,CAAC,EAAE,gBAAgB;YACnB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC;YACvB,eAAe,EAAE,aAAa;YAC9B,MAAM,EAAE,CAAC;YACT,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC;YAC3B,UAAU,EAAE,KAAK;SAClB,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM;QAE5D,YAAY,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;QAEvC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YAClC,8BAA8B;YAC9B,MAAM,GAAG,GAAG,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC;YAClC,IAAI,GAAG,EAAE,SAAS,IAAI,GAAG,CAAC,QAAQ,IAAI,mBAAmB,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;gBACzE,QAAQ,CAAC,IAAI,CAAC;oBACZ,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;oBAClB,QAAQ,EAAE,GAAG,CAAC,SAAS;oBACvB,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAC/B,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC;oBACnC,IAAI,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC;iBAChC,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;YAED,gDAAgD;YAChD,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC;YACjC,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC;YACjD,IAAI,KAAK,EAAE,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC3C,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAChC,MAAM,CAAC,IAAI,CAAC;oBACV,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;oBAClB,IAAI,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC;oBAC/B,OAAO;oBACP,MAAM,EAAE,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;oBACjC,QAAQ,EAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI,QAAQ,CAAC,KAAK,CAAC,aAAa;iBAC9D,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,2CAA2C;QAC3C,UAAU,EAAE,CAAC,YAAY,CAAC,CAAC;QAE3B,aAAa,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QAE/D,qFAAqF;QACrF,IAAI,aAAa,KAAK,cAAc,EAAE,CAAC;YACrC,GAAG,CAAC,IAAI,CACN,EAAE,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE,EAAE,aAAa,EAAE,YAAY,EAAE,EAC1D,4DAA4D,CAC7D,CAAC;YACF,MAAM;QACR,CAAC;QAED,8EAA8E;QAC9E,IAAI,QAAQ,IAAI,aAAa,GAAG,QAAQ;YAAE,MAAM;QAEhD,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC;YAAE,MAAM;QAEzD,mBAAmB;QACnB,MAAM,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IACjC,CAAC;IAED,GAAG,CAAC,IAAI,CACN,EAAE,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,EAAE,KAAK,EAAE,SAAS,EAAE,EAC/G,uBAAuB,CACxB,CAAC;IAEF,4EAA4E;IAC5E,OAAO;QACL,QAAQ,EAAE,QAAQ,CAAC,OAAO,EAAE;QAC5B,MAAM,EAAE,MAAM,CAAC,OAAO,EAAE;QACxB,YAAY;KACb,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,MAAc,EACd,MAAc;IAEd,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAEvC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,CAAC,MAAM,MAAM,CAAC,MAAM,CAAC;YAClC,CAAC,EAAE,cAAc;YACjB,OAAO,EAAE,SAAS;YAClB,QAAQ,EAAE,CAAC,EAAE,6CAA6C;YAC1D,MAAM,EAAE,CAAC;YACT,KAAK,EAAE,CAAC;YACR,WAAW,EAAE,IAAI,EAAE,2BAA2B;SAC/C,CAAC,CAAW,CAAC;QAEd,IAAI,MAAM,EAAE,KAAK,EAAE,wBAAwB,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;YACjE,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAC/C,GAAG,CAAC,KAAK,CACP,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,MAAM,EAAE,EAC9B,4BAA4B,CAC7B,CAAC;YACF,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,oCAAoC,CAAC,CAAC;IAClE,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAaD;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,MAAc,EACd,MAAc,EACd,QAAgB,EAChB,YAAoB,EACpB,QAAgB,EAChB,UAA6B;IAE7B,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACvC,MAAM,UAAU,GAAG,MAAM,CAAC,YAAY,CAAC,CAAC;IAExC,GAAG,CAAC,IAAI,CACN,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAE,EAC1C,wBAAwB,CACzB,CAAC;IAEF,0BAA0B;IAC1B,UAAU,EAAE,CAAC;QACX,MAAM;QACN,QAAQ;QACR,eAAe,EAAE,CAAC;QAClB,UAAU;QACV,OAAO,EAAE,CAAC;QACV,UAAU,EAAE,KAAK;KAClB,CAAC,CAAC;IAEH,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC3C,IAAI,iBAAiB,GAAG,CAAC,CAAC;QAC1B,IAAI,OAAO,GAAG,KAAK,CAAC;QAEpB,gDAAgD;QAChD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CACxB,CAAC,GAAG,MAAM,EACV,CAAC,UAAU,GAAG,CAAC,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,GAAG,EAAE,GAAG,MAAM,CAClD,CAAC;QACF,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,OAAO,GAAG,IAAI,CAAC;gBACf,OAAO,EAAE,CAAC;gBACV,MAAM,CACJ,IAAI,KAAK,CACP,4BAA4B,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,MAAM,CAAC,WAAW,QAAQ,EAAE,CAChF,CACF,CAAC;YACJ,CAAC;QACH,CAAC,EAAE,SAAS,CAAC,CAAC;QAEd,kDAAkD;QAClD,8DAA8D;QAC9D,MAAM,YAAY,GAAG,CAAC,MAAW,EAAE,EAAE;YACnC,IAAI,MAAM,EAAE,CAAC,KAAK,YAAY;gBAAE,OAAO;YACvC,MAAM,IAAI,GAAG,MAAM,CAAC,IAA0B,CAAC;YAC/C,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE,KAAK,SAAS;gBAAE,OAAO;YAE3C,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC;YAC9C,MAAM,OAAO,GACX,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,UAAU,GAAG,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAEnE,6BAA6B;YAC7B,IAAI,OAAO,IAAI,iBAAiB,GAAG,EAAE,EAAE,CAAC;gBACtC,iBAAiB,GAAG,OAAO,GAAG,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC;gBAC7C,GAAG,CAAC,IAAI,CACN,EAAE,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,OAAO,GAAG,EAAE,EACpE,mBAAmB,CACpB,CAAC;YACJ,CAAC;YAED,qBAAqB;YACrB,UAAU,EAAE,CAAC;gBACX,MAAM;gBACN,QAAQ;gBACR,eAAe,EAAE,UAAU;gBAC3B,UAAU;gBACV,OAAO;gBACP,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,wBAAwB;aAChD,CAAC,CAAC;YAEH,oBAAoB;YACpB,IAAI,IAAI,CAAC,KAAK,CAAC,wBAAwB,EAAE,CAAC;gBACxC,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,OAAO,GAAG,IAAI,CAAC;oBACf,OAAO,EAAE,CAAC;oBACV,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,CAAC;yBACnE,IAAI,CAAC,OAAO,CAAC;yBACb,KAAK,CAAC,MAAM,CAAC,CAAC;gBACnB,CAAC;YACH,CAAC;YAED,uEAAuE;YACvE,IACE,CAAC,IAAI,CAAC,KAAK,CAAC,qBAAqB;gBACjC,CAAC,IAAI,CAAC,KAAK,CAAC,wBAAwB,EACpC,CAAC;gBACD,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,OAAO,GAAG,IAAI,CAAC;oBACf,OAAO,EAAE,CAAC;oBACV,MAAM,CACJ,IAAI,KAAK,CACP,qCAAqC,QAAQ,GAAG;wBAC9C,IAAI,UAAU,IAAI,UAAU,WAAW,OAAO,IAAI,CACrD,CACF,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC,CAAC;QAEF,MAAM,OAAO,GAAG,GAAG,EAAE;YACnB,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;QACrC,CAAC,CAAC;QAEF,gDAAgD;QAChD,MAAM,CAAC,EAAE,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;QAElC,uEAAuE;QACvE,MAAM;aACH,MAAM,CAAC;YACN,CAAC,EAAE,cAAc;YACjB,OAAO,EAAE,SAAS;YAClB,QAAQ,EAAE,EAAE;YACZ,MAAM,EAAE,CAAC;YACT,KAAK,EAAE,CAAC;YACR,WAAW,EAAE,KAAK;SACnB,CAAC;aACD,IAAI,CAAC,CAAC,MAAe,EAAE,EAAE;YACxB,qEAAqE;YACrE,MAAM,IAAI,GAAG,MAA4B,CAAC;YAC1C,IAAI,IAAI,EAAE,KAAK,EAAE,wBAAwB,IAAI,CAAC,OAAO,EAAE,CAAC;gBACtD,OAAO,GAAG,IAAI,CAAC;gBACf,OAAO,EAAE,CAAC;gBACV,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,CAAC;qBACnE,IAAI,CAAC,OAAO,CAAC;qBACb,KAAK,CAAC,MAAM,CAAC,CAAC;YACnB,CAAC;QACH,CAAC,CAAC;aACD,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;YACtB,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,OAAO,GAAG,IAAI,CAAC;gBACf,OAAO,EAAE,CAAC;gBACV,MAAM,CAAC,GAAG,CAAC,CAAC;YACd,CAAC;QACH,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,aAAa,CAC1B,SAAiB,EACjB,QAAgB,EAChB,aAAqB,EACrB,QAAgB,EAChB,MAAc;IAEd,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,CAAC;IACpC,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC;IAE/B,IAAI,aAAa,GAAG,CAAC,IAAI,WAAW,KAAK,aAAa,EAAE,CAAC;QACvD,GAAG,CAAC,KAAK,CACP,EAAE,MAAM,EAAE,QAAQ,EAAE,aAAa,EAAE,WAAW,EAAE,EAChD,0DAA0D,CAC3D,CAAC;QACF,MAAM,IAAI,KAAK,CACb,oCAAoC,QAAQ,IAAI;YAC9C,YAAY,aAAa,eAAe,WAAW,QAAQ,CAC9D,CAAC;IACJ,CAAC;IAED,GAAG,CAAC,IAAI,CACN,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,EAClD,4BAA4B,CAC7B,CAAC;IAEF,iDAAiD;IACjD,qEAAqE;IACrE,oEAAoE;IACpE,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IACpC,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,IAAK,GAA6B,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YACpD,GAAG,CAAC,KAAK,CACP,EAAE,MAAM,EAAE,QAAQ,EAAE,EACpB,qDAAqD,CACtD,CAAC;YACF,MAAM,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;YACpC,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;QAC1B,CAAC;aAAM,CAAC;YACN,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC"} \ No newline at end of file diff --git a/worker/dist/tdlib/topics.d.ts b/worker/dist/tdlib/topics.d.ts deleted file mode 100644 index 23ecbae..0000000 --- a/worker/dist/tdlib/topics.d.ts +++ /dev/null @@ -1,32 +0,0 @@ -import type { Client } from "tdl"; -import type { ChannelScanResult, ScanProgressCallback } from "./download.js"; -export interface ForumTopic { - topicId: bigint; - name: string; -} -/** - * Check if a chat is a forum supergroup (topics enabled). - */ -export declare function isChatForum(client: Client, chatId: bigint): Promise; -/** - * Get all forum topics in a supergroup. - * Includes stuck detection and timeout protection on API calls. - */ -export declare function getForumTopicList(client: Client, chatId: bigint): Promise; -/** - * Fetch messages from a specific forum topic (thread), stopping once - * we've scanned past the last-processed boundary (with one page of lookback). - * Uses searchChatMessages with message_thread_id to scan within a topic. - * - * Returns messages in chronological order (oldest first). - * - * When `lastProcessedMessageId` is null (first run), scans everything. - * The worker applies a post-grouping filter to skip fully-processed sets, - * and keeps `packageExistsBySourceMessage` as a safety net. - * - * Safety features: - * - Max page limit to prevent infinite loops - * - Stuck detection: breaks if from_message_id stops advancing - * - Timeout on each TDLib API call - */ -export declare function getTopicMessages(client: Client, chatId: bigint, topicId: bigint, lastProcessedMessageId?: bigint | null, limit?: number, onProgress?: ScanProgressCallback): Promise; diff --git a/worker/dist/tdlib/topics.js b/worker/dist/tdlib/topics.js deleted file mode 100644 index 328730f..0000000 --- a/worker/dist/tdlib/topics.js +++ /dev/null @@ -1,196 +0,0 @@ -import { config } from "../util/config.js"; -import { childLogger } from "../util/logger.js"; -import { isArchiveAttachment } from "../archive/detect.js"; -import { invokeWithTimeout, MAX_SCAN_PAGES } from "./download.js"; -const log = childLogger("topics"); -/** - * Check if a chat is a forum supergroup (topics enabled). - */ -export async function isChatForum(client, chatId) { - try { - const chat = await invokeWithTimeout(client, { - _: "getChat", - chat_id: Number(chatId), - }); - if (chat.type?._ === "chatTypeSupergroup" && chat.type.is_forum) { - return true; - } - // Also check via getSupergroup for older TDLib versions - if (chat.type?._ === "chatTypeSupergroup" && chat.type.supergroup_id) { - const sg = await invokeWithTimeout(client, { - _: "getSupergroup", - supergroup_id: chat.type.supergroup_id, - }); - return sg.is_forum === true; - } - return false; - } - catch (err) { - log.warn({ err, chatId: chatId.toString() }, "Failed to check if chat is forum"); - return false; - } -} -/** - * Get all forum topics in a supergroup. - * Includes stuck detection and timeout protection on API calls. - */ -export async function getForumTopicList(client, chatId) { - const topics = []; - let offsetDate = 0; - let offsetMessageId = 0; - let offsetMessageThreadId = 0; - let pageCount = 0; - // eslint-disable-next-line no-constant-condition - while (true) { - if (pageCount >= MAX_SCAN_PAGES) { - log.warn({ chatId: chatId.toString(), pageCount, topicCount: topics.length }, "Hit max page limit for topic enumeration, stopping"); - break; - } - pageCount++; - const prevOffsetDate = offsetDate; - const prevOffsetMessageId = offsetMessageId; - const prevOffsetMessageThreadId = offsetMessageThreadId; - const result = await invokeWithTimeout(client, { - _: "getForumTopics", - chat_id: Number(chatId), - query: "", - offset_date: offsetDate, - offset_message_id: offsetMessageId, - offset_message_thread_id: offsetMessageThreadId, - limit: 100, - }); - if (!result.topics || result.topics.length === 0) - break; - for (const t of result.topics) { - if (!t.info?.message_thread_id) - continue; - // Skip the "General" topic — it's not creator-specific - if (t.info.is_general) - continue; - topics.push({ - topicId: BigInt(t.info.message_thread_id), - name: t.info.name ?? "Unnamed", - }); - } - // Check if there are more pages - if (!result.next_offset_date && - !result.next_offset_message_id && - !result.next_offset_message_thread_id) { - break; - } - offsetDate = result.next_offset_date ?? 0; - offsetMessageId = result.next_offset_message_id ?? 0; - offsetMessageThreadId = result.next_offset_message_thread_id ?? 0; - // Stuck detection: if offsets didn't advance, break - if (offsetDate === prevOffsetDate && - offsetMessageId === prevOffsetMessageId && - offsetMessageThreadId === prevOffsetMessageThreadId) { - log.warn({ chatId: chatId.toString(), topicCount: topics.length }, "Topic pagination stuck (offsets not advancing), breaking"); - break; - } - await sleep(config.apiDelayMs); - } - log.info({ chatId: chatId.toString(), topicCount: topics.length }, "Enumerated forum topics"); - return topics; -} -/** - * Fetch messages from a specific forum topic (thread), stopping once - * we've scanned past the last-processed boundary (with one page of lookback). - * Uses searchChatMessages with message_thread_id to scan within a topic. - * - * Returns messages in chronological order (oldest first). - * - * When `lastProcessedMessageId` is null (first run), scans everything. - * The worker applies a post-grouping filter to skip fully-processed sets, - * and keeps `packageExistsBySourceMessage` as a safety net. - * - * Safety features: - * - Max page limit to prevent infinite loops - * - Stuck detection: breaks if from_message_id stops advancing - * - Timeout on each TDLib API call - */ -export async function getTopicMessages(client, chatId, topicId, lastProcessedMessageId, limit = 100, onProgress) { - const archives = []; - const photos = []; - const boundary = lastProcessedMessageId ? Number(lastProcessedMessageId) : null; - let currentFromId = 0; - let totalScanned = 0; - let pageCount = 0; - // eslint-disable-next-line no-constant-condition - while (true) { - if (pageCount >= MAX_SCAN_PAGES) { - log.warn({ chatId: chatId.toString(), topicId: topicId.toString(), pageCount, totalScanned }, "Hit max page limit for topic scan, stopping"); - break; - } - pageCount++; - const previousFromId = currentFromId; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const result = await invokeWithTimeout(client, { - _: "searchChatMessages", - chat_id: Number(chatId), - query: "", - message_thread_id: Number(topicId), - from_message_id: currentFromId, - offset: 0, - limit: Math.min(limit, 100), - filter: null, - sender_id: null, - saved_messages_topic_id: 0, - }); - if (!result.messages || result.messages.length === 0) - break; - totalScanned += result.messages.length; - for (const msg of result.messages) { - // Check for archive documents - const doc = msg.content?.document; - if (doc?.file_name && doc.document && isArchiveAttachment(doc.file_name)) { - archives.push({ - id: BigInt(msg.id), - fileName: doc.file_name, - fileId: String(doc.document.id), - fileSize: BigInt(doc.document.size), - date: new Date(msg.date * 1000), - }); - continue; - } - // Check for photo messages (potential previews) - const photo = msg.content?.photo; - const caption = msg.content?.caption?.text ?? ""; - if (photo?.sizes && photo.sizes.length > 0) { - const smallest = photo.sizes[0]; - photos.push({ - id: BigInt(msg.id), - date: new Date(msg.date * 1000), - caption, - fileId: String(smallest.photo.id), - fileSize: smallest.photo.size || smallest.photo.expected_size, - }); - } - } - // Report scanning progress after each page - onProgress?.(totalScanned); - currentFromId = result.messages[result.messages.length - 1].id; - // Stuck detection: if from_message_id didn't advance, break to prevent infinite loop - if (currentFromId === previousFromId) { - log.warn({ chatId: chatId.toString(), topicId: topicId.toString(), currentFromId, totalScanned }, "Topic pagination stuck (from_message_id not advancing), breaking"); - break; - } - // Stop scanning once we've gone past the boundary (this page is the lookback) - if (boundary && currentFromId < boundary) - break; - if (result.messages.length < Math.min(limit, 100)) - break; - await sleep(config.apiDelayMs); - } - log.info({ chatId: chatId.toString(), topicId: topicId.toString(), archives: archives.length, photos: photos.length, totalScanned, pages: pageCount }, "Topic scan complete"); - // Reverse to chronological order (oldest first) so worker processes old→new - return { - archives: archives.reverse(), - photos: photos.reverse(), - totalScanned, - }; -} -function sleep(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} -//# sourceMappingURL=topics.js.map \ No newline at end of file diff --git a/worker/dist/tdlib/topics.js.map b/worker/dist/tdlib/topics.js.map deleted file mode 100644 index 940c3cf..0000000 --- a/worker/dist/tdlib/topics.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"topics.js","sourceRoot":"","sources":["../../src/tdlib/topics.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAI3D,OAAO,EAAE,iBAAiB,EAAE,cAAc,EAAqB,MAAM,eAAe,CAAC;AAErF,MAAM,GAAG,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC;AAOlC;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,MAAc,EACd,MAAc;IAEd,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAMjC,MAAM,EAAE;YACT,CAAC,EAAE,SAAS;YACZ,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC;SACxB,CAAC,CAAC;QAEH,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,oBAAoB,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YAChE,OAAO,IAAI,CAAC;QACd,CAAC;QAED,wDAAwD;QACxD,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,oBAAoB,IAAI,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACrE,MAAM,EAAE,GAAG,MAAM,iBAAiB,CAAyB,MAAM,EAAE;gBACjE,CAAC,EAAE,eAAe;gBAClB,aAAa,EAAE,IAAI,CAAC,IAAI,CAAC,aAAa;aACvC,CAAC,CAAC;YACH,OAAO,EAAE,CAAC,QAAQ,KAAK,IAAI,CAAC;QAC9B,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE,EAAE,EAAE,kCAAkC,CAAC,CAAC;QACjF,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,MAAc,EACd,MAAc;IAEd,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,IAAI,qBAAqB,GAAG,CAAC,CAAC;IAC9B,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,iDAAiD;IACjD,OAAO,IAAI,EAAE,CAAC;QACZ,IAAI,SAAS,IAAI,cAAc,EAAE,CAAC;YAChC,GAAG,CAAC,IAAI,CACN,EAAE,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,EACnE,oDAAoD,CACrD,CAAC;YACF,MAAM;QACR,CAAC;QACD,SAAS,EAAE,CAAC;QAEZ,MAAM,cAAc,GAAG,UAAU,CAAC;QAClC,MAAM,mBAAmB,GAAG,eAAe,CAAC;QAC5C,MAAM,yBAAyB,GAAG,qBAAqB,CAAC;QAExD,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAWnC,MAAM,EAAE;YACT,CAAC,EAAE,gBAAgB;YACnB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC;YACvB,KAAK,EAAE,EAAE;YACT,WAAW,EAAE,UAAU;YACvB,iBAAiB,EAAE,eAAe;YAClC,wBAAwB,EAAE,qBAAqB;YAC/C,KAAK,EAAE,GAAG;SACX,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM;QAExD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YAC9B,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,iBAAiB;gBAAE,SAAS;YACzC,uDAAuD;YACvD,IAAI,CAAC,CAAC,IAAI,CAAC,UAAU;gBAAE,SAAS;YAEhC,MAAM,CAAC,IAAI,CAAC;gBACV,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC;gBACzC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,SAAS;aAC/B,CAAC,CAAC;QACL,CAAC;QAED,gCAAgC;QAChC,IACE,CAAC,MAAM,CAAC,gBAAgB;YACxB,CAAC,MAAM,CAAC,sBAAsB;YAC9B,CAAC,MAAM,CAAC,6BAA6B,EACrC,CAAC;YACD,MAAM;QACR,CAAC;QAED,UAAU,GAAG,MAAM,CAAC,gBAAgB,IAAI,CAAC,CAAC;QAC1C,eAAe,GAAG,MAAM,CAAC,sBAAsB,IAAI,CAAC,CAAC;QACrD,qBAAqB,GAAG,MAAM,CAAC,6BAA6B,IAAI,CAAC,CAAC;QAElE,oDAAoD;QACpD,IACE,UAAU,KAAK,cAAc;YAC7B,eAAe,KAAK,mBAAmB;YACvC,qBAAqB,KAAK,yBAAyB,EACnD,CAAC;YACD,GAAG,CAAC,IAAI,CACN,EAAE,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE,EAAE,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,EACxD,0DAA0D,CAC3D,CAAC;YACF,MAAM;QACR,CAAC;QAED,MAAM,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IACjC,CAAC;IAED,GAAG,CAAC,IAAI,CACN,EAAE,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE,EAAE,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,EACxD,yBAAyB,CAC1B,CAAC;IAEF,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,MAAc,EACd,MAAc,EACd,OAAe,EACf,sBAAsC,EACtC,KAAK,GAAG,GAAG,EACX,UAAiC;IAEjC,MAAM,QAAQ,GAAsB,EAAE,CAAC;IACvC,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,MAAM,QAAQ,GAAG,sBAAsB,CAAC,CAAC,CAAC,MAAM,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAEhF,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,iDAAiD;IACjD,OAAO,IAAI,EAAE,CAAC;QACZ,IAAI,SAAS,IAAI,cAAc,EAAE,CAAC;YAChC,GAAG,CAAC,IAAI,CACN,EAAE,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,YAAY,EAAE,EACnF,6CAA6C,CAC9C,CAAC;YACF,MAAM;QACR,CAAC;QACD,SAAS,EAAE,CAAC;QAEZ,MAAM,cAAc,GAAG,aAAa,CAAC;QAErC,8DAA8D;QAC9D,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAwBnC,MAAM,EAAE;YACT,CAAC,EAAE,oBAAoB;YACvB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC;YACvB,KAAK,EAAE,EAAE;YACT,iBAAiB,EAAE,MAAM,CAAC,OAAO,CAAC;YAClC,eAAe,EAAE,aAAa;YAC9B,MAAM,EAAE,CAAC;YACT,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC;YAC3B,MAAM,EAAE,IAAI;YACZ,SAAS,EAAE,IAAI;YACf,uBAAuB,EAAE,CAAC;SAC3B,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM;QAE5D,YAAY,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;QAEvC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YAClC,8BAA8B;YAC9B,MAAM,GAAG,GAAG,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC;YAClC,IAAI,GAAG,EAAE,SAAS,IAAI,GAAG,CAAC,QAAQ,IAAI,mBAAmB,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;gBACzE,QAAQ,CAAC,IAAI,CAAC;oBACZ,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;oBAClB,QAAQ,EAAE,GAAG,CAAC,SAAS;oBACvB,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAC/B,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC;oBACnC,IAAI,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC;iBAChC,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;YAED,gDAAgD;YAChD,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC;YACjC,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC;YACjD,IAAI,KAAK,EAAE,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC3C,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAChC,MAAM,CAAC,IAAI,CAAC;oBACV,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;oBAClB,IAAI,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC;oBAC/B,OAAO;oBACP,MAAM,EAAE,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;oBACjC,QAAQ,EAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI,QAAQ,CAAC,KAAK,CAAC,aAAa;iBAC9D,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,2CAA2C;QAC3C,UAAU,EAAE,CAAC,YAAY,CAAC,CAAC;QAE3B,aAAa,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QAE/D,qFAAqF;QACrF,IAAI,aAAa,KAAK,cAAc,EAAE,CAAC;YACrC,GAAG,CAAC,IAAI,CACN,EAAE,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,QAAQ,EAAE,EAAE,aAAa,EAAE,YAAY,EAAE,EACvF,kEAAkE,CACnE,CAAC;YACF,MAAM;QACR,CAAC;QAED,8EAA8E;QAC9E,IAAI,QAAQ,IAAI,aAAa,GAAG,QAAQ;YAAE,MAAM;QAEhD,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC;YAAE,MAAM;QAEzD,MAAM,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IACjC,CAAC;IAED,GAAG,CAAC,IAAI,CACN,EAAE,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,EAAE,KAAK,EAAE,SAAS,EAAE,EAC5I,qBAAqB,CACtB,CAAC;IAEF,4EAA4E;IAC5E,OAAO;QACL,QAAQ,EAAE,QAAQ,CAAC,OAAO,EAAE;QAC5B,MAAM,EAAE,MAAM,CAAC,OAAO,EAAE;QACxB,YAAY;KACb,CAAC;AACJ,CAAC;AAED,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC"} \ No newline at end of file diff --git a/worker/dist/upload/channel.d.ts b/worker/dist/upload/channel.d.ts deleted file mode 100644 index c2db2a9..0000000 --- a/worker/dist/upload/channel.d.ts +++ /dev/null @@ -1,16 +0,0 @@ -import type { Client } from "tdl"; -export interface UploadResult { - messageId: bigint; -} -/** - * Upload one or more files to a destination Telegram channel. - * For multipart archives, each file is sent as a separate message. - * Returns the **final** (server-assigned) message ID of the first uploaded message. - * - * IMPORTANT: `sendMessage` returns a *temporary* message immediately. - * The actual file upload happens asynchronously in TDLib. We listen for - * `updateMessageSendSucceeded` to get the real server-side message ID and - * to make sure the upload is fully committed before we clean up temp files - * or close the TDLib client (which would cancel pending uploads). - */ -export declare function uploadToChannel(client: Client, chatId: bigint, filePaths: string[], caption?: string): Promise; diff --git a/worker/dist/upload/channel.js b/worker/dist/upload/channel.js deleted file mode 100644 index 5a4a045..0000000 --- a/worker/dist/upload/channel.js +++ /dev/null @@ -1,137 +0,0 @@ -import path from "path"; -import { stat } from "fs/promises"; -import { config } from "../util/config.js"; -import { childLogger } from "../util/logger.js"; -const log = childLogger("upload"); -/** - * Upload one or more files to a destination Telegram channel. - * For multipart archives, each file is sent as a separate message. - * Returns the **final** (server-assigned) message ID of the first uploaded message. - * - * IMPORTANT: `sendMessage` returns a *temporary* message immediately. - * The actual file upload happens asynchronously in TDLib. We listen for - * `updateMessageSendSucceeded` to get the real server-side message ID and - * to make sure the upload is fully committed before we clean up temp files - * or close the TDLib client (which would cancel pending uploads). - */ -export async function uploadToChannel(client, chatId, filePaths, caption) { - let firstMessageId = null; - for (let i = 0; i < filePaths.length; i++) { - const filePath = filePaths[i]; - const fileCaption = i === 0 && caption ? caption : undefined; - const fileName = path.basename(filePath); - let fileSizeMB = 0; - try { - const s = await stat(filePath); - fileSizeMB = Math.round(s.size / (1024 * 1024)); - } - catch { - // Non-critical - } - log.info({ chatId: Number(chatId), fileName, sizeMB: fileSizeMB, part: i + 1, total: filePaths.length }, "Uploading file to channel"); - const serverMsgId = await sendAndWaitForUpload(client, chatId, filePath, fileCaption, fileName, fileSizeMB); - if (i === 0) { - firstMessageId = serverMsgId; - } - // Rate limit delay between uploads - if (i < filePaths.length - 1) { - await sleep(config.apiDelayMs); - } - } - if (firstMessageId === null) { - throw new Error("Upload failed: no messages sent"); - } - log.info({ chatId: Number(chatId), messageId: Number(firstMessageId), files: filePaths.length }, "All uploads confirmed by Telegram"); - return { messageId: firstMessageId }; -} -/** - * Send a single file message and wait for Telegram to confirm the upload. - * Returns the final server-assigned message ID. - */ -async function sendAndWaitForUpload(client, chatId, filePath, caption, fileName, fileSizeMB) { - // Send the message — this returns a temporary message immediately - const tempMsg = (await client.invoke({ - _: "sendMessage", - chat_id: Number(chatId), - input_message_content: { - _: "inputMessageDocument", - document: { - _: "inputFileLocal", - path: filePath, - }, - caption: caption - ? { - _: "formattedText", - text: caption, - } - : undefined, - }, - })); - const tempMsgId = tempMsg.id; - log.debug({ fileName, tempMsgId }, "Message queued, waiting for upload confirmation"); - // Wait for the actual upload to complete - return new Promise((resolve, reject) => { - let settled = false; - let lastLoggedPercent = 0; - // Timeout: 10 minutes per GB, minimum 10 minutes - const timeoutMs = Math.max(10 * 60_000, (fileSizeMB / 1024) * 10 * 60_000); - const timer = setTimeout(() => { - if (!settled) { - settled = true; - cleanup(); - reject(new Error(`Upload timed out after ${Math.round(timeoutMs / 60_000)}min for ${fileName}`)); - } - }, timeoutMs); - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const handleUpdate = (update) => { - // Track upload progress via updateFile events - if (update?._ === "updateFile") { - const file = update.file; - if (file?.remote?.is_uploading_active && file.expected_size > 0) { - const uploaded = file.remote.uploaded_size ?? 0; - const total = file.expected_size; - const percent = Math.round((uploaded / total) * 100); - if (percent >= lastLoggedPercent + 20) { - lastLoggedPercent = percent - (percent % 20); - log.info({ fileName, uploaded, total, percent: `${percent}%` }, "Upload progress"); - } - } - } - // The money event: upload succeeded, we get the final server message ID - if (update?._ === "updateMessageSendSucceeded") { - const msg = update.message; - const oldMsgId = update.old_message_id; - if (oldMsgId === tempMsgId) { - if (!settled) { - settled = true; - cleanup(); - const finalId = BigInt(msg.id); - log.info({ fileName, tempMsgId, finalMsgId: Number(finalId) }, "Upload confirmed by Telegram"); - resolve(finalId); - } - } - } - // Upload failed - if (update?._ === "updateMessageSendFailed") { - const oldMsgId = update.old_message_id; - if (oldMsgId === tempMsgId) { - if (!settled) { - settled = true; - cleanup(); - const errorMsg = update.error?.message ?? "Unknown upload error"; - reject(new Error(`Upload failed for ${fileName}: ${errorMsg}`)); - } - } - } - }; - const cleanup = () => { - clearTimeout(timer); - client.off("update", handleUpdate); - }; - client.on("update", handleUpdate); - }); -} -function sleep(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} -//# sourceMappingURL=channel.js.map \ No newline at end of file diff --git a/worker/dist/upload/channel.js.map b/worker/dist/upload/channel.js.map deleted file mode 100644 index d2692bd..0000000 --- a/worker/dist/upload/channel.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"channel.js","sourceRoot":"","sources":["../../src/upload/channel.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AAEnC,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,MAAM,GAAG,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC;AAMlC;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,MAAc,EACd,MAAc,EACd,SAAmB,EACnB,OAAgB;IAEhB,IAAI,cAAc,GAAkB,IAAI,CAAC;IAEzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;QAC9B,MAAM,WAAW,GACf,CAAC,KAAK,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;QAE3C,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACzC,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,CAAC;YACH,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC/B,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC;QAClD,CAAC;QAAC,MAAM,CAAC;YACP,eAAe;QACjB,CAAC;QAED,GAAG,CAAC,IAAI,CACN,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,SAAS,CAAC,MAAM,EAAE,EAC9F,2BAA2B,CAC5B,CAAC;QAEF,MAAM,WAAW,GAAG,MAAM,oBAAoB,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;QAE5G,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YACZ,cAAc,GAAG,WAAW,CAAC;QAC/B,CAAC;QAED,mCAAmC;QACnC,IAAI,CAAC,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7B,MAAM,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;IAED,IAAI,cAAc,KAAK,IAAI,EAAE,CAAC;QAC5B,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;IACrD,CAAC;IAED,GAAG,CAAC,IAAI,CACN,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,cAAc,CAAC,EAAE,KAAK,EAAE,SAAS,CAAC,MAAM,EAAE,EACtF,mCAAmC,CACpC,CAAC;IAEF,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,CAAC;AACvC,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,oBAAoB,CACjC,MAAc,EACd,MAAc,EACd,QAAgB,EAChB,OAA2B,EAC3B,QAAgB,EAChB,UAAkB;IAElB,kEAAkE;IAClE,MAAM,OAAO,GAAG,CAAC,MAAM,MAAM,CAAC,MAAM,CAAC;QACnC,CAAC,EAAE,aAAa;QAChB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC;QACvB,qBAAqB,EAAE;YACrB,CAAC,EAAE,sBAAsB;YACzB,QAAQ,EAAE;gBACR,CAAC,EAAE,gBAAgB;gBACnB,IAAI,EAAE,QAAQ;aACf;YACD,OAAO,EAAE,OAAO;gBACd,CAAC,CAAC;oBACE,CAAC,EAAE,eAAe;oBAClB,IAAI,EAAE,OAAO;iBACd;gBACH,CAAC,CAAC,SAAS;SACd;KACF,CAAC,CAAmB,CAAC;IAEtB,MAAM,SAAS,GAAG,OAAO,CAAC,EAAE,CAAC;IAE7B,GAAG,CAAC,KAAK,CACP,EAAE,QAAQ,EAAE,SAAS,EAAE,EACvB,iDAAiD,CAClD,CAAC;IAEF,yCAAyC;IACzC,OAAO,IAAI,OAAO,CAAS,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC7C,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,IAAI,iBAAiB,GAAG,CAAC,CAAC;QAE1B,iDAAiD;QACjD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CACxB,EAAE,GAAG,MAAM,EACX,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAClC,CAAC;QAEF,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,OAAO,GAAG,IAAI,CAAC;gBACf,OAAO,EAAE,CAAC;gBACV,MAAM,CACJ,IAAI,KAAK,CACP,0BAA0B,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,MAAM,CAAC,WAAW,QAAQ,EAAE,CAC9E,CACF,CAAC;YACJ,CAAC;QACH,CAAC,EAAE,SAAS,CAAC,CAAC;QAEd,8DAA8D;QAC9D,MAAM,YAAY,GAAG,CAAC,MAAW,EAAE,EAAE;YACnC,8CAA8C;YAC9C,IAAI,MAAM,EAAE,CAAC,KAAK,YAAY,EAAE,CAAC;gBAC/B,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;gBACzB,IAAI,IAAI,EAAE,MAAM,EAAE,mBAAmB,IAAI,IAAI,CAAC,aAAa,GAAG,CAAC,EAAE,CAAC;oBAChE,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa,IAAI,CAAC,CAAC;oBAChD,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC;oBACjC,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC;oBACrD,IAAI,OAAO,IAAI,iBAAiB,GAAG,EAAE,EAAE,CAAC;wBACtC,iBAAiB,GAAG,OAAO,GAAG,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC;wBAC7C,GAAG,CAAC,IAAI,CACN,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,OAAO,GAAG,EAAE,EACrD,iBAAiB,CAClB,CAAC;oBACJ,CAAC;gBACH,CAAC;YACH,CAAC;YAED,wEAAwE;YACxE,IAAI,MAAM,EAAE,CAAC,KAAK,4BAA4B,EAAE,CAAC;gBAC/C,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC;gBAC3B,MAAM,QAAQ,GAAG,MAAM,CAAC,cAAc,CAAC;gBACvC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;oBAC3B,IAAI,CAAC,OAAO,EAAE,CAAC;wBACb,OAAO,GAAG,IAAI,CAAC;wBACf,OAAO,EAAE,CAAC;wBACV,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;wBAC/B,GAAG,CAAC,IAAI,CACN,EAAE,QAAQ,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,CAAC,OAAO,CAAC,EAAE,EACpD,8BAA8B,CAC/B,CAAC;wBACF,OAAO,CAAC,OAAO,CAAC,CAAC;oBACnB,CAAC;gBACH,CAAC;YACH,CAAC;YAED,gBAAgB;YAChB,IAAI,MAAM,EAAE,CAAC,KAAK,yBAAyB,EAAE,CAAC;gBAC5C,MAAM,QAAQ,GAAG,MAAM,CAAC,cAAc,CAAC;gBACvC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;oBAC3B,IAAI,CAAC,OAAO,EAAE,CAAC;wBACb,OAAO,GAAG,IAAI,CAAC;wBACf,OAAO,EAAE,CAAC;wBACV,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,EAAE,OAAO,IAAI,sBAAsB,CAAC;wBACjE,MAAM,CAAC,IAAI,KAAK,CAAC,qBAAqB,QAAQ,KAAK,QAAQ,EAAE,CAAC,CAAC,CAAC;oBAClE,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC,CAAC;QAEF,MAAM,OAAO,GAAG,GAAG,EAAE;YACnB,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;QACrC,CAAC,CAAC;QAEF,MAAM,CAAC,EAAE,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IACpC,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC"} \ No newline at end of file diff --git a/worker/dist/util/config.d.ts b/worker/dist/util/config.d.ts deleted file mode 100644 index c83bdf9..0000000 --- a/worker/dist/util/config.d.ts +++ /dev/null @@ -1,18 +0,0 @@ -export declare const config: { - readonly databaseUrl: string; - readonly workerIntervalMinutes: number; - readonly tempDir: string; - readonly tdlibStateDir: string; - readonly maxZipSizeMB: number; - readonly logLevel: "debug" | "info" | "warn" | "error"; - readonly telegramApiId: number; - readonly telegramApiHash: string; - /** Maximum jitter added to scheduler interval (in minutes) */ - readonly jitterMinutes: 5; - /** Maximum time span for multipart archive parts (in hours). 0 = no limit. */ - readonly multipartTimeoutHours: number; - /** Delay between Telegram API calls (in ms) to avoid rate limits */ - readonly apiDelayMs: 1000; - /** Max retries for rate-limited requests */ - readonly maxRetries: 5; -}; diff --git a/worker/dist/util/config.js b/worker/dist/util/config.js deleted file mode 100644 index 2719058..0000000 --- a/worker/dist/util/config.js +++ /dev/null @@ -1,19 +0,0 @@ -export const config = { - databaseUrl: process.env.DATABASE_URL ?? "", - workerIntervalMinutes: parseInt(process.env.WORKER_INTERVAL_MINUTES ?? "60", 10), - tempDir: process.env.WORKER_TEMP_DIR ?? "/tmp/zips", - tdlibStateDir: process.env.TDLIB_STATE_DIR ?? "/data/tdlib", - maxZipSizeMB: parseInt(process.env.WORKER_MAX_ZIP_SIZE_MB ?? "4096", 10), - logLevel: (process.env.LOG_LEVEL ?? "info"), - telegramApiId: parseInt(process.env.TELEGRAM_API_ID ?? "0", 10), - telegramApiHash: process.env.TELEGRAM_API_HASH ?? "", - /** Maximum jitter added to scheduler interval (in minutes) */ - jitterMinutes: 5, - /** Maximum time span for multipart archive parts (in hours). 0 = no limit. */ - multipartTimeoutHours: parseInt(process.env.MULTIPART_TIMEOUT_HOURS ?? "0", 10), - /** Delay between Telegram API calls (in ms) to avoid rate limits */ - apiDelayMs: 1000, - /** Max retries for rate-limited requests */ - maxRetries: 5, -}; -//# sourceMappingURL=config.js.map \ No newline at end of file diff --git a/worker/dist/util/config.js.map b/worker/dist/util/config.js.map deleted file mode 100644 index d14cfed..0000000 --- a/worker/dist/util/config.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/util/config.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,MAAM,GAAG;IACpB,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,EAAE;IAC3C,qBAAqB,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB,IAAI,IAAI,EAAE,EAAE,CAAC;IAChF,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,WAAW;IACnD,aAAa,EAAE,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,aAAa;IAC3D,YAAY,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,sBAAsB,IAAI,MAAM,EAAE,EAAE,CAAC;IACxE,QAAQ,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,MAAM,CAAwC;IAClF,aAAa,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,GAAG,EAAE,EAAE,CAAC;IAC/D,eAAe,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,EAAE;IACpD,8DAA8D;IAC9D,aAAa,EAAE,CAAC;IAChB,8EAA8E;IAC9E,qBAAqB,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB,IAAI,GAAG,EAAE,EAAE,CAAC;IAC/E,oEAAoE;IACpE,UAAU,EAAE,IAAI;IAChB,4CAA4C;IAC5C,UAAU,EAAE,CAAC;CACL,CAAC"} \ No newline at end of file diff --git a/worker/dist/util/logger.d.ts b/worker/dist/util/logger.d.ts deleted file mode 100644 index 17f45d7..0000000 --- a/worker/dist/util/logger.d.ts +++ /dev/null @@ -1,3 +0,0 @@ -import pino from "pino"; -export declare const logger: pino.Logger; -export declare function childLogger(name: string, extra?: Record): pino.Logger; diff --git a/worker/dist/util/logger.js b/worker/dist/util/logger.js deleted file mode 100644 index c593d46..0000000 --- a/worker/dist/util/logger.js +++ /dev/null @@ -1,12 +0,0 @@ -import pino from "pino"; -import { config } from "./config.js"; -export const logger = pino({ - level: config.logLevel, - transport: config.logLevel === "debug" - ? { target: "pino/file", options: { destination: 1 } } - : undefined, -}); -export function childLogger(name, extra) { - return logger.child({ module: name, ...extra }); -} -//# sourceMappingURL=logger.js.map \ No newline at end of file diff --git a/worker/dist/util/logger.js.map b/worker/dist/util/logger.js.map deleted file mode 100644 index 8d0066f..0000000 --- a/worker/dist/util/logger.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"logger.js","sourceRoot":"","sources":["../../src/util/logger.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC,MAAM,CAAC,MAAM,MAAM,GAAG,IAAI,CAAC;IACzB,KAAK,EAAE,MAAM,CAAC,QAAQ;IACtB,SAAS,EACP,MAAM,CAAC,QAAQ,KAAK,OAAO;QACzB,CAAC,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,EAAE;QACtD,CAAC,CAAC,SAAS;CAChB,CAAC,CAAC;AAEH,MAAM,UAAU,WAAW,CAAC,IAAY,EAAE,KAA+B;IACvE,OAAO,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,KAAK,EAAE,CAAC,CAAC;AAClD,CAAC"} \ No newline at end of file diff --git a/worker/dist/util/mutex.d.ts b/worker/dist/util/mutex.d.ts deleted file mode 100644 index 24fabf6..0000000 --- a/worker/dist/util/mutex.d.ts +++ /dev/null @@ -1,8 +0,0 @@ -/** - * Ensures only one TDLib client runs at a time across the entire worker process. - * Both the scheduler (auth, ingestion) and the fetch listener acquire this - * before creating any TDLib client. - * - * Includes a wait timeout to prevent indefinite blocking if the current holder hangs. - */ -export declare function withTdlibMutex(label: string, fn: () => Promise): Promise; diff --git a/worker/dist/util/mutex.js b/worker/dist/util/mutex.js deleted file mode 100644 index e72c5dd..0000000 --- a/worker/dist/util/mutex.js +++ /dev/null @@ -1,61 +0,0 @@ -import { childLogger } from "./logger.js"; -const log = childLogger("mutex"); -let locked = false; -let holder = ""; -const queue = []; -/** - * Maximum time to wait for the TDLib mutex (ms). - * If the mutex is not available within this time, the operation is rejected. - * Default: 30 minutes (long enough for large downloads, short enough to detect hangs). - */ -const MUTEX_WAIT_TIMEOUT_MS = 30 * 60 * 1000; -/** - * Ensures only one TDLib client runs at a time across the entire worker process. - * Both the scheduler (auth, ingestion) and the fetch listener acquire this - * before creating any TDLib client. - * - * Includes a wait timeout to prevent indefinite blocking if the current holder hangs. - */ -export async function withTdlibMutex(label, fn) { - if (locked) { - log.info({ waiting: label, holder }, "Waiting for TDLib mutex"); - await new Promise((resolve, reject) => { - const entry = { resolve, reject, label }; - queue.push(entry); - // Timeout: reject if we've been waiting too long - const timer = setTimeout(() => { - const idx = queue.indexOf(entry); - if (idx !== -1) { - queue.splice(idx, 1); - reject(new Error(`TDLib mutex wait timeout after ${MUTEX_WAIT_TIMEOUT_MS / 60_000}min ` + - `(waiting: ${label}, holder: ${holder})`)); - } - }, MUTEX_WAIT_TIMEOUT_MS); - // Wrap resolve to clear the timer - const origResolve = entry.resolve; - entry.resolve = () => { - clearTimeout(timer); - origResolve(); - }; - }); - } - locked = true; - holder = label; - log.debug({ label }, "TDLib mutex acquired"); - try { - return await fn(); - } - finally { - locked = false; - holder = ""; - const next = queue.shift(); - if (next) { - log.debug({ next: next.label }, "TDLib mutex releasing to next waiter"); - next.resolve(); - } - else { - log.debug({ label }, "TDLib mutex released"); - } - } -} -//# sourceMappingURL=mutex.js.map \ No newline at end of file diff --git a/worker/dist/util/mutex.js.map b/worker/dist/util/mutex.js.map deleted file mode 100644 index db05e69..0000000 --- a/worker/dist/util/mutex.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"mutex.js","sourceRoot":"","sources":["../../src/util/mutex.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAE1C,MAAM,GAAG,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;AAEjC,IAAI,MAAM,GAAG,KAAK,CAAC;AACnB,IAAI,MAAM,GAAG,EAAE,CAAC;AAChB,MAAM,KAAK,GAAgF,EAAE,CAAC;AAE9F;;;;GAIG;AACH,MAAM,qBAAqB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAE7C;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,KAAa,EACb,EAAoB;IAEpB,IAAI,MAAM,EAAE,CAAC;QACX,GAAG,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,yBAAyB,CAAC,CAAC;QAChE,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC1C,MAAM,KAAK,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;YACzC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAElB,iDAAiD;YACjD,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;gBAC5B,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;gBACjC,IAAI,GAAG,KAAK,CAAC,CAAC,EAAE,CAAC;oBACf,KAAK,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;oBACrB,MAAM,CAAC,IAAI,KAAK,CACd,kCAAkC,qBAAqB,GAAG,MAAM,MAAM;wBACtE,aAAa,KAAK,aAAa,MAAM,GAAG,CACzC,CAAC,CAAC;gBACL,CAAC;YACH,CAAC,EAAE,qBAAqB,CAAC,CAAC;YAE1B,kCAAkC;YAClC,MAAM,WAAW,GAAG,KAAK,CAAC,OAAO,CAAC;YAClC,KAAK,CAAC,OAAO,GAAG,GAAG,EAAE;gBACnB,YAAY,CAAC,KAAK,CAAC,CAAC;gBACpB,WAAW,EAAE,CAAC;YAChB,CAAC,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IAED,MAAM,GAAG,IAAI,CAAC;IACd,MAAM,GAAG,KAAK,CAAC;IACf,GAAG,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,EAAE,sBAAsB,CAAC,CAAC;IAE7C,IAAI,CAAC;QACH,OAAO,MAAM,EAAE,EAAE,CAAC;IACpB,CAAC;YAAS,CAAC;QACT,MAAM,GAAG,KAAK,CAAC;QACf,MAAM,GAAG,EAAE,CAAC;QACZ,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;QAC3B,IAAI,IAAI,EAAE,CAAC;YACT,GAAG,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,KAAK,EAAE,EAAE,sCAAsC,CAAC,CAAC;YACxE,IAAI,CAAC,OAAO,EAAE,CAAC;QACjB,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,EAAE,sBAAsB,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;AACH,CAAC"} \ No newline at end of file diff --git a/worker/dist/worker.d.ts b/worker/dist/worker.d.ts deleted file mode 100644 index 2185fa9..0000000 --- a/worker/dist/worker.d.ts +++ /dev/null @@ -1,28 +0,0 @@ -import type { TelegramAccount } from "@prisma/client"; -/** - * Authenticate a PENDING account by creating a TDLib client. - * TDLib will send an SMS code to the phone number, and the client.login() - * callbacks set the authState to AWAITING_CODE. Once the admin enters the - * code via the UI, pollForAuthCode picks it up and completes the login. - * - * After successful auth: - * 1. Fetches channels from Telegram and writes as a ChannelFetchRequest - * (so the admin can select sources in the UI) - * 2. Auto-joins the destination group if an invite link is configured - */ -export declare function authenticateAccount(account: TelegramAccount): Promise; -/** - * Process a ChannelFetchRequest: fetch channels from Telegram, - * enrich with DB state, and write the result JSON. - * Called by the fetch listener (pg_notify) and by authenticateAccount. - */ -export declare function processFetchRequest(requestId: string): Promise; -/** - * Run a full ingestion cycle for a single Telegram account. - * Every step writes live activity to the DB so the admin UI can display it. - */ -export declare function runWorkerForAccount(account: TelegramAccount): Promise; -/** - * Clean up any leftover temp files/directories from previous runs. - */ -export declare function cleanupTempDir(): Promise; diff --git a/worker/dist/worker.js b/worker/dist/worker.js deleted file mode 100644 index c6f336e..0000000 --- a/worker/dist/worker.js +++ /dev/null @@ -1,745 +0,0 @@ -import path from "path"; -import { unlink, readdir, mkdir, rm } from "fs/promises"; -import { config } from "./util/config.js"; -import { childLogger } from "./util/logger.js"; -import { tryAcquireLock, releaseLock } from "./db/locks.js"; -import { getSourceChannelMappings, getGlobalDestinationChannel, packageExistsByHash, packageExistsBySourceMessage, createPackageWithFiles, createIngestionRun, completeIngestionRun, failIngestionRun, updateLastProcessedMessage, updateRunActivity, setChannelForum, getTopicProgress, upsertTopicProgress, upsertChannel, ensureAccountChannelLink, getGlobalSetting, getChannelFetchRequest, updateFetchRequestStatus, getAccountLinkedChannelIds, getExistingChannelsByTelegramId, deleteOrphanedPackageByHash, } from "./db/queries.js"; -import { createTdlibClient, closeTdlibClient } from "./tdlib/client.js"; -import { getAccountChats, joinChatByInviteLink } from "./tdlib/chats.js"; -import { getChannelMessages, downloadFile, downloadPhotoThumbnail } from "./tdlib/download.js"; -import { isChatForum, getForumTopicList, getTopicMessages } from "./tdlib/topics.js"; -import { matchPreviewToArchive } from "./preview/match.js"; -import { groupArchiveSets } from "./archive/multipart.js"; -import { extractCreatorFromFileName } from "./archive/creator.js"; -import { hashParts } from "./archive/hash.js"; -import { readZipCentralDirectory } from "./archive/zip-reader.js"; -import { readRarContents } from "./archive/rar-reader.js"; -import { byteLevelSplit, concatenateFiles } from "./archive/split.js"; -import { uploadToChannel } from "./upload/channel.js"; -const log = childLogger("worker"); -/** - * Authenticate a PENDING account by creating a TDLib client. - * TDLib will send an SMS code to the phone number, and the client.login() - * callbacks set the authState to AWAITING_CODE. Once the admin enters the - * code via the UI, pollForAuthCode picks it up and completes the login. - * - * After successful auth: - * 1. Fetches channels from Telegram and writes as a ChannelFetchRequest - * (so the admin can select sources in the UI) - * 2. Auto-joins the destination group if an invite link is configured - */ -export async function authenticateAccount(account) { - const aLog = childLogger("auth", { accountId: account.id, phone: account.phone }); - aLog.info("Starting authentication flow"); - let client; - try { - client = await createTdlibClient({ - id: account.id, - phone: account.phone, - }); - aLog.info("Authentication successful"); - // Auto-fetch channels and create a fetch request result - aLog.info("Fetching channels from Telegram..."); - await createAutoFetchRequest(client, account.id, aLog); - // Auto-join the destination group if an invite link exists - const inviteLink = await getGlobalSetting("destination_invite_link"); - if (inviteLink) { - aLog.info("Attempting to join destination group via invite link..."); - try { - await joinChatByInviteLink(client, inviteLink); - // Link this account as WRITER to the destination channel - const destChannel = await getGlobalDestinationChannel(); - if (destChannel) { - await ensureAccountChannelLink(account.id, destChannel.id, "WRITER"); - aLog.info({ destChannel: destChannel.title }, "Joined destination group and linked as WRITER"); - } - } - catch (err) { - // May already be a member — that's fine - aLog.warn({ err }, "Could not join destination group (may already be a member)"); - // Still try to link as WRITER - const destChannel = await getGlobalDestinationChannel(); - if (destChannel) { - await ensureAccountChannelLink(account.id, destChannel.id, "WRITER"); - } - } - } - } - catch (err) { - aLog.error({ err }, "Authentication failed"); - } - finally { - if (client) { - await closeTdlibClient(client); - } - } -} -/** - * Process a ChannelFetchRequest: fetch channels from Telegram, - * enrich with DB state, and write the result JSON. - * Called by the fetch listener (pg_notify) and by authenticateAccount. - */ -export async function processFetchRequest(requestId) { - const aLog = childLogger("fetch-request", { requestId }); - const request = await getChannelFetchRequest(requestId); - if (!request || request.status !== "PENDING") { - aLog.warn("Fetch request not found or not pending, skipping"); - return; - } - await updateFetchRequestStatus(requestId, "IN_PROGRESS"); - aLog.info({ accountId: request.accountId }, "Processing fetch request"); - const client = await createTdlibClient({ - id: request.account.id, - phone: request.account.phone, - }); - try { - const chats = await getAccountChats(client); - // Enrich with DB state - const linkedTelegramIds = await getAccountLinkedChannelIds(request.accountId); - const existingChannels = await getExistingChannelsByTelegramId(); - const enrichedChats = chats.map((chat) => { - const telegramIdStr = chat.chatId.toString(); - return { - chatId: telegramIdStr, - title: chat.title, - type: chat.type, - isForum: chat.isForum, - memberCount: chat.memberCount ?? null, - alreadyLinked: linkedTelegramIds.has(telegramIdStr), - existingChannelId: existingChannels.get(telegramIdStr) ?? null, - }; - }); - // Also upsert channel metadata while we have the data - for (const chat of chats) { - try { - await upsertChannel({ - telegramId: chat.chatId, - title: chat.title, - type: "SOURCE", - isForum: chat.isForum, - }); - } - catch { - // Non-critical — metadata sync can fail silently - } - } - await updateFetchRequestStatus(requestId, "COMPLETED", { - resultJson: JSON.stringify(enrichedChats), - }); - aLog.info({ total: chats.length, linked: [...linkedTelegramIds].length }, "Fetch request completed"); - } - catch (err) { - const message = err instanceof Error ? err.message : String(err); - aLog.error({ err }, "Fetch request failed"); - await updateFetchRequestStatus(requestId, "FAILED", { error: message }); - } - finally { - await closeTdlibClient(client); - } -} -/** - * Internal helper called after authentication to auto-create a fetch request - * with the channel list (so the UI can show the picker immediately). - */ -async function createAutoFetchRequest(client, accountId, aLog) { - const chats = await getAccountChats(client); - const linkedTelegramIds = await getAccountLinkedChannelIds(accountId); - const existingChannels = await getExistingChannelsByTelegramId(); - const enrichedChats = chats.map((chat) => { - const telegramIdStr = chat.chatId.toString(); - return { - chatId: telegramIdStr, - title: chat.title, - type: chat.type, - isForum: chat.isForum, - memberCount: chat.memberCount ?? null, - alreadyLinked: linkedTelegramIds.has(telegramIdStr), - existingChannelId: existingChannels.get(telegramIdStr) ?? null, - }; - }); - // Upsert channel metadata - for (const chat of chats) { - try { - await upsertChannel({ - telegramId: chat.chatId, - title: chat.title, - type: "SOURCE", - isForum: chat.isForum, - }); - } - catch { - // Non-critical - } - } - // Create the fetch request record with the result already filled in - const { db } = await import("./db/client.js"); - await db.channelFetchRequest.create({ - data: { - accountId, - status: "COMPLETED", - resultJson: JSON.stringify(enrichedChats), - }, - }); - aLog.info({ total: chats.length }, "Auto-fetch request created with channel list"); -} -/** - * Throttle DB writes for download progress to avoid hammering the DB. - * Only writes if at least 2 seconds have passed since the last write. - */ -function createThrottledActivityUpdater(runId, minIntervalMs = 2000) { - let lastWriteTime = 0; - let pendingUpdate = null; - let flushTimer = null; - const flush = async () => { - if (pendingUpdate) { - const update = pendingUpdate; - pendingUpdate = null; - lastWriteTime = Date.now(); - await updateRunActivity(runId, update).catch(() => { }); - } - }; - return { - update: (activity) => { - pendingUpdate = activity; - const elapsed = Date.now() - lastWriteTime; - if (elapsed >= minIntervalMs) { - if (flushTimer) - clearTimeout(flushTimer); - flush(); - } - else if (!flushTimer) { - flushTimer = setTimeout(() => { - flushTimer = null; - flush(); - }, minIntervalMs - elapsed); - } - }, - flush, - }; -} -/** - * Run a full ingestion cycle for a single Telegram account. - * Every step writes live activity to the DB so the admin UI can display it. - */ -export async function runWorkerForAccount(account) { - const accountLog = childLogger("worker", { accountId: account.id, phone: account.phone }); - // 1. Acquire advisory lock - const acquired = await tryAcquireLock(account.id); - if (!acquired) { - accountLog.info("Account already locked, skipping"); - return; - } - let runId; - try { - // 2. Create ingestion run - const run = await createIngestionRun(account.id); - runId = run.id; - const activeRunId = runId; - accountLog.info({ runId }, "Ingestion run started"); - const throttled = createThrottledActivityUpdater(activeRunId); - // 3. Initialize TDLib client - await updateRunActivity(activeRunId, { - currentActivity: "Connecting to Telegram", - currentStep: "connecting", - }); - const client = await createTdlibClient({ - id: account.id, - phone: account.phone, - }); - const counters = { - messagesScanned: 0, - zipsFound: 0, - zipsDuplicate: 0, - zipsIngested: 0, - }; - try { - // 4. Get assigned source channels and global destination - const channelMappings = await getSourceChannelMappings(account.id); - const destChannel = await getGlobalDestinationChannel(); - if (!destChannel) { - throw new Error("No global destination channel configured — set one in the admin UI"); - } - const totalChannels = channelMappings.length; - for (let chIdx = 0; chIdx < channelMappings.length; chIdx++) { - const mapping = channelMappings[chIdx]; - const channel = mapping.channel; - const channelLabel = totalChannels > 1 - ? `[${chIdx + 1}/${totalChannels}] ${channel.title}` - : channel.title; - try { - // ── Check if channel is a forum ── - const forum = await isChatForum(client, channel.telegramId); - if (forum !== channel.isForum) { - await setChannelForum(channel.id, forum); - accountLog.info({ channelId: channel.id, title: channel.title, isForum: forum }, "Updated channel forum status"); - } - const pipelineCtx = { - client, - runId: activeRunId, - channelTitle: channel.title, - channel, - destChannelTelegramId: destChannel.telegramId, - destChannelId: destChannel.id, - throttled, - counters, - topicCreator: null, - sourceTopicId: null, - accountLog, - }; - if (forum) { - // ── Forum channel: scan per-topic ── - await updateRunActivity(activeRunId, { - currentActivity: `Enumerating topics in "${channelLabel}"`, - currentStep: "scanning", - currentChannel: channelLabel, - currentFile: null, - currentFileNum: null, - totalFiles: null, - downloadedBytes: null, - totalBytes: null, - downloadPercent: null, - messagesScanned: counters.messagesScanned, - }); - const topics = await getForumTopicList(client, channel.telegramId); - const topicProgressList = await getTopicProgress(mapping.id); - accountLog.info({ channelId: channel.id, title: channel.title, topicCount: topics.length }, "Scanning forum channel by topic"); - for (let tIdx = 0; tIdx < topics.length; tIdx++) { - const topic = topics[tIdx]; - try { - const progress = topicProgressList.find((tp) => tp.topicId === topic.topicId); - const topicLabel = `${channel.title} › ${topic.name}`; - const topicProgress = topics.length > 1 - ? ` (topic ${tIdx + 1}/${topics.length})` - : ""; - await updateRunActivity(activeRunId, { - currentActivity: `Scanning "${topicLabel}"${topicProgress}`, - currentStep: "scanning", - currentChannel: channelLabel, - currentFile: null, - currentFileNum: null, - totalFiles: null, - downloadedBytes: null, - totalBytes: null, - downloadPercent: null, - messagesScanned: counters.messagesScanned, - }); - const scanResult = await getTopicMessages(client, channel.telegramId, topic.topicId, progress?.lastProcessedMessageId, 100, (scanned) => { - throttled.update({ - currentActivity: `Scanning "${topicLabel}"${topicProgress} — ${scanned} messages scanned`, - currentStep: "scanning", - currentChannel: channelLabel, - messagesScanned: counters.messagesScanned + scanned, - }); - }); - // Add scanned messages to global counter - counters.messagesScanned += scanResult.totalScanned; - if (scanResult.archives.length === 0) { - accountLog.debug({ channelId: channel.id, topic: topic.name }, "No new archives in topic"); - continue; - } - accountLog.info({ topic: topic.name, archives: scanResult.archives.length, photos: scanResult.photos.length }, "Found messages in topic"); - // Process archives with topic creator - pipelineCtx.topicCreator = topic.name; - pipelineCtx.sourceTopicId = topic.topicId; - pipelineCtx.channelTitle = `${channel.title} › ${topic.name}`; - const maxProcessedId = await processArchiveSets(pipelineCtx, scanResult, run.id, progress?.lastProcessedMessageId); - // Only advance progress to the highest successfully processed message - if (maxProcessedId) { - await upsertTopicProgress(mapping.id, topic.topicId, topic.name, maxProcessedId); - } - } - catch (topicErr) { - accountLog.warn({ err: topicErr, channelId: channel.id, topic: topic.name, topicId: topic.topicId.toString() }, "Failed to process topic, skipping"); - } - } - } - else { - // ── Non-forum channel: flat scan (existing behavior) ── - await updateRunActivity(activeRunId, { - currentActivity: `Scanning "${channelLabel}" for new archives`, - currentStep: "scanning", - currentChannel: channelLabel, - currentFile: null, - currentFileNum: null, - totalFiles: null, - downloadedBytes: null, - totalBytes: null, - downloadPercent: null, - messagesScanned: counters.messagesScanned, - }); - accountLog.info({ channelId: channel.id, title: channel.title }, "Processing source channel"); - const scanResult = await getChannelMessages(client, channel.telegramId, mapping.lastProcessedMessageId, 100, (scanned) => { - throttled.update({ - currentActivity: `Scanning "${channelLabel}" — ${scanned} messages scanned`, - currentStep: "scanning", - currentChannel: channelLabel, - messagesScanned: counters.messagesScanned + scanned, - }); - }); - // Add scanned messages to global counter - counters.messagesScanned += scanResult.totalScanned; - if (scanResult.archives.length === 0) { - accountLog.debug({ channelId: channel.id }, "No new archives"); - continue; - } - accountLog.info({ archives: scanResult.archives.length, photos: scanResult.photos.length }, "Found messages in channel"); - // For non-forum, creator comes from filename (set to null, resolved per-archive) - pipelineCtx.topicCreator = null; - pipelineCtx.sourceTopicId = null; - pipelineCtx.channelTitle = channel.title; - const maxProcessedId = await processArchiveSets(pipelineCtx, scanResult, run.id, mapping.lastProcessedMessageId); - // Only advance progress to the highest successfully processed message - if (maxProcessedId) { - await updateLastProcessedMessage(mapping.id, maxProcessedId); - } - } - } - catch (channelErr) { - accountLog.warn({ err: channelErr, channelId: channel.id, title: channel.title }, "Failed to process channel, skipping to next"); - } - } - // ── Done ── - await completeIngestionRun(activeRunId, counters); - accountLog.info({ counters }, "Ingestion run completed"); - } - finally { - await closeTdlibClient(client); - } - } - catch (err) { - const message = err instanceof Error ? err.message : String(err); - accountLog.error({ err }, "Ingestion run failed"); - if (runId) { - await failIngestionRun(runId, message).catch((e) => accountLog.error({ e }, "Failed to mark run as failed")); - } - } - finally { - await releaseLock(account.id); - } -} -/** - * Process a scan result through the archive pipeline: - * group → download → hash → dedup → metadata → split → upload → preview → index. - * - * Returns the highest message ID that was successfully processed (ingested or - * confirmed duplicate). The caller should only advance the progress boundary - * to this value — never to the max of all scanned messages. - */ -async function processArchiveSets(ctx, scanResult, ingestionRunId, lastProcessedMessageId) { - const { client, runId, channelTitle, channel, throttled, counters, accountLog } = ctx; - // Group into archive sets - let archiveSets = groupArchiveSets(scanResult.archives); - // Filter out sets where ALL parts are at or below the boundary (already processed) - if (lastProcessedMessageId) { - const totalBefore = archiveSets.length; - archiveSets = archiveSets.filter((set) => set.parts.some((p) => p.id > lastProcessedMessageId)); - const filtered = totalBefore - archiveSets.length; - if (filtered > 0) { - accountLog.info({ filtered, remaining: archiveSets.length }, "Filtered out already-processed archive sets"); - } - } - counters.zipsFound += archiveSets.length; - // Match preview photos to archive sets - const previewMatches = matchPreviewToArchive(scanResult.photos, archiveSets.map((s) => ({ - baseName: s.baseName, - firstMessageId: s.parts[0].id, - firstMessageDate: s.parts[0].date, - }))); - if (previewMatches.size > 0) { - accountLog.info({ matched: previewMatches.size, total: archiveSets.length }, "Matched preview photos to archives"); - } - await updateRunActivity(runId, { - currentActivity: `Found ${archiveSets.length} archive(s) in "${channelTitle}"`, - currentStep: "scanning", - currentChannel: channelTitle, - totalFiles: archiveSets.length, - zipsFound: counters.zipsFound, - messagesScanned: counters.messagesScanned, - }); - // Track the highest message ID that was successfully processed - let maxProcessedId = null; - for (let setIdx = 0; setIdx < archiveSets.length; setIdx++) { - try { - await processOneArchiveSet(ctx, archiveSets[setIdx], setIdx, archiveSets.length, previewMatches, ingestionRunId); - // Set completed (ingested or confirmed duplicate) — advance watermark - const setMaxId = archiveSets[setIdx].parts.reduce((max, p) => (p.id > max ? p.id : max), 0n); - if (setMaxId > (maxProcessedId ?? 0n)) { - maxProcessedId = setMaxId; - } - } - catch (setErr) { - // If a set fails, do NOT advance the watermark past it - accountLog.warn({ err: setErr, baseName: archiveSets[setIdx].baseName }, "Archive set failed, watermark will not advance past this set"); - } - } - return maxProcessedId; -} -/** - * Process a single archive set through the full pipeline. - */ -async function processOneArchiveSet(ctx, archiveSet, setIdx, totalSets, previewMatches, ingestionRunId) { - const { client, runId, channelTitle, channel, destChannelTelegramId, destChannelId, throttled, counters, topicCreator, sourceTopicId, accountLog, } = ctx; - const archiveName = archiveSet.parts[0].fileName; - // ── Early skip: check if this archive set was already ingested ── - // This avoids re-downloading large archives that were processed in a prior run. - const alreadyIngested = await packageExistsBySourceMessage(channel.id, archiveSet.parts[0].id); - if (alreadyIngested) { - counters.zipsDuplicate++; - accountLog.debug({ fileName: archiveName, sourceMessageId: Number(archiveSet.parts[0].id) }, "Archive already ingested (by source message), skipping"); - await updateRunActivity(runId, { - currentActivity: `Skipped ${archiveName} (already ingested)`, - currentStep: "deduplicating", - currentChannel: channelTitle, - currentFile: archiveName, - currentFileNum: setIdx + 1, - totalFiles: totalSets, - zipsDuplicate: counters.zipsDuplicate, - }); - return; - } - const tempPaths = []; - let splitPaths = []; - // Per-set subdirectory so uploaded files keep their original filenames - const setDir = path.join(config.tempDir, `${ingestionRunId}_${archiveSet.parts[0].id}`); - await mkdir(setDir, { recursive: true }); - try { - // ── Downloading ── - for (let partIdx = 0; partIdx < archiveSet.parts.length; partIdx++) { - const part = archiveSet.parts[partIdx]; - const tempPath = path.join(setDir, part.fileName); - const partLabel = archiveSet.parts.length > 1 - ? ` (part ${partIdx + 1}/${archiveSet.parts.length})` - : ""; - await updateRunActivity(runId, { - currentActivity: `Downloading ${part.fileName}${partLabel}`, - currentStep: "downloading", - currentChannel: channelTitle, - currentFile: part.fileName, - currentFileNum: setIdx + 1, - totalFiles: totalSets, - downloadedBytes: 0n, - totalBytes: part.fileSize, - downloadPercent: 0, - messagesScanned: counters.messagesScanned, - }); - accountLog.info({ - fileName: part.fileName, - fileSize: Number(part.fileSize), - part: partIdx + 1, - totalParts: archiveSet.parts.length, - }, "Downloading archive part"); - await downloadFile(client, part.fileId, tempPath, part.fileSize, part.fileName, (progress) => { - throttled.update({ - currentActivity: `Downloading ${part.fileName}${partLabel} — ${progress.percent}%`, - currentStep: "downloading", - currentChannel: channelTitle, - currentFile: part.fileName, - currentFileNum: setIdx + 1, - totalFiles: totalSets, - downloadedBytes: BigInt(progress.downloadedBytes), - totalBytes: BigInt(progress.totalBytes), - downloadPercent: progress.percent, - }); - }); - await throttled.flush(); - tempPaths.push(tempPath); - } - // ── Hashing ── - await updateRunActivity(runId, { - currentActivity: `Computing hash for ${archiveName}`, - currentStep: "hashing", - currentChannel: channelTitle, - currentFile: archiveName, - currentFileNum: setIdx + 1, - totalFiles: totalSets, - downloadedBytes: null, - totalBytes: null, - downloadPercent: null, - }); - const contentHash = await hashParts(tempPaths); - // ── Deduplicating ── - await updateRunActivity(runId, { - currentActivity: `Checking if ${archiveName} is a duplicate`, - currentStep: "deduplicating", - currentChannel: channelTitle, - currentFile: archiveName, - currentFileNum: setIdx + 1, - totalFiles: totalSets, - }); - const exists = await packageExistsByHash(contentHash); - if (exists) { - counters.zipsDuplicate++; - accountLog.debug({ contentHash }, "Duplicate archive, skipping"); - await updateRunActivity(runId, { - currentActivity: `Skipped ${archiveName} (duplicate)`, - currentStep: "deduplicating", - currentChannel: channelTitle, - currentFile: archiveName, - currentFileNum: setIdx + 1, - totalFiles: totalSets, - zipsDuplicate: counters.zipsDuplicate, - }); - return; - } - // ── Reading metadata ── - await updateRunActivity(runId, { - currentActivity: `Reading file list from ${archiveName}`, - currentStep: "reading_metadata", - currentChannel: channelTitle, - currentFile: archiveName, - currentFileNum: setIdx + 1, - totalFiles: totalSets, - }); - let entries = []; - try { - if (archiveSet.type === "ZIP") { - entries = await readZipCentralDirectory(tempPaths); - } - else { - entries = await readRarContents(tempPaths[0]); - } - } - catch (err) { - accountLog.warn({ err, baseName: archiveSet.baseName }, "Failed to read archive metadata, ingesting without file list"); - } - // ── Splitting / Repacking (if needed) ── - let uploadPaths = [...tempPaths]; - const totalSize = archiveSet.parts.reduce((sum, p) => sum + p.fileSize, 0n); - const MAX_UPLOAD_SIZE = 2n * 1024n * 1024n * 1024n; - const hasOversizedPart = archiveSet.parts.some((p) => p.fileSize > MAX_UPLOAD_SIZE); - if (hasOversizedPart) { - // Full repack: concatenate all parts → single file → re-split into uniform 2GB chunks - await updateRunActivity(runId, { - currentActivity: `Repacking ${archiveName} (parts >2GB, concatenating + re-splitting)`, - currentStep: "splitting", - currentChannel: channelTitle, - currentFile: archiveName, - currentFileNum: setIdx + 1, - totalFiles: totalSets, - }); - const concatPath = path.join(setDir, `${archiveSet.baseName}.concat`); - await concatenateFiles(tempPaths, concatPath); - splitPaths = await byteLevelSplit(concatPath); - uploadPaths = splitPaths; - // Clean up the concat intermediate file - await unlink(concatPath).catch(() => { }); - } - else if (!archiveSet.isMultipart && totalSize > MAX_UPLOAD_SIZE) { - // Single file >2GB: split directly - await updateRunActivity(runId, { - currentActivity: `Splitting ${archiveName} for upload (>2GB)`, - currentStep: "splitting", - currentChannel: channelTitle, - currentFile: archiveName, - currentFileNum: setIdx + 1, - totalFiles: totalSets, - }); - splitPaths = await byteLevelSplit(tempPaths[0]); - uploadPaths = splitPaths; - } - // ── Uploading ── - const uploadLabel = uploadPaths.length > 1 - ? ` (${uploadPaths.length} parts)` - : ""; - await updateRunActivity(runId, { - currentActivity: `Uploading ${archiveName} to archive channel${uploadLabel}`, - currentStep: "uploading", - currentChannel: channelTitle, - currentFile: archiveName, - currentFileNum: setIdx + 1, - totalFiles: totalSets, - }); - const destResult = await uploadToChannel(client, destChannelTelegramId, uploadPaths); - // ── Preview thumbnail ── - let previewData = null; - let previewMsgId = null; - const matchedPhoto = previewMatches.get(archiveSet.baseName); - if (matchedPhoto) { - await updateRunActivity(runId, { - currentActivity: `Downloading preview image for ${archiveName}`, - currentStep: "preview", - currentChannel: channelTitle, - currentFile: archiveName, - currentFileNum: setIdx + 1, - totalFiles: totalSets, - }); - previewData = await downloadPhotoThumbnail(client, matchedPhoto.fileId); - previewMsgId = matchedPhoto.id; - } - // ── Resolve creator: topic name > filename extraction > null ── - const creator = topicCreator ?? extractCreatorFromFileName(archiveName) ?? null; - // ── Indexing ── - await updateRunActivity(runId, { - currentActivity: `Saving metadata for ${archiveName} (${entries.length} files)`, - currentStep: "indexing", - currentChannel: channelTitle, - currentFile: archiveName, - currentFileNum: setIdx + 1, - totalFiles: totalSets, - }); - // Clean up any orphaned record (same hash but no dest upload) before creating - await deleteOrphanedPackageByHash(contentHash); - await createPackageWithFiles({ - contentHash, - fileName: archiveName, - fileSize: totalSize, - archiveType: archiveSet.type, - sourceChannelId: channel.id, - sourceMessageId: archiveSet.parts[0].id, - sourceTopicId, - destChannelId, - destMessageId: destResult.messageId, - isMultipart: archiveSet.parts.length > 1 || uploadPaths.length > 1, - partCount: uploadPaths.length, - ingestionRunId, - creator, - previewData, - previewMsgId, - files: entries, - }); - counters.zipsIngested++; - await updateRunActivity(runId, { - currentActivity: `Ingested ${archiveName} (${entries.length} files indexed)`, - currentStep: "complete", - currentChannel: channelTitle, - currentFile: archiveName, - currentFileNum: setIdx + 1, - totalFiles: totalSets, - zipsIngested: counters.zipsIngested, - }); - accountLog.info({ fileName: archiveName, contentHash, fileCount: entries.length, creator }, "Archive ingested"); - } - finally { - // ALWAYS delete temp files and the set directory - await deleteFiles([...tempPaths, ...splitPaths]); - await rm(setDir, { recursive: true, force: true }).catch(() => { }); - } -} -async function deleteFiles(paths) { - for (const p of paths) { - try { - await unlink(p); - } - catch { - // File may already be deleted or never created - } - } -} -/** - * Clean up any leftover temp files/directories from previous runs. - */ -export async function cleanupTempDir() { - try { - const entries = await readdir(config.tempDir); - for (const entry of entries) { - await rm(path.join(config.tempDir, entry), { recursive: true, force: true }).catch(() => { }); - } - if (entries.length > 0) { - log.info({ count: entries.length }, "Cleaned up stale temp files"); - } - } - catch { - // Directory might not exist yet - } -} -//# sourceMappingURL=worker.js.map \ No newline at end of file diff --git a/worker/dist/worker.js.map b/worker/dist/worker.js.map deleted file mode 100644 index 663c5b5..0000000 --- a/worker/dist/worker.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"worker.js","sourceRoot":"","sources":["../src/worker.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,aAAa,CAAC;AACzD,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC5D,OAAO,EACL,wBAAwB,EACxB,2BAA2B,EAC3B,mBAAmB,EACnB,4BAA4B,EAC5B,sBAAsB,EACtB,kBAAkB,EAClB,oBAAoB,EACpB,gBAAgB,EAChB,0BAA0B,EAC1B,iBAAiB,EACjB,eAAe,EACf,gBAAgB,EAChB,mBAAmB,EACnB,aAAa,EACb,wBAAwB,EACxB,gBAAgB,EAChB,sBAAsB,EACtB,wBAAwB,EACxB,0BAA0B,EAC1B,+BAA+B,EAE/B,2BAA2B,GAC5B,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACxE,OAAO,EAAE,eAAe,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AACzE,OAAO,EAAE,kBAAkB,EAAE,YAAY,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAE/F,OAAO,EAAE,WAAW,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACrF,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAE1D,OAAO,EAAE,0BAA0B,EAAE,MAAM,sBAAsB,CAAC;AAClE,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EAAE,uBAAuB,EAAE,MAAM,yBAAyB,CAAC;AAClE,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAC1D,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtE,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAItD,MAAM,GAAG,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC;AAElC;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,OAAwB;IAExB,MAAM,IAAI,GAAG,WAAW,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;IAClF,IAAI,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;IAE1C,IAAI,MAA0B,CAAC;IAC/B,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,iBAAiB,CAAC;YAC/B,EAAE,EAAE,OAAO,CAAC,EAAE;YACd,KAAK,EAAE,OAAO,CAAC,KAAK;SACrB,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QAEvC,wDAAwD;QACxD,IAAI,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;QAChD,MAAM,sBAAsB,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QAEvD,2DAA2D;QAC3D,MAAM,UAAU,GAAG,MAAM,gBAAgB,CAAC,yBAAyB,CAAC,CAAC;QACrE,IAAI,UAAU,EAAE,CAAC;YACf,IAAI,CAAC,IAAI,CAAC,yDAAyD,CAAC,CAAC;YACrE,IAAI,CAAC;gBACH,MAAM,oBAAoB,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;gBAC/C,yDAAyD;gBACzD,MAAM,WAAW,GAAG,MAAM,2BAA2B,EAAE,CAAC;gBACxD,IAAI,WAAW,EAAE,CAAC;oBAChB,MAAM,wBAAwB,CAAC,OAAO,CAAC,EAAE,EAAE,WAAW,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;oBACrE,IAAI,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,WAAW,CAAC,KAAK,EAAE,EAAE,+CAA+C,CAAC,CAAC;gBACjG,CAAC;YACH,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,wCAAwC;gBACxC,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,EAAE,4DAA4D,CAAC,CAAC;gBACjF,8BAA8B;gBAC9B,MAAM,WAAW,GAAG,MAAM,2BAA2B,EAAE,CAAC;gBACxD,IAAI,WAAW,EAAE,CAAC;oBAChB,MAAM,wBAAwB,CAAC,OAAO,CAAC,EAAE,EAAE,WAAW,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;gBACvE,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,EAAE,uBAAuB,CAAC,CAAC;IAC/C,CAAC;YAAS,CAAC;QACT,IAAI,MAAM,EAAE,CAAC;YACX,MAAM,gBAAgB,CAAC,MAAM,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,SAAiB;IACzD,MAAM,IAAI,GAAG,WAAW,CAAC,eAAe,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;IACzD,MAAM,OAAO,GAAG,MAAM,sBAAsB,CAAC,SAAS,CAAC,CAAC;IAExD,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QAC7C,IAAI,CAAC,IAAI,CAAC,kDAAkD,CAAC,CAAC;QAC9D,OAAO;IACT,CAAC;IAED,MAAM,wBAAwB,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;IACzD,IAAI,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,EAAE,0BAA0B,CAAC,CAAC;IAExE,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC;QACrC,EAAE,EAAE,OAAO,CAAC,OAAO,CAAC,EAAE;QACtB,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,KAAK;KAC7B,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,CAAC;QAE5C,uBAAuB;QACvB,MAAM,iBAAiB,GAAG,MAAM,0BAA0B,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAC9E,MAAM,gBAAgB,GAAG,MAAM,+BAA+B,EAAE,CAAC;QAEjE,MAAM,aAAa,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACvC,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;YAC7C,OAAO;gBACL,MAAM,EAAE,aAAa;gBACrB,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,IAAI;gBACrC,aAAa,EAAE,iBAAiB,CAAC,GAAG,CAAC,aAAa,CAAC;gBACnD,iBAAiB,EAAE,gBAAgB,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI,IAAI;aAC/D,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,sDAAsD;QACtD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,CAAC;gBACH,MAAM,aAAa,CAAC;oBAClB,UAAU,EAAE,IAAI,CAAC,MAAM;oBACvB,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,IAAI,EAAE,QAAQ;oBACd,OAAO,EAAE,IAAI,CAAC,OAAO;iBACtB,CAAC,CAAC;YACL,CAAC;YAAC,MAAM,CAAC;gBACP,iDAAiD;YACnD,CAAC;QACH,CAAC;QAED,MAAM,wBAAwB,CAAC,SAAS,EAAE,WAAW,EAAE;YACrD,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC;SAC1C,CAAC,CAAC;QAEH,IAAI,CAAC,IAAI,CACP,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,GAAG,iBAAiB,CAAC,CAAC,MAAM,EAAE,EAC9D,yBAAyB,CAC1B,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjE,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,EAAE,sBAAsB,CAAC,CAAC;QAC5C,MAAM,wBAAwB,CAAC,SAAS,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;IAC1E,CAAC;YAAS,CAAC;QACT,MAAM,gBAAgB,CAAC,MAAM,CAAC,CAAC;IACjC,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,sBAAsB,CACnC,MAAc,EACd,SAAiB,EACjB,IAAoC;IAEpC,MAAM,KAAK,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,CAAC;IAE5C,MAAM,iBAAiB,GAAG,MAAM,0BAA0B,CAAC,SAAS,CAAC,CAAC;IACtE,MAAM,gBAAgB,GAAG,MAAM,+BAA+B,EAAE,CAAC;IAEjE,MAAM,aAAa,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACvC,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;QAC7C,OAAO;YACL,MAAM,EAAE,aAAa;YACrB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,IAAI;YACrC,aAAa,EAAE,iBAAiB,CAAC,GAAG,CAAC,aAAa,CAAC;YACnD,iBAAiB,EAAE,gBAAgB,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI,IAAI;SAC/D,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,0BAA0B;IAC1B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC;YACH,MAAM,aAAa,CAAC;gBAClB,UAAU,EAAE,IAAI,CAAC,MAAM;gBACvB,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,IAAI,CAAC,OAAO;aACtB,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,eAAe;QACjB,CAAC;IACH,CAAC;IAED,oEAAoE;IACpE,MAAM,EAAE,EAAE,EAAE,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,CAAC;IAC9C,MAAM,EAAE,CAAC,mBAAmB,CAAC,MAAM,CAAC;QAClC,IAAI,EAAE;YACJ,SAAS;YACT,MAAM,EAAE,WAAW;YACnB,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC;SAC1C;KACF,CAAC,CAAC;IAEH,IAAI,CAAC,IAAI,CACP,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,EACvB,8CAA8C,CAC/C,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,SAAS,8BAA8B,CAAC,KAAa,EAAE,aAAa,GAAG,IAAI;IACzE,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,aAAa,GAA0B,IAAI,CAAC;IAChD,IAAI,UAAU,GAAyC,IAAI,CAAC;IAE5D,MAAM,KAAK,GAAG,KAAK,IAAI,EAAE;QACvB,IAAI,aAAa,EAAE,CAAC;YAClB,MAAM,MAAM,GAAG,aAAa,CAAC;YAC7B,aAAa,GAAG,IAAI,CAAC;YACrB,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC3B,MAAM,iBAAiB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACzD,CAAC;IACH,CAAC,CAAC;IAEF,OAAO;QACL,MAAM,EAAE,CAAC,QAAwB,EAAE,EAAE;YACnC,aAAa,GAAG,QAAQ,CAAC;YACzB,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,aAAa,CAAC;YAC3C,IAAI,OAAO,IAAI,aAAa,EAAE,CAAC;gBAC7B,IAAI,UAAU;oBAAE,YAAY,CAAC,UAAU,CAAC,CAAC;gBACzC,KAAK,EAAE,CAAC;YACV,CAAC;iBAAM,IAAI,CAAC,UAAU,EAAE,CAAC;gBACvB,UAAU,GAAG,UAAU,CAAC,GAAG,EAAE;oBAC3B,UAAU,GAAG,IAAI,CAAC;oBAClB,KAAK,EAAE,CAAC;gBACV,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC;QACD,KAAK;KACN,CAAC;AACJ,CAAC;AAwBD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,OAAwB;IAExB,MAAM,UAAU,GAAG,WAAW,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;IAE1F,2BAA2B;IAC3B,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAClD,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,UAAU,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;QACpD,OAAO;IACT,CAAC;IAED,IAAI,KAAyB,CAAC;IAE9B,IAAI,CAAC;QACH,0BAA0B;QAC1B,MAAM,GAAG,GAAG,MAAM,kBAAkB,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QACjD,KAAK,GAAG,GAAG,CAAC,EAAE,CAAC;QACf,MAAM,WAAW,GAAG,KAAK,CAAC;QAC1B,UAAU,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,EAAE,uBAAuB,CAAC,CAAC;QAEpD,MAAM,SAAS,GAAG,8BAA8B,CAAC,WAAW,CAAC,CAAC;QAE9D,6BAA6B;QAC7B,MAAM,iBAAiB,CAAC,WAAW,EAAE;YACnC,eAAe,EAAE,wBAAwB;YACzC,WAAW,EAAE,YAAY;SAC1B,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC;YACrC,EAAE,EAAE,OAAO,CAAC,EAAE;YACd,KAAK,EAAE,OAAO,CAAC,KAAK;SACrB,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG;YACf,eAAe,EAAE,CAAC;YAClB,SAAS,EAAE,CAAC;YACZ,aAAa,EAAE,CAAC;YAChB,YAAY,EAAE,CAAC;SAChB,CAAC;QAEF,IAAI,CAAC;YACH,yDAAyD;YACzD,MAAM,eAAe,GAAG,MAAM,wBAAwB,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YACnE,MAAM,WAAW,GAAG,MAAM,2BAA2B,EAAE,CAAC;YAExD,IAAI,CAAC,WAAW,EAAE,CAAC;gBACjB,MAAM,IAAI,KAAK,CAAC,oEAAoE,CAAC,CAAC;YACxF,CAAC;YAED,MAAM,aAAa,GAAG,eAAe,CAAC,MAAM,CAAC;YAE7C,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,eAAe,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;gBAC5D,MAAM,OAAO,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;gBACvC,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;gBAChC,MAAM,YAAY,GAAG,aAAa,GAAG,CAAC;oBACpC,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,aAAa,KAAK,OAAO,CAAC,KAAK,EAAE;oBACpD,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;gBAElB,IAAI,CAAC;oBACL,oCAAoC;oBACpC,MAAM,KAAK,GAAG,MAAM,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;oBAC5D,IAAI,KAAK,KAAK,OAAO,CAAC,OAAO,EAAE,CAAC;wBAC9B,MAAM,eAAe,CAAC,OAAO,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;wBACzC,UAAU,CAAC,IAAI,CACb,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,EAC/D,8BAA8B,CAC/B,CAAC;oBACJ,CAAC;oBAED,MAAM,WAAW,GAAoB;wBACnC,MAAM;wBACN,KAAK,EAAE,WAAW;wBAClB,YAAY,EAAE,OAAO,CAAC,KAAK;wBAC3B,OAAO;wBACP,qBAAqB,EAAE,WAAW,CAAC,UAAU;wBAC7C,aAAa,EAAE,WAAW,CAAC,EAAE;wBAC7B,SAAS;wBACT,QAAQ;wBACR,YAAY,EAAE,IAAI;wBAClB,aAAa,EAAE,IAAI;wBACnB,UAAU;qBACX,CAAC;oBAEF,IAAI,KAAK,EAAE,CAAC;wBACV,sCAAsC;wBACtC,MAAM,iBAAiB,CAAC,WAAW,EAAE;4BACnC,eAAe,EAAE,0BAA0B,YAAY,GAAG;4BAC1D,WAAW,EAAE,UAAU;4BACvB,cAAc,EAAE,YAAY;4BAC5B,WAAW,EAAE,IAAI;4BACjB,cAAc,EAAE,IAAI;4BACpB,UAAU,EAAE,IAAI;4BAChB,eAAe,EAAE,IAAI;4BACrB,UAAU,EAAE,IAAI;4BAChB,eAAe,EAAE,IAAI;4BACrB,eAAe,EAAE,QAAQ,CAAC,eAAe;yBAC1C,CAAC,CAAC;wBAEH,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;wBACnE,MAAM,iBAAiB,GAAG,MAAM,gBAAgB,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;wBAE7D,UAAU,CAAC,IAAI,CACb,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,EAC1E,iCAAiC,CAClC,CAAC;wBAEF,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC;4BAChD,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;4BAC3B,IAAI,CAAC;gCACH,MAAM,QAAQ,GAAG,iBAAiB,CAAC,IAAI,CACrC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,OAAO,KAAK,KAAK,CAAC,OAAO,CACrC,CAAC;gCAEF,MAAM,UAAU,GAAG,GAAG,OAAO,CAAC,KAAK,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;gCACtD,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC;oCACrC,CAAC,CAAC,WAAW,IAAI,GAAG,CAAC,IAAI,MAAM,CAAC,MAAM,GAAG;oCACzC,CAAC,CAAC,EAAE,CAAC;gCAEP,MAAM,iBAAiB,CAAC,WAAW,EAAE;oCACnC,eAAe,EAAE,aAAa,UAAU,IAAI,aAAa,EAAE;oCAC3D,WAAW,EAAE,UAAU;oCACvB,cAAc,EAAE,YAAY;oCAC5B,WAAW,EAAE,IAAI;oCACjB,cAAc,EAAE,IAAI;oCACpB,UAAU,EAAE,IAAI;oCAChB,eAAe,EAAE,IAAI;oCACrB,UAAU,EAAE,IAAI;oCAChB,eAAe,EAAE,IAAI;oCACrB,eAAe,EAAE,QAAQ,CAAC,eAAe;iCAC1C,CAAC,CAAC;gCAEH,MAAM,UAAU,GAAG,MAAM,gBAAgB,CACvC,MAAM,EACN,OAAO,CAAC,UAAU,EAClB,KAAK,CAAC,OAAO,EACb,QAAQ,EAAE,sBAAsB,EAChC,GAAG,EACH,CAAC,OAAO,EAAE,EAAE;oCACV,SAAS,CAAC,MAAM,CAAC;wCACf,eAAe,EAAE,aAAa,UAAU,IAAI,aAAa,MAAM,OAAO,mBAAmB;wCACzF,WAAW,EAAE,UAAU;wCACvB,cAAc,EAAE,YAAY;wCAC5B,eAAe,EAAE,QAAQ,CAAC,eAAe,GAAG,OAAO;qCACpD,CAAC,CAAC;gCACL,CAAC,CACF,CAAC;gCAEF,yCAAyC;gCACzC,QAAQ,CAAC,eAAe,IAAI,UAAU,CAAC,YAAY,CAAC;gCAEpD,IAAI,UAAU,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oCACrC,UAAU,CAAC,KAAK,CACd,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,EAC5C,0BAA0B,CAC3B,CAAC;oCACF,SAAS;gCACX,CAAC;gCAED,UAAU,CAAC,IAAI,CACb,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE,UAAU,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,CAAC,MAAM,EAAE,EAC7F,yBAAyB,CAC1B,CAAC;gCAEF,sCAAsC;gCACtC,WAAW,CAAC,YAAY,GAAG,KAAK,CAAC,IAAI,CAAC;gCACtC,WAAW,CAAC,aAAa,GAAG,KAAK,CAAC,OAAO,CAAC;gCAC1C,WAAW,CAAC,YAAY,GAAG,GAAG,OAAO,CAAC,KAAK,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;gCAE9D,MAAM,cAAc,GAAG,MAAM,kBAAkB,CAAC,WAAW,EAAE,UAAU,EAAE,GAAG,CAAC,EAAE,EAAE,QAAQ,EAAE,sBAAsB,CAAC,CAAC;gCAEnH,sEAAsE;gCACtE,IAAI,cAAc,EAAE,CAAC;oCACnB,MAAM,mBAAmB,CACvB,OAAO,CAAC,EAAE,EACV,KAAK,CAAC,OAAO,EACb,KAAK,CAAC,IAAI,EACV,cAAc,CACf,CAAC;gCACJ,CAAC;4BACH,CAAC;4BAAC,OAAO,QAAQ,EAAE,CAAC;gCAClB,UAAU,CAAC,IAAI,CACb,EAAE,GAAG,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,EAC9F,mCAAmC,CACpC,CAAC;4BACJ,CAAC;wBACH,CAAC;oBACH,CAAC;yBAAM,CAAC;wBACN,yDAAyD;wBACzD,MAAM,iBAAiB,CAAC,WAAW,EAAE;4BACnC,eAAe,EAAE,aAAa,YAAY,oBAAoB;4BAC9D,WAAW,EAAE,UAAU;4BACvB,cAAc,EAAE,YAAY;4BAC5B,WAAW,EAAE,IAAI;4BACjB,cAAc,EAAE,IAAI;4BACpB,UAAU,EAAE,IAAI;4BAChB,eAAe,EAAE,IAAI;4BACrB,UAAU,EAAE,IAAI;4BAChB,eAAe,EAAE,IAAI;4BACrB,eAAe,EAAE,QAAQ,CAAC,eAAe;yBAC1C,CAAC,CAAC;wBAEH,UAAU,CAAC,IAAI,CACb,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,EAC/C,2BAA2B,CAC5B,CAAC;wBAEF,MAAM,UAAU,GAAG,MAAM,kBAAkB,CACzC,MAAM,EACN,OAAO,CAAC,UAAU,EAClB,OAAO,CAAC,sBAAsB,EAC9B,GAAG,EACH,CAAC,OAAO,EAAE,EAAE;4BACV,SAAS,CAAC,MAAM,CAAC;gCACf,eAAe,EAAE,aAAa,YAAY,OAAO,OAAO,mBAAmB;gCAC3E,WAAW,EAAE,UAAU;gCACvB,cAAc,EAAE,YAAY;gCAC5B,eAAe,EAAE,QAAQ,CAAC,eAAe,GAAG,OAAO;6BACpD,CAAC,CAAC;wBACL,CAAC,CACF,CAAC;wBAEF,yCAAyC;wBACzC,QAAQ,CAAC,eAAe,IAAI,UAAU,CAAC,YAAY,CAAC;wBAEpD,IAAI,UAAU,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;4BACrC,UAAU,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,EAAE,iBAAiB,CAAC,CAAC;4BAC/D,SAAS;wBACX,CAAC;wBAED,UAAU,CAAC,IAAI,CACb,EAAE,QAAQ,EAAE,UAAU,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,CAAC,MAAM,EAAE,EAC1E,2BAA2B,CAC5B,CAAC;wBAEF,iFAAiF;wBACjF,WAAW,CAAC,YAAY,GAAG,IAAI,CAAC;wBAChC,WAAW,CAAC,aAAa,GAAG,IAAI,CAAC;wBACjC,WAAW,CAAC,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC;wBAEzC,MAAM,cAAc,GAAG,MAAM,kBAAkB,CAAC,WAAW,EAAE,UAAU,EAAE,GAAG,CAAC,EAAE,EAAE,OAAO,CAAC,sBAAsB,CAAC,CAAC;wBAEjH,sEAAsE;wBACtE,IAAI,cAAc,EAAE,CAAC;4BACnB,MAAM,0BAA0B,CAAC,OAAO,CAAC,EAAE,EAAE,cAAc,CAAC,CAAC;wBAC/D,CAAC;oBACH,CAAC;gBACD,CAAC;gBAAC,OAAO,UAAU,EAAE,CAAC;oBACpB,UAAU,CAAC,IAAI,CACb,EAAE,GAAG,EAAE,UAAU,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,EAChE,6CAA6C,CAC9C,CAAC;gBACJ,CAAC;YACH,CAAC;YAED,aAAa;YACb,MAAM,oBAAoB,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;YAClD,UAAU,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,EAAE,yBAAyB,CAAC,CAAC;QAC3D,CAAC;gBAAS,CAAC;YACT,MAAM,gBAAgB,CAAC,MAAM,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjE,UAAU,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,EAAE,sBAAsB,CAAC,CAAC;QAClD,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,gBAAgB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CACjD,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,8BAA8B,CAAC,CACxD,CAAC;QACJ,CAAC;IACH,CAAC;YAAS,CAAC;QACT,MAAM,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAChC,CAAC;AACH,CAAC;AAED;;;;;;;GAOG;AACH,KAAK,UAAU,kBAAkB,CAC/B,GAAoB,EACpB,UAA6B,EAC7B,cAAsB,EACtB,sBAAsC;IAEtC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,YAAY,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU,EAAE,GAAG,GAAG,CAAC;IAEtF,0BAA0B;IAC1B,IAAI,WAAW,GAAG,gBAAgB,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IAExD,mFAAmF;IACnF,IAAI,sBAAsB,EAAE,CAAC;QAC3B,MAAM,WAAW,GAAG,WAAW,CAAC,MAAM,CAAC;QACvC,WAAW,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CACvC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,sBAAsB,CAAC,CACrD,CAAC;QACF,MAAM,QAAQ,GAAG,WAAW,GAAG,WAAW,CAAC,MAAM,CAAC;QAClD,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;YACjB,UAAU,CAAC,IAAI,CACb,EAAE,QAAQ,EAAE,SAAS,EAAE,WAAW,CAAC,MAAM,EAAE,EAC3C,6CAA6C,CAC9C,CAAC;QACJ,CAAC;IACH,CAAC;IAED,QAAQ,CAAC,SAAS,IAAI,WAAW,CAAC,MAAM,CAAC;IAEzC,uCAAuC;IACvC,MAAM,cAAc,GAAG,qBAAqB,CAC1C,UAAU,CAAC,MAAM,EACjB,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACtB,QAAQ,EAAE,CAAC,CAAC,QAAQ;QACpB,cAAc,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE;QAC7B,gBAAgB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI;KAClC,CAAC,CAAC,CACJ,CAAC;IAEF,IAAI,cAAc,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;QAC5B,UAAU,CAAC,IAAI,CACb,EAAE,OAAO,EAAE,cAAc,CAAC,IAAI,EAAE,KAAK,EAAE,WAAW,CAAC,MAAM,EAAE,EAC3D,oCAAoC,CACrC,CAAC;IACJ,CAAC;IAED,MAAM,iBAAiB,CAAC,KAAK,EAAE;QAC7B,eAAe,EAAE,SAAS,WAAW,CAAC,MAAM,mBAAmB,YAAY,GAAG;QAC9E,WAAW,EAAE,UAAU;QACvB,cAAc,EAAE,YAAY;QAC5B,UAAU,EAAE,WAAW,CAAC,MAAM;QAC9B,SAAS,EAAE,QAAQ,CAAC,SAAS;QAC7B,eAAe,EAAE,QAAQ,CAAC,eAAe;KAC1C,CAAC,CAAC;IAEH,+DAA+D;IAC/D,IAAI,cAAc,GAAkB,IAAI,CAAC;IAEzC,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC;QAC3D,IAAI,CAAC;YACH,MAAM,oBAAoB,CACxB,GAAG,EACH,WAAW,CAAC,MAAM,CAAC,EACnB,MAAM,EACN,WAAW,CAAC,MAAM,EAClB,cAAc,EACd,cAAc,CACf,CAAC;YAEF,sEAAsE;YACtE,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,MAAM,CAC/C,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,EACrC,EAAE,CACH,CAAC;YACF,IAAI,QAAQ,GAAG,CAAC,cAAc,IAAI,EAAE,CAAC,EAAE,CAAC;gBACtC,cAAc,GAAG,QAAQ,CAAC;YAC5B,CAAC;QACH,CAAC;QAAC,OAAO,MAAM,EAAE,CAAC;YAChB,uDAAuD;YACvD,UAAU,CAAC,IAAI,CACb,EAAE,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EACvD,8DAA8D,CAC/D,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,cAAc,CAAC;AACxB,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,oBAAoB,CACjC,GAAoB,EACpB,UAAsB,EACtB,MAAc,EACd,SAAiB,EACjB,cAA2D,EAC3D,cAAsB;IAEtB,MAAM,EACJ,MAAM,EAAE,KAAK,EAAE,YAAY,EAAE,OAAO,EACpC,qBAAqB,EAAE,aAAa,EACpC,SAAS,EAAE,QAAQ,EAAE,YAAY,EAAE,aAAa,EAAE,UAAU,GAC7D,GAAG,GAAG,CAAC;IAER,MAAM,WAAW,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;IAEjD,mEAAmE;IACnE,gFAAgF;IAChF,MAAM,eAAe,GAAG,MAAM,4BAA4B,CACxD,OAAO,CAAC,EAAE,EACV,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CACvB,CAAC;IACF,IAAI,eAAe,EAAE,CAAC;QACpB,QAAQ,CAAC,aAAa,EAAE,CAAC;QACzB,UAAU,CAAC,KAAK,CACd,EAAE,QAAQ,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAC1E,wDAAwD,CACzD,CAAC;QACF,MAAM,iBAAiB,CAAC,KAAK,EAAE;YAC7B,eAAe,EAAE,WAAW,WAAW,qBAAqB;YAC5D,WAAW,EAAE,eAAe;YAC5B,cAAc,EAAE,YAAY;YAC5B,WAAW,EAAE,WAAW;YACxB,cAAc,EAAE,MAAM,GAAG,CAAC;YAC1B,UAAU,EAAE,SAAS;YACrB,aAAa,EAAE,QAAQ,CAAC,aAAa;SACtC,CAAC,CAAC;QACH,OAAO;IACT,CAAC;IAED,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,IAAI,UAAU,GAAa,EAAE,CAAC;IAE9B,uEAAuE;IACvE,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,cAAc,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACxF,MAAM,KAAK,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAEzC,IAAI,CAAC;QACH,oBAAoB;QACpB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,UAAU,CAAC,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC;YACnE,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;YAElD,MAAM,SAAS,GAAG,UAAU,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC;gBAC3C,CAAC,CAAC,UAAU,OAAO,GAAG,CAAC,IAAI,UAAU,CAAC,KAAK,CAAC,MAAM,GAAG;gBACrD,CAAC,CAAC,EAAE,CAAC;YAEP,MAAM,iBAAiB,CAAC,KAAK,EAAE;gBAC7B,eAAe,EAAE,eAAe,IAAI,CAAC,QAAQ,GAAG,SAAS,EAAE;gBAC3D,WAAW,EAAE,aAAa;gBAC1B,cAAc,EAAE,YAAY;gBAC5B,WAAW,EAAE,IAAI,CAAC,QAAQ;gBAC1B,cAAc,EAAE,MAAM,GAAG,CAAC;gBAC1B,UAAU,EAAE,SAAS;gBACrB,eAAe,EAAE,EAAE;gBACnB,UAAU,EAAE,IAAI,CAAC,QAAQ;gBACzB,eAAe,EAAE,CAAC;gBAClB,eAAe,EAAE,QAAQ,CAAC,eAAe;aAC1C,CAAC,CAAC;YAEH,UAAU,CAAC,IAAI,CACb;gBACE,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;gBAC/B,IAAI,EAAE,OAAO,GAAG,CAAC;gBACjB,UAAU,EAAE,UAAU,CAAC,KAAK,CAAC,MAAM;aACpC,EACD,0BAA0B,CAC3B,CAAC;YAEF,MAAM,YAAY,CAChB,MAAM,EACN,IAAI,CAAC,MAAM,EACX,QAAQ,EACR,IAAI,CAAC,QAAQ,EACb,IAAI,CAAC,QAAQ,EACb,CAAC,QAA0B,EAAE,EAAE;gBAC7B,SAAS,CAAC,MAAM,CAAC;oBACf,eAAe,EAAE,eAAe,IAAI,CAAC,QAAQ,GAAG,SAAS,MAAM,QAAQ,CAAC,OAAO,GAAG;oBAClF,WAAW,EAAE,aAAa;oBAC1B,cAAc,EAAE,YAAY;oBAC5B,WAAW,EAAE,IAAI,CAAC,QAAQ;oBAC1B,cAAc,EAAE,MAAM,GAAG,CAAC;oBAC1B,UAAU,EAAE,SAAS;oBACrB,eAAe,EAAE,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAC;oBACjD,UAAU,EAAE,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC;oBACvC,eAAe,EAAE,QAAQ,CAAC,OAAO;iBAClC,CAAC,CAAC;YACL,CAAC,CACF,CAAC;YACF,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;YACxB,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC3B,CAAC;QAED,gBAAgB;QAChB,MAAM,iBAAiB,CAAC,KAAK,EAAE;YAC7B,eAAe,EAAE,sBAAsB,WAAW,EAAE;YACpD,WAAW,EAAE,SAAS;YACtB,cAAc,EAAE,YAAY;YAC5B,WAAW,EAAE,WAAW;YACxB,cAAc,EAAE,MAAM,GAAG,CAAC;YAC1B,UAAU,EAAE,SAAS;YACrB,eAAe,EAAE,IAAI;YACrB,UAAU,EAAE,IAAI;YAChB,eAAe,EAAE,IAAI;SACtB,CAAC,CAAC;QAEH,MAAM,WAAW,GAAG,MAAM,SAAS,CAAC,SAAS,CAAC,CAAC;QAE/C,sBAAsB;QACtB,MAAM,iBAAiB,CAAC,KAAK,EAAE;YAC7B,eAAe,EAAE,eAAe,WAAW,iBAAiB;YAC5D,WAAW,EAAE,eAAe;YAC5B,cAAc,EAAE,YAAY;YAC5B,WAAW,EAAE,WAAW;YACxB,cAAc,EAAE,MAAM,GAAG,CAAC;YAC1B,UAAU,EAAE,SAAS;SACtB,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC,WAAW,CAAC,CAAC;QACtD,IAAI,MAAM,EAAE,CAAC;YACX,QAAQ,CAAC,aAAa,EAAE,CAAC;YACzB,UAAU,CAAC,KAAK,CAAC,EAAE,WAAW,EAAE,EAAE,6BAA6B,CAAC,CAAC;YAEjE,MAAM,iBAAiB,CAAC,KAAK,EAAE;gBAC7B,eAAe,EAAE,WAAW,WAAW,cAAc;gBACrD,WAAW,EAAE,eAAe;gBAC5B,cAAc,EAAE,YAAY;gBAC5B,WAAW,EAAE,WAAW;gBACxB,cAAc,EAAE,MAAM,GAAG,CAAC;gBAC1B,UAAU,EAAE,SAAS;gBACrB,aAAa,EAAE,QAAQ,CAAC,aAAa;aACtC,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,yBAAyB;QACzB,MAAM,iBAAiB,CAAC,KAAK,EAAE;YAC7B,eAAe,EAAE,0BAA0B,WAAW,EAAE;YACxD,WAAW,EAAE,kBAAkB;YAC/B,cAAc,EAAE,YAAY;YAC5B,WAAW,EAAE,WAAW;YACxB,cAAc,EAAE,MAAM,GAAG,CAAC;YAC1B,UAAU,EAAE,SAAS;SACtB,CAAC,CAAC;QAEH,IAAI,OAAO,GAA2I,EAAE,CAAC;QACzJ,IAAI,CAAC;YACH,IAAI,UAAU,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;gBAC9B,OAAO,GAAG,MAAM,uBAAuB,CAAC,SAAS,CAAC,CAAC;YACrD,CAAC;iBAAM,CAAC;gBACN,OAAO,GAAG,MAAM,eAAe,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;YAChD,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,UAAU,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,UAAU,CAAC,QAAQ,EAAE,EAAE,8DAA8D,CAAC,CAAC;QAC1H,CAAC;QAED,0CAA0C;QAC1C,IAAI,WAAW,GAAG,CAAC,GAAG,SAAS,CAAC,CAAC;QACjC,MAAM,SAAS,GAAG,UAAU,CAAC,KAAK,CAAC,MAAM,CACvC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,EAC5B,EAAE,CACH,CAAC;QACF,MAAM,eAAe,GAAG,EAAE,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,CAAC;QACnD,MAAM,gBAAgB,GAAG,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,eAAe,CAAC,CAAC;QAEpF,IAAI,gBAAgB,EAAE,CAAC;YACrB,sFAAsF;YACtF,MAAM,iBAAiB,CAAC,KAAK,EAAE;gBAC7B,eAAe,EAAE,aAAa,WAAW,6CAA6C;gBACtF,WAAW,EAAE,WAAW;gBACxB,cAAc,EAAE,YAAY;gBAC5B,WAAW,EAAE,WAAW;gBACxB,cAAc,EAAE,MAAM,GAAG,CAAC;gBAC1B,UAAU,EAAE,SAAS;aACtB,CAAC,CAAC;YACH,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,UAAU,CAAC,QAAQ,SAAS,CAAC,CAAC;YACtE,MAAM,gBAAgB,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;YAC9C,UAAU,GAAG,MAAM,cAAc,CAAC,UAAU,CAAC,CAAC;YAC9C,WAAW,GAAG,UAAU,CAAC;YACzB,wCAAwC;YACxC,MAAM,MAAM,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAC3C,CAAC;aAAM,IAAI,CAAC,UAAU,CAAC,WAAW,IAAI,SAAS,GAAG,eAAe,EAAE,CAAC;YAClE,mCAAmC;YACnC,MAAM,iBAAiB,CAAC,KAAK,EAAE;gBAC7B,eAAe,EAAE,aAAa,WAAW,oBAAoB;gBAC7D,WAAW,EAAE,WAAW;gBACxB,cAAc,EAAE,YAAY;gBAC5B,WAAW,EAAE,WAAW;gBACxB,cAAc,EAAE,MAAM,GAAG,CAAC;gBAC1B,UAAU,EAAE,SAAS;aACtB,CAAC,CAAC;YACH,UAAU,GAAG,MAAM,cAAc,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;YAChD,WAAW,GAAG,UAAU,CAAC;QAC3B,CAAC;QAED,kBAAkB;QAClB,MAAM,WAAW,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC;YACxC,CAAC,CAAC,KAAK,WAAW,CAAC,MAAM,SAAS;YAClC,CAAC,CAAC,EAAE,CAAC;QACP,MAAM,iBAAiB,CAAC,KAAK,EAAE;YAC7B,eAAe,EAAE,aAAa,WAAW,sBAAsB,WAAW,EAAE;YAC5E,WAAW,EAAE,WAAW;YACxB,cAAc,EAAE,YAAY;YAC5B,WAAW,EAAE,WAAW;YACxB,cAAc,EAAE,MAAM,GAAG,CAAC;YAC1B,UAAU,EAAE,SAAS;SACtB,CAAC,CAAC;QAEH,MAAM,UAAU,GAAG,MAAM,eAAe,CACtC,MAAM,EACN,qBAAqB,EACrB,WAAW,CACZ,CAAC;QAEF,0BAA0B;QAC1B,IAAI,WAAW,GAAkB,IAAI,CAAC;QACtC,IAAI,YAAY,GAAkB,IAAI,CAAC;QACvC,MAAM,YAAY,GAAG,cAAc,CAAC,GAAG,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;QAC7D,IAAI,YAAY,EAAE,CAAC;YACjB,MAAM,iBAAiB,CAAC,KAAK,EAAE;gBAC7B,eAAe,EAAE,iCAAiC,WAAW,EAAE;gBAC/D,WAAW,EAAE,SAAS;gBACtB,cAAc,EAAE,YAAY;gBAC5B,WAAW,EAAE,WAAW;gBACxB,cAAc,EAAE,MAAM,GAAG,CAAC;gBAC1B,UAAU,EAAE,SAAS;aACtB,CAAC,CAAC;YACH,WAAW,GAAG,MAAM,sBAAsB,CAAC,MAAM,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC;YACxE,YAAY,GAAG,YAAY,CAAC,EAAE,CAAC;QACjC,CAAC;QAED,iEAAiE;QACjE,MAAM,OAAO,GAAG,YAAY,IAAI,0BAA0B,CAAC,WAAW,CAAC,IAAI,IAAI,CAAC;QAEhF,iBAAiB;QACjB,MAAM,iBAAiB,CAAC,KAAK,EAAE;YAC7B,eAAe,EAAE,uBAAuB,WAAW,KAAK,OAAO,CAAC,MAAM,SAAS;YAC/E,WAAW,EAAE,UAAU;YACvB,cAAc,EAAE,YAAY;YAC5B,WAAW,EAAE,WAAW;YACxB,cAAc,EAAE,MAAM,GAAG,CAAC;YAC1B,UAAU,EAAE,SAAS;SACtB,CAAC,CAAC;QAEH,8EAA8E;QAC9E,MAAM,2BAA2B,CAAC,WAAW,CAAC,CAAC;QAE/C,MAAM,sBAAsB,CAAC;YAC3B,WAAW;YACX,QAAQ,EAAE,WAAW;YACrB,QAAQ,EAAE,SAAS;YACnB,WAAW,EAAE,UAAU,CAAC,IAAI;YAC5B,eAAe,EAAE,OAAO,CAAC,EAAE;YAC3B,eAAe,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE;YACvC,aAAa;YACb,aAAa;YACb,aAAa,EAAE,UAAU,CAAC,SAAS;YACnC,WAAW,EACT,UAAU,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC;YACvD,SAAS,EAAE,WAAW,CAAC,MAAM;YAC7B,cAAc;YACd,OAAO;YACP,WAAW;YACX,YAAY;YACZ,KAAK,EAAE,OAAO;SACf,CAAC,CAAC;QAEH,QAAQ,CAAC,YAAY,EAAE,CAAC;QAExB,MAAM,iBAAiB,CAAC,KAAK,EAAE;YAC7B,eAAe,EAAE,YAAY,WAAW,KAAK,OAAO,CAAC,MAAM,iBAAiB;YAC5E,WAAW,EAAE,UAAU;YACvB,cAAc,EAAE,YAAY;YAC5B,WAAW,EAAE,WAAW;YACxB,cAAc,EAAE,MAAM,GAAG,CAAC;YAC1B,UAAU,EAAE,SAAS;YACrB,YAAY,EAAE,QAAQ,CAAC,YAAY;SACpC,CAAC,CAAC;QAEH,UAAU,CAAC,IAAI,CACb,EAAE,QAAQ,EAAE,WAAW,EAAE,WAAW,EAAE,SAAS,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,EAC1E,kBAAkB,CACnB,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,iDAAiD;QACjD,MAAM,WAAW,CAAC,CAAC,GAAG,SAAS,EAAE,GAAG,UAAU,CAAC,CAAC,CAAC;QACjD,MAAM,EAAE,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACrE,CAAC;AACH,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,KAAe;IACxC,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAAC,MAAM,CAAC;YACP,+CAA+C;QACjD,CAAC;IACH,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc;IAClC,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC9C,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAC/F,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,GAAG,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,EAAE,6BAA6B,CAAC,CAAC;QACrE,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,gCAAgC;IAClC,CAAC;AACH,CAAC"} \ No newline at end of file From 49b82a352bc2c91abae06aeb658657758f9268df Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 5 Mar 2026 13:17:56 +0000 Subject: [PATCH 4/4] Fix review issues: race condition in invokeWithTimeout and mutex queue entry - Add settled flag to invokeWithTimeout to prevent double-settling - Create mutex queue entry with wrapped resolve before pushing to queue Co-authored-by: xCyanGrizzly <53275238+xCyanGrizzly@users.noreply.github.com> --- worker/src/tdlib/download.ts | 21 ++++++++++++++++----- worker/src/util/mutex.ts | 17 ++++++++--------- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/worker/src/tdlib/download.ts b/worker/src/tdlib/download.ts index d9c4a88..600c7d7 100644 --- a/worker/src/tdlib/download.ts +++ b/worker/src/tdlib/download.ts @@ -88,18 +88,29 @@ export async function invokeWithTimeout( timeoutMs = INVOKE_TIMEOUT_MS ): Promise { return new Promise((resolve, reject) => { + let settled = false; + const timer = setTimeout(() => { - reject(new Error(`TDLib invoke timed out after ${timeoutMs}ms for ${request._}`)); + if (!settled) { + settled = true; + reject(new Error(`TDLib invoke timed out after ${timeoutMs}ms for ${request._}`)); + } }, timeoutMs); (client.invoke(request) as Promise) .then((result) => { - clearTimeout(timer); - resolve(result); + if (!settled) { + settled = true; + clearTimeout(timer); + resolve(result); + } }) .catch((err) => { - clearTimeout(timer); - reject(err); + if (!settled) { + settled = true; + clearTimeout(timer); + reject(err); + } }); }); } diff --git a/worker/src/util/mutex.ts b/worker/src/util/mutex.ts index dcb95cf..e559318 100644 --- a/worker/src/util/mutex.ts +++ b/worker/src/util/mutex.ts @@ -27,10 +27,6 @@ export async function withTdlibMutex( if (locked) { log.info({ waiting: label, holder }, "Waiting for TDLib mutex"); await new Promise((resolve, reject) => { - const entry = { resolve, reject, label }; - queue.push(entry); - - // Timeout: reject if we've been waiting too long const timer = setTimeout(() => { const idx = queue.indexOf(entry); if (idx !== -1) { @@ -42,12 +38,15 @@ export async function withTdlibMutex( } }, MUTEX_WAIT_TIMEOUT_MS); - // Wrap resolve to clear the timer - const origResolve = entry.resolve; - entry.resolve = () => { - clearTimeout(timer); - origResolve(); + const entry = { + resolve: () => { + clearTimeout(timer); + resolve(); + }, + reject, + label, }; + queue.push(entry); }); }