mirror of
https://github.com/xCyanGrizzly/DragonsStash.git
synced 2026-05-10 22:01:16 +00:00
Remove worker/dist build artifacts from git, add to .gitignore
Co-authored-by: xCyanGrizzly <53275238+xCyanGrizzly@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
12
worker/dist/archive/creator.d.ts
vendored
12
worker/dist/archive/creator.d.ts
vendored
@@ -1,12 +0,0 @@
|
||||
/**
|
||||
* Extract a creator name from common archive file naming patterns.
|
||||
*
|
||||
* Priority in the worker: topic name > filename extraction.
|
||||
* This is the fallback when no forum topic name is available.
|
||||
*
|
||||
* Patterns handled (split on ` - `):
|
||||
* "Mammoth Factory - 2026-01.zip" → "Mammoth Factory"
|
||||
* "Artist Name - Pack Title.part01.rar" → "Artist Name"
|
||||
* "some_random_file.zip" → null
|
||||
*/
|
||||
export declare function extractCreatorFromFileName(fileName: string): string | null;
|
||||
21
worker/dist/archive/creator.js
vendored
21
worker/dist/archive/creator.js
vendored
@@ -1,21 +0,0 @@
|
||||
/**
|
||||
* Extract a creator name from common archive file naming patterns.
|
||||
*
|
||||
* Priority in the worker: topic name > filename extraction.
|
||||
* This is the fallback when no forum topic name is available.
|
||||
*
|
||||
* Patterns handled (split on ` - `):
|
||||
* "Mammoth Factory - 2026-01.zip" → "Mammoth Factory"
|
||||
* "Artist Name - Pack Title.part01.rar" → "Artist Name"
|
||||
* "some_random_file.zip" → null
|
||||
*/
|
||||
export function extractCreatorFromFileName(fileName) {
|
||||
// Strip archive extensions (.zip, .rar, .part01.rar, .z01, etc.)
|
||||
const bare = fileName.replace(/(\.(part\d+\.rar|z\d{2}|zip|rar))+$/i, "");
|
||||
const idx = bare.indexOf(" - ");
|
||||
if (idx <= 0)
|
||||
return null;
|
||||
const creator = bare.slice(0, idx).trim();
|
||||
return creator.length > 0 ? creator : null;
|
||||
}
|
||||
//# sourceMappingURL=creator.js.map
|
||||
1
worker/dist/archive/creator.js.map
vendored
1
worker/dist/archive/creator.js.map
vendored
@@ -1 +0,0 @@
|
||||
{"version":3,"file":"creator.js","sourceRoot":"","sources":["../../src/archive/creator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AACH,MAAM,UAAU,0BAA0B,CAAC,QAAgB;IACzD,iEAAiE;IACjE,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,sCAAsC,EAAE,EAAE,CAAC,CAAC;IAE1E,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAChC,IAAI,GAAG,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IAE1B,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IAC1C,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;AAC7C,CAAC"}
|
||||
15
worker/dist/archive/detect.d.ts
vendored
15
worker/dist/archive/detect.d.ts
vendored
@@ -1,15 +0,0 @@
|
||||
export type ArchiveFormat = "ZIP" | "RAR";
|
||||
export interface MultipartInfo {
|
||||
baseName: string;
|
||||
partNumber: number;
|
||||
format: ArchiveFormat;
|
||||
pattern: "ZIP_NUMBERED" | "ZIP_LEGACY" | "RAR_PART" | "RAR_LEGACY" | "SINGLE";
|
||||
}
|
||||
/**
|
||||
* Detect if a filename is an archive and extract multipart info.
|
||||
*/
|
||||
export declare function detectArchive(fileName: string): MultipartInfo | null;
|
||||
/**
|
||||
* Check if a filename looks like any archive attachment we should process.
|
||||
*/
|
||||
export declare function isArchiveAttachment(fileName: string): boolean;
|
||||
77
worker/dist/archive/detect.js
vendored
77
worker/dist/archive/detect.js
vendored
@@ -1,77 +0,0 @@
|
||||
const patterns = [
|
||||
// pack.zip.001, pack.zip.002
|
||||
{
|
||||
regex: /^(.+\.zip)\.(\d{3,})$/i,
|
||||
format: "ZIP",
|
||||
pattern: "ZIP_NUMBERED",
|
||||
getBaseName: (m) => m[1],
|
||||
getPartNumber: (m) => parseInt(m[2], 10),
|
||||
},
|
||||
// pack.z01, pack.z02 (legacy split — final part is pack.zip)
|
||||
{
|
||||
regex: /^(.+)\.z(\d{2,})$/i,
|
||||
format: "ZIP",
|
||||
pattern: "ZIP_LEGACY",
|
||||
getBaseName: (m) => m[1],
|
||||
getPartNumber: (m) => parseInt(m[2], 10),
|
||||
},
|
||||
// pack.part1.rar, pack.part2.rar
|
||||
{
|
||||
regex: /^(.+)\.part(\d+)\.rar$/i,
|
||||
format: "RAR",
|
||||
pattern: "RAR_PART",
|
||||
getBaseName: (m) => m[1],
|
||||
getPartNumber: (m) => parseInt(m[2], 10),
|
||||
},
|
||||
// pack.r00, pack.r01 (legacy split — final part is pack.rar)
|
||||
{
|
||||
regex: /^(.+)\.r(\d{2,})$/i,
|
||||
format: "RAR",
|
||||
pattern: "RAR_LEGACY",
|
||||
getBaseName: (m) => m[1],
|
||||
getPartNumber: (m) => parseInt(m[2], 10),
|
||||
},
|
||||
];
|
||||
/**
|
||||
* Detect if a filename is an archive and extract multipart info.
|
||||
*/
|
||||
export function detectArchive(fileName) {
|
||||
// Check multipart patterns first
|
||||
for (const p of patterns) {
|
||||
const match = fileName.match(p.regex);
|
||||
if (match) {
|
||||
return {
|
||||
baseName: p.getBaseName(match),
|
||||
partNumber: p.getPartNumber(match),
|
||||
format: p.format,
|
||||
pattern: p.pattern,
|
||||
};
|
||||
}
|
||||
}
|
||||
// Single .zip file — could be a standalone or the final part of a ZIP_LEGACY set
|
||||
if (/\.zip$/i.test(fileName)) {
|
||||
return {
|
||||
baseName: fileName.replace(/\.zip$/i, ""),
|
||||
partNumber: -1, // -1 signals "could be single or final legacy part"
|
||||
format: "ZIP",
|
||||
pattern: "SINGLE",
|
||||
};
|
||||
}
|
||||
// Single .rar file — could be standalone or final part of RAR_LEGACY set
|
||||
if (/\.rar$/i.test(fileName)) {
|
||||
return {
|
||||
baseName: fileName.replace(/\.rar$/i, ""),
|
||||
partNumber: -1,
|
||||
format: "RAR",
|
||||
pattern: "SINGLE",
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
/**
|
||||
* Check if a filename looks like any archive attachment we should process.
|
||||
*/
|
||||
export function isArchiveAttachment(fileName) {
|
||||
return detectArchive(fileName) !== null;
|
||||
}
|
||||
//# sourceMappingURL=detect.js.map
|
||||
1
worker/dist/archive/detect.js.map
vendored
1
worker/dist/archive/detect.js.map
vendored
@@ -1 +0,0 @@
|
||||
{"version":3,"file":"detect.js","sourceRoot":"","sources":["../../src/archive/detect.ts"],"names":[],"mappings":"AASA,MAAM,QAAQ,GAMR;IACJ,6BAA6B;IAC7B;QACE,KAAK,EAAE,wBAAwB;QAC/B,MAAM,EAAE,KAAK;QACb,OAAO,EAAE,cAAc;QACvB,WAAW,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACxB,aAAa,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;KACzC;IACD,6DAA6D;IAC7D;QACE,KAAK,EAAE,oBAAoB;QAC3B,MAAM,EAAE,KAAK;QACb,OAAO,EAAE,YAAY;QACrB,WAAW,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACxB,aAAa,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;KACzC;IACD,iCAAiC;IACjC;QACE,KAAK,EAAE,yBAAyB;QAChC,MAAM,EAAE,KAAK;QACb,OAAO,EAAE,UAAU;QACnB,WAAW,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACxB,aAAa,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;KACzC;IACD,6DAA6D;IAC7D;QACE,KAAK,EAAE,oBAAoB;QAC3B,MAAM,EAAE,KAAK;QACb,OAAO,EAAE,YAAY;QACrB,WAAW,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACxB,aAAa,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;KACzC;CACF,CAAC;AAEF;;GAEG;AACH,MAAM,UAAU,aAAa,CAAC,QAAgB;IAC5C,iCAAiC;IACjC,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QACtC,IAAI,KAAK,EAAE,CAAC;YACV,OAAO;gBACL,QAAQ,EAAE,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC;gBAC9B,UAAU,EAAE,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC;gBAClC,MAAM,EAAE,CAAC,CAAC,MAAM;gBAChB,OAAO,EAAE,CAAC,CAAC,OAAO;aACnB,CAAC;QACJ,CAAC;IACH,CAAC;IAED,iFAAiF;IACjF,IAAI,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC7B,OAAO;YACL,QAAQ,EAAE,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC;YACzC,UAAU,EAAE,CAAC,CAAC,EAAE,oDAAoD;YACpE,MAAM,EAAE,KAAK;YACb,OAAO,EAAE,QAAQ;SAClB,CAAC;IACJ,CAAC;IAED,yEAAyE;IACzE,IAAI,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC7B,OAAO;YACL,QAAQ,EAAE,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC;YACzC,UAAU,EAAE,CAAC,CAAC;YACd,MAAM,EAAE,KAAK;YACb,OAAO,EAAE,QAAQ;SAClB,CAAC;IACJ,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,QAAgB;IAClD,OAAO,aAAa,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC;AAC1C,CAAC"}
|
||||
6
worker/dist/archive/hash.d.ts
vendored
6
worker/dist/archive/hash.d.ts
vendored
@@ -1,6 +0,0 @@
|
||||
/**
|
||||
* Compute SHA-256 hash of one or more files by streaming them in order.
|
||||
* Memory usage: O(1) — reads in 64KB chunks regardless of total size.
|
||||
* For multipart archives, pass all parts sorted by part number.
|
||||
*/
|
||||
export declare function hashParts(filePaths: string[]): Promise<string>;
|
||||
22
worker/dist/archive/hash.js
vendored
22
worker/dist/archive/hash.js
vendored
@@ -1,22 +0,0 @@
|
||||
import { createReadStream } from "fs";
|
||||
import { createHash } from "crypto";
|
||||
import { pipeline } from "stream/promises";
|
||||
import { PassThrough } from "stream";
|
||||
/**
|
||||
* Compute SHA-256 hash of one or more files by streaming them in order.
|
||||
* Memory usage: O(1) — reads in 64KB chunks regardless of total size.
|
||||
* For multipart archives, pass all parts sorted by part number.
|
||||
*/
|
||||
export async function hashParts(filePaths) {
|
||||
const hash = createHash("sha256");
|
||||
for (const filePath of filePaths) {
|
||||
await pipeline(createReadStream(filePath), new PassThrough({
|
||||
transform(chunk, _encoding, callback) {
|
||||
hash.update(chunk);
|
||||
callback();
|
||||
},
|
||||
}));
|
||||
}
|
||||
return hash.digest("hex");
|
||||
}
|
||||
//# sourceMappingURL=hash.js.map
|
||||
1
worker/dist/archive/hash.js.map
vendored
1
worker/dist/archive/hash.js.map
vendored
@@ -1 +0,0 @@
|
||||
{"version":3,"file":"hash.js","sourceRoot":"","sources":["../../src/archive/hash.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,IAAI,CAAC;AACtC,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAC3C,OAAO,EAAE,WAAW,EAAE,MAAM,QAAQ,CAAC;AAErC;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,SAAmB;IACjD,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IAClC,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,MAAM,QAAQ,CACZ,gBAAgB,CAAC,QAAQ,CAAC,EAC1B,IAAI,WAAW,CAAC;YACd,SAAS,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ;gBAClC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACnB,QAAQ,EAAE,CAAC;YACb,CAAC;SACF,CAAC,CACH,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC5B,CAAC"}
|
||||
19
worker/dist/archive/multipart.d.ts
vendored
19
worker/dist/archive/multipart.d.ts
vendored
@@ -1,19 +0,0 @@
|
||||
import { type ArchiveFormat } from "./detect.js";
|
||||
export interface TelegramMessage {
|
||||
id: bigint;
|
||||
fileName: string;
|
||||
fileId: string;
|
||||
fileSize: bigint;
|
||||
date: Date;
|
||||
}
|
||||
export interface ArchiveSet {
|
||||
type: ArchiveFormat;
|
||||
baseName: string;
|
||||
parts: TelegramMessage[];
|
||||
isMultipart: boolean;
|
||||
}
|
||||
/**
|
||||
* Group messages into archive sets (single files + multipart groups).
|
||||
* Messages should be pre-filtered to only include archive attachments.
|
||||
*/
|
||||
export declare function groupArchiveSets(messages: TelegramMessage[]): ArchiveSet[];
|
||||
74
worker/dist/archive/multipart.js
vendored
74
worker/dist/archive/multipart.js
vendored
@@ -1,74 +0,0 @@
|
||||
import { detectArchive } from "./detect.js";
|
||||
import { config } from "../util/config.js";
|
||||
import { childLogger } from "../util/logger.js";
|
||||
const log = childLogger("multipart");
|
||||
/**
|
||||
* Group messages into archive sets (single files + multipart groups).
|
||||
* Messages should be pre-filtered to only include archive attachments.
|
||||
*/
|
||||
export function groupArchiveSets(messages) {
|
||||
// Detect and annotate each message
|
||||
const annotated = [];
|
||||
for (const msg of messages) {
|
||||
const info = detectArchive(msg.fileName);
|
||||
if (info) {
|
||||
annotated.push({ msg, info });
|
||||
}
|
||||
}
|
||||
// Group by baseName + format
|
||||
const groups = new Map();
|
||||
for (const item of annotated) {
|
||||
const key = `${item.info.format}:${item.info.baseName.toLowerCase()}`;
|
||||
const group = groups.get(key) ?? [];
|
||||
group.push(item);
|
||||
groups.set(key, group);
|
||||
}
|
||||
const results = [];
|
||||
for (const [, group] of groups) {
|
||||
const format = group[0].info.format;
|
||||
const baseName = group[0].info.baseName;
|
||||
// Separate explicit multipart entries from potential singles
|
||||
const multipartEntries = group.filter((g) => g.info.pattern !== "SINGLE");
|
||||
const singleEntries = group.filter((g) => g.info.pattern === "SINGLE");
|
||||
if (multipartEntries.length > 0) {
|
||||
// This is a multipart set
|
||||
// Check if any single entry is the "final part" of a legacy split
|
||||
const allEntries = [...multipartEntries, ...singleEntries];
|
||||
// Check time span — skip if parts span too long (0 = no limit)
|
||||
if (config.multipartTimeoutHours > 0) {
|
||||
const dates = allEntries.map((e) => e.msg.date.getTime());
|
||||
const span = Math.max(...dates) - Math.min(...dates);
|
||||
const maxSpanMs = config.multipartTimeoutHours * 60 * 60 * 1000;
|
||||
if (span > maxSpanMs) {
|
||||
log.warn({ baseName, format, span: span / 3600000 }, "Multipart set spans too long, skipping");
|
||||
continue;
|
||||
}
|
||||
}
|
||||
// Sort by part number (singles get a very high number so they come last — they're the final part)
|
||||
allEntries.sort((a, b) => {
|
||||
const aNum = a.info.partNumber === -1 ? 999999 : a.info.partNumber;
|
||||
const bNum = b.info.partNumber === -1 ? 999999 : b.info.partNumber;
|
||||
return aNum - bNum;
|
||||
});
|
||||
results.push({
|
||||
type: format,
|
||||
baseName,
|
||||
parts: allEntries.map((e) => e.msg),
|
||||
isMultipart: true,
|
||||
});
|
||||
}
|
||||
else {
|
||||
// All entries are singles — each is its own archive set
|
||||
for (const entry of singleEntries) {
|
||||
results.push({
|
||||
type: format,
|
||||
baseName: entry.info.baseName,
|
||||
parts: [entry.msg],
|
||||
isMultipart: false,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
//# sourceMappingURL=multipart.js.map
|
||||
1
worker/dist/archive/multipart.js.map
vendored
1
worker/dist/archive/multipart.js.map
vendored
@@ -1 +0,0 @@
|
||||
{"version":3,"file":"multipart.js","sourceRoot":"","sources":["../../src/archive/multipart.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAA0C,MAAM,aAAa,CAAC;AACpF,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,MAAM,GAAG,GAAG,WAAW,CAAC,WAAW,CAAC,CAAC;AAiBrC;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,QAA2B;IAC1D,mCAAmC;IACnC,MAAM,SAAS,GAAoD,EAAE,CAAC;IACtE,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACzC,IAAI,IAAI,EAAE,CAAC;YACT,SAAS,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;IAED,6BAA6B;IAC7B,MAAM,MAAM,GAAG,IAAI,GAAG,EAA2D,CAAC;IAClF,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,EAAE,CAAC;QACtE,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACpC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjB,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACzB,CAAC;IAED,MAAM,OAAO,GAAiB,EAAE,CAAC;IAEjC,KAAK,MAAM,CAAC,EAAE,KAAK,CAAC,IAAI,MAAM,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;QACpC,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC;QAExC,6DAA6D;QAC7D,MAAM,gBAAgB,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC;QAC1E,MAAM,aAAa,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC;QAEvE,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChC,0BAA0B;YAC1B,kEAAkE;YAClE,MAAM,UAAU,GAAG,CAAC,GAAG,gBAAgB,EAAE,GAAG,aAAa,CAAC,CAAC;YAE3D,+DAA+D;YAC/D,IAAI,MAAM,CAAC,qBAAqB,GAAG,CAAC,EAAE,CAAC;gBACrC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;gBAC1D,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC;gBACrD,MAAM,SAAS,GAAG,MAAM,CAAC,qBAAqB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;gBAEhE,IAAI,IAAI,GAAG,SAAS,EAAE,CAAC;oBACrB,GAAG,CAAC,IAAI,CACN,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,GAAG,OAAO,EAAE,EAC1C,wCAAwC,CACzC,CAAC;oBACF,SAAS;gBACX,CAAC;YACH,CAAC;YAED,kGAAkG;YAClG,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;gBACvB,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,UAAU,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC;gBACnE,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,UAAU,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC;gBACnE,OAAO,IAAI,GAAG,IAAI,CAAC;YACrB,CAAC,CAAC,CAAC;YAEH,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,MAAM;gBACZ,QAAQ;gBACR,KAAK,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;gBACnC,WAAW,EAAE,IAAI;aAClB,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,wDAAwD;YACxD,KAAK,MAAM,KAAK,IAAI,aAAa,EAAE,CAAC;gBAClC,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,MAAM;oBACZ,QAAQ,EAAE,KAAK,CAAC,IAAI,CAAC,QAAQ;oBAC7B,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC;oBAClB,WAAW,EAAE,KAAK;iBACnB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
||||
6
worker/dist/archive/rar-reader.d.ts
vendored
6
worker/dist/archive/rar-reader.d.ts
vendored
@@ -1,6 +0,0 @@
|
||||
import type { FileEntry } from "./zip-reader.js";
|
||||
/**
|
||||
* Parse output of `unrar l -v <file>` to extract file metadata.
|
||||
* unrar automatically discovers sibling parts when they're co-located.
|
||||
*/
|
||||
export declare function readRarContents(firstPartPath: string): Promise<FileEntry[]>;
|
||||
77
worker/dist/archive/rar-reader.js
vendored
77
worker/dist/archive/rar-reader.js
vendored
@@ -1,77 +0,0 @@
|
||||
import { execFile } from "child_process";
|
||||
import { promisify } from "util";
|
||||
import path from "path";
|
||||
import { childLogger } from "../util/logger.js";
|
||||
const execFileAsync = promisify(execFile);
|
||||
const log = childLogger("rar-reader");
|
||||
/**
|
||||
* Parse output of `unrar l -v <file>` to extract file metadata.
|
||||
* unrar automatically discovers sibling parts when they're co-located.
|
||||
*/
|
||||
export async function readRarContents(firstPartPath) {
|
||||
try {
|
||||
const { stdout } = await execFileAsync("unrar", ["l", "-v", firstPartPath], {
|
||||
timeout: 30000,
|
||||
maxBuffer: 10 * 1024 * 1024, // 10MB for very large archives
|
||||
});
|
||||
return parseUnrarOutput(stdout);
|
||||
}
|
||||
catch (err) {
|
||||
log.warn({ err, file: firstPartPath }, "Failed to read RAR contents");
|
||||
return []; // Fallback: return empty on error
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Parse the tabular output of `unrar l -v`.
|
||||
*
|
||||
* Example output format:
|
||||
* Archive: test.rar
|
||||
* Details: RAR 5
|
||||
*
|
||||
* Attributes Size Packed Ratio Date Time CRC-32 Name
|
||||
* ----------- --------- --------- ----- -------- ----- -------- ----
|
||||
* ...A.... 12345 10234 83% 2024-01-15 10:30 DEADBEEF folder/file.stl
|
||||
* ----------- --------- --------- ----- -------- ----- -------- ----
|
||||
*/
|
||||
function parseUnrarOutput(output) {
|
||||
const entries = [];
|
||||
const lines = output.split("\n");
|
||||
let inFileList = false;
|
||||
let separatorCount = 0;
|
||||
for (const line of lines) {
|
||||
const trimmed = line.trim();
|
||||
// Detect separator lines (------- pattern)
|
||||
if (/^-{5,}/.test(trimmed)) {
|
||||
separatorCount++;
|
||||
if (separatorCount === 1) {
|
||||
inFileList = true;
|
||||
}
|
||||
else if (separatorCount >= 2) {
|
||||
inFileList = false;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (!inFileList)
|
||||
continue;
|
||||
// Parse file entry line
|
||||
// Format: Attributes Size Packed Ratio Date Time CRC Name
|
||||
const match = trimmed.match(/^\S+\s+(\d+)\s+(\d+)\s+\d+%\s+\S+\s+\S+\s+([0-9A-Fa-f]+)\s+(.+)$/);
|
||||
if (match) {
|
||||
const [, uncompressedStr, compressedStr, crc32, filePath] = match;
|
||||
// Skip directory entries (typically end with / or have size 0 with dir attributes)
|
||||
if (filePath.endsWith("/") || filePath.endsWith("\\"))
|
||||
continue;
|
||||
const ext = path.extname(filePath).toLowerCase();
|
||||
entries.push({
|
||||
path: filePath,
|
||||
fileName: path.basename(filePath),
|
||||
extension: ext ? ext.slice(1) : null,
|
||||
compressedSize: BigInt(compressedStr),
|
||||
uncompressedSize: BigInt(uncompressedStr),
|
||||
crc32: crc32.toLowerCase(),
|
||||
});
|
||||
}
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
//# sourceMappingURL=rar-reader.js.map
|
||||
1
worker/dist/archive/rar-reader.js.map
vendored
1
worker/dist/archive/rar-reader.js.map
vendored
@@ -1 +0,0 @@
|
||||
{"version":3,"file":"rar-reader.js","sourceRoot":"","sources":["../../src/archive/rar-reader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,MAAM,CAAC;AACjC,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAGhD,MAAM,aAAa,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;AAC1C,MAAM,GAAG,GAAG,WAAW,CAAC,YAAY,CAAC,CAAC;AAEtC;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,aAAqB;IAErB,IAAI,CAAC;QACH,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,aAAa,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,IAAI,EAAE,aAAa,CAAC,EAAE;YAC1E,OAAO,EAAE,KAAK;YACd,SAAS,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI,EAAE,+BAA+B;SAC7D,CAAC,CAAC;QAEH,OAAO,gBAAgB,CAAC,MAAM,CAAC,CAAC;IAClC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,aAAa,EAAE,EAAE,6BAA6B,CAAC,CAAC;QACtE,OAAO,EAAE,CAAC,CAAC,kCAAkC;IAC/C,CAAC;AACH,CAAC;AAED;;;;;;;;;;;GAWG;AACH,SAAS,gBAAgB,CAAC,MAAc;IACtC,MAAM,OAAO,GAAgB,EAAE,CAAC;IAChC,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAEjC,IAAI,UAAU,GAAG,KAAK,CAAC;IACvB,IAAI,cAAc,GAAG,CAAC,CAAC;IAEvB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAE5B,2CAA2C;QAC3C,IAAI,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAC3B,cAAc,EAAE,CAAC;YACjB,IAAI,cAAc,KAAK,CAAC,EAAE,CAAC;gBACzB,UAAU,GAAG,IAAI,CAAC;YACpB,CAAC;iBAAM,IAAI,cAAc,IAAI,CAAC,EAAE,CAAC;gBAC/B,UAAU,GAAG,KAAK,CAAC;YACrB,CAAC;YACD,SAAS;QACX,CAAC;QAED,IAAI,CAAC,UAAU;YAAE,SAAS;QAE1B,wBAAwB;QACxB,0DAA0D;QAC1D,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CACzB,kEAAkE,CACnE,CAAC;QAEF,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,CAAC,EAAE,eAAe,EAAE,aAAa,EAAE,KAAK,EAAE,QAAQ,CAAC,GAAG,KAAK,CAAC;YAElE,mFAAmF;YACnF,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC;gBAAE,SAAS;YAEhE,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;YACjD,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,QAAQ;gBACd,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBACjC,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI;gBACpC,cAAc,EAAE,MAAM,CAAC,aAAa,CAAC;gBACrC,gBAAgB,EAAE,MAAM,CAAC,eAAe,CAAC;gBACzC,KAAK,EAAE,KAAK,CAAC,WAAW,EAAE;aAC3B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
||||
11
worker/dist/archive/split.d.ts
vendored
11
worker/dist/archive/split.d.ts
vendored
@@ -1,11 +0,0 @@
|
||||
/**
|
||||
* Split a file into ≤2GB parts using byte-level splitting.
|
||||
* Returns paths to the split parts. If the file is already ≤2GB, returns the original path.
|
||||
*/
|
||||
export declare function byteLevelSplit(filePath: string): Promise<string[]>;
|
||||
/**
|
||||
* Concatenate multiple files into a single output file by streaming
|
||||
* each input sequentially. Used for repacking multipart archives
|
||||
* that have oversized parts (>2GB) before re-splitting.
|
||||
*/
|
||||
export declare function concatenateFiles(inputPaths: string[], outputPath: string): Promise<void>;
|
||||
55
worker/dist/archive/split.js
vendored
55
worker/dist/archive/split.js
vendored
@@ -1,55 +0,0 @@
|
||||
import { createReadStream, createWriteStream } from "fs";
|
||||
import { stat } from "fs/promises";
|
||||
import path from "path";
|
||||
import { pipeline } from "stream/promises";
|
||||
import { childLogger } from "../util/logger.js";
|
||||
const log = childLogger("split");
|
||||
/** 2GB in bytes — Telegram's file size limit */
|
||||
const MAX_PART_SIZE = 2n * 1024n * 1024n * 1024n;
|
||||
/**
|
||||
* Split a file into ≤2GB parts using byte-level splitting.
|
||||
* Returns paths to the split parts. If the file is already ≤2GB, returns the original path.
|
||||
*/
|
||||
export async function byteLevelSplit(filePath) {
|
||||
const stats = await stat(filePath);
|
||||
const fileSize = BigInt(stats.size);
|
||||
if (fileSize <= MAX_PART_SIZE) {
|
||||
return [filePath];
|
||||
}
|
||||
const dir = path.dirname(filePath);
|
||||
const baseName = path.basename(filePath);
|
||||
const partSize = Number(MAX_PART_SIZE);
|
||||
const totalParts = Math.ceil(Number(fileSize) / partSize);
|
||||
const parts = [];
|
||||
log.info({ filePath, fileSize: Number(fileSize), totalParts }, "Splitting file");
|
||||
for (let i = 0; i < totalParts; i++) {
|
||||
const partNum = String(i + 1).padStart(3, "0");
|
||||
const partPath = path.join(dir, `${baseName}.${partNum}`);
|
||||
const start = i * partSize;
|
||||
const end = Math.min(start + partSize - 1, Number(fileSize) - 1);
|
||||
await pipeline(createReadStream(filePath, { start, end }), createWriteStream(partPath));
|
||||
parts.push(partPath);
|
||||
}
|
||||
log.info({ filePath, parts: parts.length }, "File split complete");
|
||||
return parts;
|
||||
}
|
||||
/**
|
||||
* Concatenate multiple files into a single output file by streaming
|
||||
* each input sequentially. Used for repacking multipart archives
|
||||
* that have oversized parts (>2GB) before re-splitting.
|
||||
*/
|
||||
export async function concatenateFiles(inputPaths, outputPath) {
|
||||
const out = createWriteStream(outputPath);
|
||||
for (let i = 0; i < inputPaths.length; i++) {
|
||||
log.info({ part: i + 1, total: inputPaths.length, file: path.basename(inputPaths[i]) }, "Concatenating part");
|
||||
await pipeline(createReadStream(inputPaths[i]), out, { end: false });
|
||||
}
|
||||
// Close the output stream
|
||||
await new Promise((resolve, reject) => {
|
||||
out.end(() => resolve());
|
||||
out.on("error", reject);
|
||||
});
|
||||
const stats = await stat(outputPath);
|
||||
log.info({ outputPath, totalBytes: stats.size, parts: inputPaths.length }, "Concatenation complete");
|
||||
}
|
||||
//# sourceMappingURL=split.js.map
|
||||
1
worker/dist/archive/split.js.map
vendored
1
worker/dist/archive/split.js.map
vendored
@@ -1 +0,0 @@
|
||||
{"version":3,"file":"split.js","sourceRoot":"","sources":["../../src/archive/split.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,IAAI,CAAC;AACzD,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AACnC,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAC3C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,MAAM,GAAG,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;AAEjC,gDAAgD;AAChD,MAAM,aAAa,GAAG,EAAE,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,CAAC;AAEjD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,QAAgB;IACnD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;IACnC,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAEpC,IAAI,QAAQ,IAAI,aAAa,EAAE,CAAC;QAC9B,OAAO,CAAC,QAAQ,CAAC,CAAC;IACpB,CAAC;IAED,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACzC,MAAM,QAAQ,GAAG,MAAM,CAAC,aAAa,CAAC,CAAC;IACvC,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,QAAQ,CAAC,CAAC;IAC1D,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,GAAG,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC,EAAE,UAAU,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAEjF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,MAAM,OAAO,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAC/C,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,IAAI,OAAO,EAAE,CAAC,CAAC;QAC1D,MAAM,KAAK,GAAG,CAAC,GAAG,QAAQ,CAAC;QAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,QAAQ,GAAG,CAAC,EAAE,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;QAEjE,MAAM,QAAQ,CACZ,gBAAgB,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,EAC1C,iBAAiB,CAAC,QAAQ,CAAC,CAC5B,CAAC;QAEF,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACvB,CAAC;IAED,GAAG,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,EAAE,qBAAqB,CAAC,CAAC;IACnE,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,UAAoB,EACpB,UAAkB;IAElB,MAAM,GAAG,GAAG,iBAAiB,CAAC,UAAU,CAAC,CAAC;IAE1C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3C,GAAG,CAAC,IAAI,CACN,EAAE,IAAI,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,UAAU,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,EAC7E,oBAAoB,CACrB,CAAC;QACF,MAAM,QAAQ,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC;IACvE,CAAC;IAED,0BAA0B;IAC1B,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC1C,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;QACzB,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAC1B,CAAC,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC;IACrC,GAAG,CAAC,IAAI,CACN,EAAE,UAAU,EAAE,UAAU,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,UAAU,CAAC,MAAM,EAAE,EAChE,wBAAwB,CACzB,CAAC;AACJ,CAAC"}
|
||||
15
worker/dist/archive/zip-reader.d.ts
vendored
15
worker/dist/archive/zip-reader.d.ts
vendored
@@ -1,15 +0,0 @@
|
||||
export interface FileEntry {
|
||||
path: string;
|
||||
fileName: string;
|
||||
extension: string | null;
|
||||
compressedSize: bigint;
|
||||
uncompressedSize: bigint;
|
||||
crc32: string | null;
|
||||
}
|
||||
/**
|
||||
* Read the central directory of a ZIP file without extracting any contents.
|
||||
* For multipart ZIPs (.zip.001, .zip.002 etc.), uses a custom random-access
|
||||
* reader that spans all parts seamlessly so yauzl can find the central
|
||||
* directory at the end of the combined data.
|
||||
*/
|
||||
export declare function readZipCentralDirectory(filePaths: string[]): Promise<FileEntry[]>;
|
||||
161
worker/dist/archive/zip-reader.js
vendored
161
worker/dist/archive/zip-reader.js
vendored
@@ -1,161 +0,0 @@
|
||||
import yauzl from "yauzl";
|
||||
import { open as fsOpen, stat as fsStat } from "fs/promises";
|
||||
import path from "path";
|
||||
import { Readable } from "stream";
|
||||
import { childLogger } from "../util/logger.js";
|
||||
const log = childLogger("zip-reader");
|
||||
/**
|
||||
* Read the central directory of a ZIP file without extracting any contents.
|
||||
* For multipart ZIPs (.zip.001, .zip.002 etc.), uses a custom random-access
|
||||
* reader that spans all parts seamlessly so yauzl can find the central
|
||||
* directory at the end of the combined data.
|
||||
*/
|
||||
export async function readZipCentralDirectory(filePaths) {
|
||||
if (filePaths.length === 1) {
|
||||
return readSingleZip(filePaths[0]);
|
||||
}
|
||||
// Multipart: use a spanning random-access reader
|
||||
return readMultipartZip(filePaths);
|
||||
}
|
||||
/** Read a single (non-split) ZIP file. */
|
||||
function readSingleZip(targetFile) {
|
||||
return new Promise((resolve) => {
|
||||
yauzl.open(targetFile, { lazyEntries: true, autoClose: true }, (err, zipFile) => {
|
||||
if (err) {
|
||||
log.warn({ err, file: targetFile }, "Failed to open ZIP for reading");
|
||||
resolve([]);
|
||||
return;
|
||||
}
|
||||
const entries = [];
|
||||
zipFile.readEntry();
|
||||
zipFile.on("entry", (entry) => {
|
||||
if (!entry.fileName.endsWith("/")) {
|
||||
const ext = path.extname(entry.fileName).toLowerCase();
|
||||
entries.push({
|
||||
path: entry.fileName,
|
||||
fileName: path.basename(entry.fileName),
|
||||
extension: ext ? ext.slice(1) : null,
|
||||
compressedSize: BigInt(entry.compressedSize),
|
||||
uncompressedSize: BigInt(entry.uncompressedSize),
|
||||
crc32: entry.crc32 !== 0 ? entry.crc32.toString(16).padStart(8, "0") : null,
|
||||
});
|
||||
}
|
||||
zipFile.readEntry();
|
||||
});
|
||||
zipFile.on("end", () => resolve(entries));
|
||||
zipFile.on("error", (error) => {
|
||||
log.warn({ error, file: targetFile }, "Error reading ZIP entries");
|
||||
resolve(entries);
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Read a multipart split ZIP using yauzl's RandomAccessReader API.
|
||||
* This creates a virtual "file" that spans all parts so yauzl can
|
||||
* seek freely across the entire archive to read the central directory.
|
||||
*/
|
||||
async function readMultipartZip(filePaths) {
|
||||
// Get sizes of all parts
|
||||
const partSizes = [];
|
||||
for (const fp of filePaths) {
|
||||
const s = await fsStat(fp);
|
||||
partSizes.push(s.size);
|
||||
}
|
||||
const totalSize = partSizes.reduce((a, b) => a + b, 0);
|
||||
log.debug({ parts: filePaths.length, totalSize }, "Reading multipart ZIP via spanning reader");
|
||||
return new Promise((resolve) => {
|
||||
const reader = createMultiPartReader(filePaths, partSizes);
|
||||
yauzl.fromRandomAccessReader(reader, totalSize, { lazyEntries: true, autoClose: true }, (err, zipFile) => {
|
||||
if (err) {
|
||||
log.warn({ err }, "Failed to open multipart ZIP for reading");
|
||||
reader.close(() => { });
|
||||
resolve([]);
|
||||
return;
|
||||
}
|
||||
const entries = [];
|
||||
zipFile.readEntry();
|
||||
zipFile.on("entry", (entry) => {
|
||||
if (!entry.fileName.endsWith("/")) {
|
||||
const ext = path.extname(entry.fileName).toLowerCase();
|
||||
entries.push({
|
||||
path: entry.fileName,
|
||||
fileName: path.basename(entry.fileName),
|
||||
extension: ext ? ext.slice(1) : null,
|
||||
compressedSize: BigInt(entry.compressedSize),
|
||||
uncompressedSize: BigInt(entry.uncompressedSize),
|
||||
crc32: entry.crc32 !== 0 ? entry.crc32.toString(16).padStart(8, "0") : null,
|
||||
});
|
||||
}
|
||||
zipFile.readEntry();
|
||||
});
|
||||
zipFile.on("end", () => {
|
||||
log.info({ entries: entries.length }, "Multipart ZIP entries read");
|
||||
resolve(entries);
|
||||
});
|
||||
zipFile.on("error", (error) => {
|
||||
log.warn({ error }, "Error reading multipart ZIP entries");
|
||||
resolve(entries);
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Create a yauzl RandomAccessReader that reads across multiple split part files.
|
||||
* Maps a global offset to the correct part file and local offset.
|
||||
*
|
||||
* Uses Object.create to properly inherit from yauzl.RandomAccessReader
|
||||
* (whose constructor + prototype is defined at runtime, not as a TS class).
|
||||
*/
|
||||
function createMultiPartReader(filePaths, partSizes) {
|
||||
// Build cumulative offset table
|
||||
const partOffsets = [];
|
||||
let offset = 0;
|
||||
for (const size of partSizes) {
|
||||
partOffsets.push(offset);
|
||||
offset += size;
|
||||
}
|
||||
// Create an instance by calling the parent constructor
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const reader = new yauzl.RandomAccessReader();
|
||||
// Override _readStreamForRange — yauzl calls this to read a range of bytes
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
reader._readStreamForRange = function (start, end) {
|
||||
const readable = new Readable({ read() { } });
|
||||
readRange(start, end, readable).catch((err) => {
|
||||
readable.destroy(err);
|
||||
});
|
||||
return readable;
|
||||
};
|
||||
async function readRange(start, end, readable) {
|
||||
let remaining = end - start;
|
||||
let globalOffset = start;
|
||||
while (remaining > 0) {
|
||||
// Find which part this offset falls in
|
||||
let partIdx = partOffsets.length - 1;
|
||||
for (let i = 0; i < partOffsets.length; i++) {
|
||||
if (i + 1 < partOffsets.length && globalOffset < partOffsets[i + 1]) {
|
||||
partIdx = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
const localOffset = globalOffset - partOffsets[partIdx];
|
||||
const partRemaining = partSizes[partIdx] - localOffset;
|
||||
const toRead = Math.min(remaining, partRemaining);
|
||||
const fh = await fsOpen(filePaths[partIdx], "r");
|
||||
try {
|
||||
const buf = Buffer.alloc(toRead);
|
||||
const { bytesRead } = await fh.read(buf, 0, toRead, localOffset);
|
||||
readable.push(buf.subarray(0, bytesRead));
|
||||
remaining -= bytesRead;
|
||||
globalOffset += bytesRead;
|
||||
}
|
||||
finally {
|
||||
await fh.close();
|
||||
}
|
||||
}
|
||||
readable.push(null); // Signal end of stream
|
||||
}
|
||||
return reader;
|
||||
}
|
||||
//# sourceMappingURL=zip-reader.js.map
|
||||
1
worker/dist/archive/zip-reader.js.map
vendored
1
worker/dist/archive/zip-reader.js.map
vendored
File diff suppressed because one or more lines are too long
7
worker/dist/db/client.d.ts
vendored
7
worker/dist/db/client.d.ts
vendored
@@ -1,7 +0,0 @@
|
||||
import { PrismaClient } from "@prisma/client";
|
||||
import { PrismaPg } from "@prisma/adapter-pg";
|
||||
declare const pool: import("pg").Pool;
|
||||
export declare const db: PrismaClient<{
|
||||
adapter: PrismaPg;
|
||||
}, never, import("@prisma/client/runtime/client").DefaultArgs>;
|
||||
export { pool };
|
||||
12
worker/dist/db/client.js
vendored
12
worker/dist/db/client.js
vendored
@@ -1,12 +0,0 @@
|
||||
import { PrismaClient } from "@prisma/client";
|
||||
import { PrismaPg } from "@prisma/adapter-pg";
|
||||
import pg from "pg";
|
||||
import { config } from "../util/config.js";
|
||||
const pool = new pg.Pool({
|
||||
connectionString: config.databaseUrl,
|
||||
max: 5,
|
||||
});
|
||||
const adapter = new PrismaPg(pool);
|
||||
export const db = new PrismaClient({ adapter });
|
||||
export { pool };
|
||||
//# sourceMappingURL=client.js.map
|
||||
1
worker/dist/db/client.js.map
vendored
1
worker/dist/db/client.js.map
vendored
@@ -1 +0,0 @@
|
||||
{"version":3,"file":"client.js","sourceRoot":"","sources":["../../src/db/client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,MAAM,IAAI,GAAG,IAAI,EAAE,CAAC,IAAI,CAAC;IACvB,gBAAgB,EAAE,MAAM,CAAC,WAAW;IACpC,GAAG,EAAE,CAAC;CACP,CAAC,CAAC;AAEH,MAAM,OAAO,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC;AACnC,MAAM,CAAC,MAAM,EAAE,GAAG,IAAI,YAAY,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC;AAEhD,OAAO,EAAE,IAAI,EAAE,CAAC"}
|
||||
9
worker/dist/db/locks.d.ts
vendored
9
worker/dist/db/locks.d.ts
vendored
@@ -1,9 +0,0 @@
|
||||
/**
|
||||
* Try to acquire a PostgreSQL advisory lock for an account.
|
||||
* Returns true if acquired, false if already held by another session.
|
||||
*/
|
||||
export declare function tryAcquireLock(accountId: string): Promise<boolean>;
|
||||
/**
|
||||
* Release the advisory lock for an account.
|
||||
*/
|
||||
export declare function releaseLock(accountId: string): Promise<void>;
|
||||
53
worker/dist/db/locks.js
vendored
53
worker/dist/db/locks.js
vendored
@@ -1,53 +0,0 @@
|
||||
import { pool } from "./client.js";
|
||||
import { childLogger } from "../util/logger.js";
|
||||
const log = childLogger("locks");
|
||||
/**
|
||||
* Derive a stable 32-bit integer lock ID from an account ID string.
|
||||
* PostgreSQL advisory locks use bigint, but we use 32-bit for safety.
|
||||
*/
|
||||
function hashToLockId(accountId) {
|
||||
let hash = 0;
|
||||
for (let i = 0; i < accountId.length; i++) {
|
||||
const char = accountId.charCodeAt(i);
|
||||
hash = (hash << 5) - hash + char;
|
||||
hash |= 0; // Convert to 32-bit integer
|
||||
}
|
||||
return Math.abs(hash);
|
||||
}
|
||||
/**
|
||||
* Try to acquire a PostgreSQL advisory lock for an account.
|
||||
* Returns true if acquired, false if already held by another session.
|
||||
*/
|
||||
export async function tryAcquireLock(accountId) {
|
||||
const lockId = hashToLockId(accountId);
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
const result = await client.query("SELECT pg_try_advisory_lock($1)", [lockId]);
|
||||
const acquired = result.rows[0]?.pg_try_advisory_lock ?? false;
|
||||
if (acquired) {
|
||||
log.debug({ accountId, lockId }, "Advisory lock acquired");
|
||||
}
|
||||
else {
|
||||
log.debug({ accountId, lockId }, "Advisory lock already held");
|
||||
}
|
||||
return acquired;
|
||||
}
|
||||
finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Release the advisory lock for an account.
|
||||
*/
|
||||
export async function releaseLock(accountId) {
|
||||
const lockId = hashToLockId(accountId);
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
await client.query("SELECT pg_advisory_unlock($1)", [lockId]);
|
||||
log.debug({ accountId, lockId }, "Advisory lock released");
|
||||
}
|
||||
finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=locks.js.map
|
||||
1
worker/dist/db/locks.js.map
vendored
1
worker/dist/db/locks.js.map
vendored
@@ -1 +0,0 @@
|
||||
{"version":3,"file":"locks.js","sourceRoot":"","sources":["../../src/db/locks.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AACnC,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,MAAM,GAAG,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;AAEjC;;;GAGG;AACH,SAAS,YAAY,CAAC,SAAiB;IACrC,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,MAAM,IAAI,GAAG,SAAS,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QACrC,IAAI,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;QACjC,IAAI,IAAI,CAAC,CAAC,CAAC,4BAA4B;IACzC,CAAC;IACD,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;AACxB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,SAAiB;IACpD,MAAM,MAAM,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC;IACvC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;IACpC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,KAAK,CAC/B,iCAAiC,EACjC,CAAC,MAAM,CAAC,CACT,CAAC;QACF,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,oBAAoB,IAAI,KAAK,CAAC;QAC/D,IAAI,QAAQ,EAAE,CAAC;YACb,GAAG,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,wBAAwB,CAAC,CAAC;QAC7D,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,4BAA4B,CAAC,CAAC;QACjE,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;YAAS,CAAC;QACT,MAAM,CAAC,OAAO,EAAE,CAAC;IACnB,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,SAAiB;IACjD,MAAM,MAAM,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC;IACvC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;IACpC,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,KAAK,CAAC,+BAA+B,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC;QAC9D,GAAG,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,wBAAwB,CAAC,CAAC;IAC7D,CAAC;YAAS,CAAC;QACT,MAAM,CAAC,OAAO,EAAE,CAAC;IACnB,CAAC;AACH,CAAC"}
|
||||
356
worker/dist/db/queries.d.ts
vendored
356
worker/dist/db/queries.d.ts
vendored
@@ -1,356 +0,0 @@
|
||||
import type { ArchiveType, FetchStatus } from "@prisma/client";
|
||||
export declare function getActiveAccounts(): Promise<{
|
||||
id: string;
|
||||
phone: string;
|
||||
displayName: string | null;
|
||||
isActive: boolean;
|
||||
authState: import("@prisma/client").$Enums.AuthState;
|
||||
authCode: string | null;
|
||||
lastSeenAt: Date | null;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
}[]>;
|
||||
export declare function getPendingAccounts(): Promise<{
|
||||
id: string;
|
||||
phone: string;
|
||||
displayName: string | null;
|
||||
isActive: boolean;
|
||||
authState: import("@prisma/client").$Enums.AuthState;
|
||||
authCode: string | null;
|
||||
lastSeenAt: Date | null;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
}[]>;
|
||||
export declare function hasAnyChannels(): Promise<boolean>;
|
||||
export declare function getSourceChannelMappings(accountId: string): Promise<({
|
||||
channel: {
|
||||
id: string;
|
||||
isActive: boolean;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
telegramId: bigint;
|
||||
title: string;
|
||||
type: import("@prisma/client").$Enums.ChannelType;
|
||||
isForum: boolean;
|
||||
};
|
||||
} & {
|
||||
accountId: string;
|
||||
id: string;
|
||||
createdAt: Date;
|
||||
channelId: string;
|
||||
role: import("@prisma/client").$Enums.ChannelRole;
|
||||
lastProcessedMessageId: bigint | null;
|
||||
})[]>;
|
||||
export declare function getGlobalDestinationChannel(): Promise<{
|
||||
id: string;
|
||||
isActive: boolean;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
telegramId: bigint;
|
||||
title: string;
|
||||
type: import("@prisma/client").$Enums.ChannelType;
|
||||
isForum: boolean;
|
||||
} | null>;
|
||||
export declare function getGlobalSetting(key: string): Promise<string | null>;
|
||||
export declare function setGlobalSetting(key: string, value: string): Promise<{
|
||||
updatedAt: Date;
|
||||
key: string;
|
||||
value: string;
|
||||
}>;
|
||||
export declare function packageExistsByHash(contentHash: string): Promise<boolean>;
|
||||
/**
|
||||
* Check if a package already exists for a given source message ID
|
||||
* AND was successfully uploaded to the destination (destMessageId is set).
|
||||
* Used as an early skip before downloading.
|
||||
*/
|
||||
export declare function packageExistsBySourceMessage(sourceChannelId: string, sourceMessageId: bigint): Promise<boolean>;
|
||||
/**
|
||||
* Delete orphaned Package rows that have the same content hash but never
|
||||
* completed the upload (destMessageId is null). Called before creating a
|
||||
* new complete record to avoid unique constraint violations.
|
||||
*/
|
||||
export declare function deleteOrphanedPackageByHash(contentHash: string): Promise<void>;
|
||||
export interface CreatePackageInput {
|
||||
contentHash: string;
|
||||
fileName: string;
|
||||
fileSize: bigint;
|
||||
archiveType: ArchiveType;
|
||||
sourceChannelId: string;
|
||||
sourceMessageId: bigint;
|
||||
sourceTopicId?: bigint | null;
|
||||
destChannelId?: string;
|
||||
destMessageId?: bigint;
|
||||
isMultipart: boolean;
|
||||
partCount: number;
|
||||
ingestionRunId: string;
|
||||
creator?: string | null;
|
||||
previewData?: Buffer | null;
|
||||
previewMsgId?: bigint | null;
|
||||
files: {
|
||||
path: string;
|
||||
fileName: string;
|
||||
extension: string | null;
|
||||
compressedSize: bigint;
|
||||
uncompressedSize: bigint;
|
||||
crc32: string | null;
|
||||
}[];
|
||||
}
|
||||
export declare function createPackageWithFiles(input: CreatePackageInput): Promise<{
|
||||
id: string;
|
||||
createdAt: Date;
|
||||
contentHash: string;
|
||||
fileName: string;
|
||||
fileSize: bigint;
|
||||
archiveType: import("@prisma/client").$Enums.ArchiveType;
|
||||
creator: string | null;
|
||||
sourceChannelId: string;
|
||||
sourceMessageId: bigint;
|
||||
sourceTopicId: bigint | null;
|
||||
destChannelId: string | null;
|
||||
destMessageId: bigint | null;
|
||||
isMultipart: boolean;
|
||||
partCount: number;
|
||||
fileCount: number;
|
||||
previewData: import("@prisma/client/runtime/client").Bytes | null;
|
||||
previewMsgId: bigint | null;
|
||||
indexedAt: Date;
|
||||
ingestionRunId: string | null;
|
||||
}>;
|
||||
export declare function createIngestionRun(accountId: string): Promise<{
|
||||
accountId: string;
|
||||
id: string;
|
||||
status: import("@prisma/client").$Enums.IngestionStatus;
|
||||
startedAt: Date;
|
||||
finishedAt: Date | null;
|
||||
messagesScanned: number;
|
||||
zipsFound: number;
|
||||
zipsDuplicate: number;
|
||||
zipsIngested: number;
|
||||
errorMessage: string | null;
|
||||
currentActivity: string | null;
|
||||
currentStep: string | null;
|
||||
currentChannel: string | null;
|
||||
currentFile: string | null;
|
||||
currentFileNum: number | null;
|
||||
totalFiles: number | null;
|
||||
downloadedBytes: bigint | null;
|
||||
totalBytes: bigint | null;
|
||||
downloadPercent: number | null;
|
||||
lastActivityAt: Date | null;
|
||||
}>;
|
||||
export interface ActivityUpdate {
|
||||
currentActivity: string;
|
||||
currentStep: string;
|
||||
currentChannel?: string | null;
|
||||
currentFile?: string | null;
|
||||
currentFileNum?: number | null;
|
||||
totalFiles?: number | null;
|
||||
downloadedBytes?: bigint | null;
|
||||
totalBytes?: bigint | null;
|
||||
downloadPercent?: number | null;
|
||||
messagesScanned?: number;
|
||||
zipsFound?: number;
|
||||
zipsDuplicate?: number;
|
||||
zipsIngested?: number;
|
||||
}
|
||||
export declare function updateRunActivity(runId: string, activity: ActivityUpdate): Promise<{
|
||||
accountId: string;
|
||||
id: string;
|
||||
status: import("@prisma/client").$Enums.IngestionStatus;
|
||||
startedAt: Date;
|
||||
finishedAt: Date | null;
|
||||
messagesScanned: number;
|
||||
zipsFound: number;
|
||||
zipsDuplicate: number;
|
||||
zipsIngested: number;
|
||||
errorMessage: string | null;
|
||||
currentActivity: string | null;
|
||||
currentStep: string | null;
|
||||
currentChannel: string | null;
|
||||
currentFile: string | null;
|
||||
currentFileNum: number | null;
|
||||
totalFiles: number | null;
|
||||
downloadedBytes: bigint | null;
|
||||
totalBytes: bigint | null;
|
||||
downloadPercent: number | null;
|
||||
lastActivityAt: Date | null;
|
||||
}>;
|
||||
export declare function completeIngestionRun(runId: string, counters: {
|
||||
messagesScanned: number;
|
||||
zipsFound: number;
|
||||
zipsDuplicate: number;
|
||||
zipsIngested: number;
|
||||
}): Promise<{
|
||||
accountId: string;
|
||||
id: string;
|
||||
status: import("@prisma/client").$Enums.IngestionStatus;
|
||||
startedAt: Date;
|
||||
finishedAt: Date | null;
|
||||
messagesScanned: number;
|
||||
zipsFound: number;
|
||||
zipsDuplicate: number;
|
||||
zipsIngested: number;
|
||||
errorMessage: string | null;
|
||||
currentActivity: string | null;
|
||||
currentStep: string | null;
|
||||
currentChannel: string | null;
|
||||
currentFile: string | null;
|
||||
currentFileNum: number | null;
|
||||
totalFiles: number | null;
|
||||
downloadedBytes: bigint | null;
|
||||
totalBytes: bigint | null;
|
||||
downloadPercent: number | null;
|
||||
lastActivityAt: Date | null;
|
||||
}>;
|
||||
export declare function failIngestionRun(runId: string, errorMessage: string): Promise<{
|
||||
accountId: string;
|
||||
id: string;
|
||||
status: import("@prisma/client").$Enums.IngestionStatus;
|
||||
startedAt: Date;
|
||||
finishedAt: Date | null;
|
||||
messagesScanned: number;
|
||||
zipsFound: number;
|
||||
zipsDuplicate: number;
|
||||
zipsIngested: number;
|
||||
errorMessage: string | null;
|
||||
currentActivity: string | null;
|
||||
currentStep: string | null;
|
||||
currentChannel: string | null;
|
||||
currentFile: string | null;
|
||||
currentFileNum: number | null;
|
||||
totalFiles: number | null;
|
||||
downloadedBytes: bigint | null;
|
||||
totalBytes: bigint | null;
|
||||
downloadPercent: number | null;
|
||||
lastActivityAt: Date | null;
|
||||
}>;
|
||||
export declare function updateLastProcessedMessage(mappingId: string, messageId: bigint): Promise<{
|
||||
accountId: string;
|
||||
id: string;
|
||||
createdAt: Date;
|
||||
channelId: string;
|
||||
role: import("@prisma/client").$Enums.ChannelRole;
|
||||
lastProcessedMessageId: bigint | null;
|
||||
}>;
|
||||
export declare function markStaleRunsAsFailed(): Promise<import("@prisma/client").Prisma.BatchPayload>;
|
||||
export declare function updateAccountAuthState(accountId: string, authState: "PENDING" | "AWAITING_CODE" | "AWAITING_PASSWORD" | "AUTHENTICATED" | "EXPIRED", authCode?: string | null): Promise<{
|
||||
id: string;
|
||||
phone: string;
|
||||
displayName: string | null;
|
||||
isActive: boolean;
|
||||
authState: import("@prisma/client").$Enums.AuthState;
|
||||
authCode: string | null;
|
||||
lastSeenAt: Date | null;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
}>;
|
||||
export declare function getAccountAuthCode(accountId: string): Promise<{
|
||||
authState: import("@prisma/client").$Enums.AuthState;
|
||||
authCode: string | null;
|
||||
} | null>;
|
||||
export interface UpsertChannelInput {
|
||||
telegramId: bigint;
|
||||
title: string;
|
||||
type: "SOURCE" | "DESTINATION";
|
||||
isForum: boolean;
|
||||
isActive?: boolean;
|
||||
}
|
||||
/**
|
||||
* Upsert a channel by telegramId. Returns the channel record.
|
||||
* If it already exists, update title and forum status.
|
||||
* New channels default to disabled (isActive: false) so the admin must
|
||||
* explicitly enable them before the worker processes them.
|
||||
* Pass isActive: true for DESTINATION channels that must be active immediately.
|
||||
*/
|
||||
export declare function upsertChannel(input: UpsertChannelInput): Promise<{
|
||||
id: string;
|
||||
isActive: boolean;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
telegramId: bigint;
|
||||
title: string;
|
||||
type: import("@prisma/client").$Enums.ChannelType;
|
||||
isForum: boolean;
|
||||
}>;
|
||||
/**
|
||||
* Link an account to a channel if not already linked.
|
||||
* Uses a try/catch on unique constraint to make it idempotent.
|
||||
*/
|
||||
export declare function ensureAccountChannelLink(accountId: string, channelId: string, role: "READER" | "WRITER"): Promise<{
|
||||
accountId: string;
|
||||
id: string;
|
||||
createdAt: Date;
|
||||
channelId: string;
|
||||
role: import("@prisma/client").$Enums.ChannelRole;
|
||||
lastProcessedMessageId: bigint | null;
|
||||
} | null>;
|
||||
export declare function setChannelForum(channelId: string, isForum: boolean): Promise<{
|
||||
id: string;
|
||||
isActive: boolean;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
telegramId: bigint;
|
||||
title: string;
|
||||
type: import("@prisma/client").$Enums.ChannelType;
|
||||
isForum: boolean;
|
||||
}>;
|
||||
export declare function getTopicProgress(mappingId: string): Promise<{
|
||||
id: string;
|
||||
lastProcessedMessageId: bigint | null;
|
||||
accountChannelMapId: string;
|
||||
topicId: bigint;
|
||||
topicName: string | null;
|
||||
}[]>;
|
||||
export declare function upsertTopicProgress(mappingId: string, topicId: bigint, topicName: string | null, lastProcessedMessageId: bigint): Promise<{
|
||||
id: string;
|
||||
lastProcessedMessageId: bigint | null;
|
||||
accountChannelMapId: string;
|
||||
topicId: bigint;
|
||||
topicName: string | null;
|
||||
}>;
|
||||
export declare function getChannelFetchRequest(requestId: string): Promise<({
|
||||
account: {
|
||||
id: string;
|
||||
phone: string;
|
||||
displayName: string | null;
|
||||
isActive: boolean;
|
||||
authState: import("@prisma/client").$Enums.AuthState;
|
||||
authCode: string | null;
|
||||
lastSeenAt: Date | null;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
};
|
||||
} & {
|
||||
error: string | null;
|
||||
accountId: string;
|
||||
id: string;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
status: import("@prisma/client").$Enums.FetchStatus;
|
||||
resultJson: string | null;
|
||||
}) | null>;
|
||||
export declare function updateFetchRequestStatus(requestId: string, status: FetchStatus, extra?: {
|
||||
resultJson?: string;
|
||||
error?: string;
|
||||
}): Promise<{
|
||||
error: string | null;
|
||||
accountId: string;
|
||||
id: string;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
status: import("@prisma/client").$Enums.FetchStatus;
|
||||
resultJson: string | null;
|
||||
}>;
|
||||
export declare function getAccountLinkedChannelIds(accountId: string): Promise<Set<string>>;
|
||||
export declare function getExistingChannelsByTelegramId(): Promise<Map<string, string>>;
|
||||
export declare function getAccountById(accountId: string): Promise<{
|
||||
id: string;
|
||||
phone: string;
|
||||
displayName: string | null;
|
||||
isActive: boolean;
|
||||
authState: import("@prisma/client").$Enums.AuthState;
|
||||
authCode: string | null;
|
||||
lastSeenAt: Date | null;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
} | null>;
|
||||
319
worker/dist/db/queries.js
vendored
319
worker/dist/db/queries.js
vendored
@@ -1,319 +0,0 @@
|
||||
import { db } from "./client.js";
|
||||
export async function getActiveAccounts() {
|
||||
return db.telegramAccount.findMany({
|
||||
where: { isActive: true, authState: "AUTHENTICATED" },
|
||||
});
|
||||
}
|
||||
export async function getPendingAccounts() {
|
||||
return db.telegramAccount.findMany({
|
||||
where: { isActive: true, authState: "PENDING" },
|
||||
});
|
||||
}
|
||||
export async function hasAnyChannels() {
|
||||
const count = await db.telegramChannel.count();
|
||||
return count > 0;
|
||||
}
|
||||
export async function getSourceChannelMappings(accountId) {
|
||||
return db.accountChannelMap.findMany({
|
||||
where: {
|
||||
accountId,
|
||||
role: "READER",
|
||||
channel: { type: "SOURCE", isActive: true },
|
||||
},
|
||||
include: { channel: true },
|
||||
});
|
||||
}
|
||||
// ── Global destination channel ──
|
||||
export async function getGlobalDestinationChannel() {
|
||||
const setting = await db.globalSetting.findUnique({
|
||||
where: { key: "destination_channel_id" },
|
||||
});
|
||||
if (!setting)
|
||||
return null;
|
||||
return db.telegramChannel.findFirst({
|
||||
where: { id: setting.value, type: "DESTINATION", isActive: true },
|
||||
});
|
||||
}
|
||||
export async function getGlobalSetting(key) {
|
||||
const setting = await db.globalSetting.findUnique({ where: { key } });
|
||||
return setting?.value ?? null;
|
||||
}
|
||||
export async function setGlobalSetting(key, value) {
|
||||
return db.globalSetting.upsert({
|
||||
where: { key },
|
||||
create: { key, value },
|
||||
update: { value },
|
||||
});
|
||||
}
|
||||
export async function packageExistsByHash(contentHash) {
|
||||
const pkg = await db.package.findFirst({
|
||||
where: { contentHash, destMessageId: { not: null } },
|
||||
select: { id: true },
|
||||
});
|
||||
return pkg !== null;
|
||||
}
|
||||
/**
|
||||
* Check if a package already exists for a given source message ID
|
||||
* AND was successfully uploaded to the destination (destMessageId is set).
|
||||
* Used as an early skip before downloading.
|
||||
*/
|
||||
export async function packageExistsBySourceMessage(sourceChannelId, sourceMessageId) {
|
||||
const pkg = await db.package.findFirst({
|
||||
where: { sourceChannelId, sourceMessageId, destMessageId: { not: null } },
|
||||
select: { id: true },
|
||||
});
|
||||
return pkg !== null;
|
||||
}
|
||||
/**
|
||||
* Delete orphaned Package rows that have the same content hash but never
|
||||
* completed the upload (destMessageId is null). Called before creating a
|
||||
* new complete record to avoid unique constraint violations.
|
||||
*/
|
||||
export async function deleteOrphanedPackageByHash(contentHash) {
|
||||
await db.package.deleteMany({
|
||||
where: { contentHash, destMessageId: null },
|
||||
});
|
||||
}
|
||||
export async function createPackageWithFiles(input) {
|
||||
const pkg = await db.package.create({
|
||||
data: {
|
||||
contentHash: input.contentHash,
|
||||
fileName: input.fileName,
|
||||
fileSize: input.fileSize,
|
||||
archiveType: input.archiveType,
|
||||
sourceChannelId: input.sourceChannelId,
|
||||
sourceMessageId: input.sourceMessageId,
|
||||
sourceTopicId: input.sourceTopicId ?? undefined,
|
||||
destChannelId: input.destChannelId,
|
||||
destMessageId: input.destMessageId,
|
||||
isMultipart: input.isMultipart,
|
||||
partCount: input.partCount,
|
||||
fileCount: input.files.length,
|
||||
ingestionRunId: input.ingestionRunId,
|
||||
creator: input.creator ?? undefined,
|
||||
previewData: input.previewData ? new Uint8Array(input.previewData) : undefined,
|
||||
previewMsgId: input.previewMsgId ?? undefined,
|
||||
files: {
|
||||
create: input.files,
|
||||
},
|
||||
},
|
||||
});
|
||||
// Notify the bot service about the new package (for subscription alerts)
|
||||
try {
|
||||
await db.$queryRawUnsafe(`SELECT pg_notify('new_package', $1)`, JSON.stringify({
|
||||
packageId: pkg.id,
|
||||
fileName: input.fileName,
|
||||
creator: input.creator ?? null,
|
||||
}));
|
||||
}
|
||||
catch {
|
||||
// Best-effort — don't fail the ingestion if notification fails
|
||||
}
|
||||
return pkg;
|
||||
}
|
||||
export async function createIngestionRun(accountId) {
|
||||
return db.ingestionRun.create({
|
||||
data: {
|
||||
accountId,
|
||||
status: "RUNNING",
|
||||
currentActivity: "Starting ingestion run",
|
||||
currentStep: "initializing",
|
||||
lastActivityAt: new Date(),
|
||||
},
|
||||
});
|
||||
}
|
||||
export async function updateRunActivity(runId, activity) {
|
||||
return db.ingestionRun.update({
|
||||
where: { id: runId },
|
||||
data: {
|
||||
currentActivity: activity.currentActivity,
|
||||
currentStep: activity.currentStep,
|
||||
currentChannel: activity.currentChannel ?? undefined,
|
||||
currentFile: activity.currentFile ?? undefined,
|
||||
currentFileNum: activity.currentFileNum ?? undefined,
|
||||
totalFiles: activity.totalFiles ?? undefined,
|
||||
downloadedBytes: activity.downloadedBytes ?? undefined,
|
||||
totalBytes: activity.totalBytes ?? undefined,
|
||||
downloadPercent: activity.downloadPercent ?? undefined,
|
||||
lastActivityAt: new Date(),
|
||||
...(activity.messagesScanned !== undefined && { messagesScanned: activity.messagesScanned }),
|
||||
...(activity.zipsFound !== undefined && { zipsFound: activity.zipsFound }),
|
||||
...(activity.zipsDuplicate !== undefined && { zipsDuplicate: activity.zipsDuplicate }),
|
||||
...(activity.zipsIngested !== undefined && { zipsIngested: activity.zipsIngested }),
|
||||
},
|
||||
});
|
||||
}
|
||||
const CLEAR_ACTIVITY = {
|
||||
currentActivity: null,
|
||||
currentStep: null,
|
||||
currentChannel: null,
|
||||
currentFile: null,
|
||||
currentFileNum: null,
|
||||
totalFiles: null,
|
||||
downloadedBytes: null,
|
||||
totalBytes: null,
|
||||
downloadPercent: null,
|
||||
lastActivityAt: new Date(),
|
||||
};
|
||||
export async function completeIngestionRun(runId, counters) {
|
||||
return db.ingestionRun.update({
|
||||
where: { id: runId },
|
||||
data: {
|
||||
status: "COMPLETED",
|
||||
finishedAt: new Date(),
|
||||
...counters,
|
||||
...CLEAR_ACTIVITY,
|
||||
},
|
||||
});
|
||||
}
|
||||
export async function failIngestionRun(runId, errorMessage) {
|
||||
return db.ingestionRun.update({
|
||||
where: { id: runId },
|
||||
data: {
|
||||
status: "FAILED",
|
||||
finishedAt: new Date(),
|
||||
errorMessage,
|
||||
...CLEAR_ACTIVITY,
|
||||
},
|
||||
});
|
||||
}
|
||||
export async function updateLastProcessedMessage(mappingId, messageId) {
|
||||
return db.accountChannelMap.update({
|
||||
where: { id: mappingId },
|
||||
data: { lastProcessedMessageId: messageId },
|
||||
});
|
||||
}
|
||||
export async function markStaleRunsAsFailed() {
|
||||
return db.ingestionRun.updateMany({
|
||||
where: { status: "RUNNING" },
|
||||
data: {
|
||||
status: "FAILED",
|
||||
finishedAt: new Date(),
|
||||
errorMessage: "Worker restarted — run was still marked as RUNNING",
|
||||
},
|
||||
});
|
||||
}
|
||||
export async function updateAccountAuthState(accountId, authState, authCode) {
|
||||
return db.telegramAccount.update({
|
||||
where: { id: accountId },
|
||||
data: { authState, authCode, lastSeenAt: authState === "AUTHENTICATED" ? new Date() : undefined },
|
||||
});
|
||||
}
|
||||
export async function getAccountAuthCode(accountId) {
|
||||
const account = await db.telegramAccount.findUnique({
|
||||
where: { id: accountId },
|
||||
select: { authCode: true, authState: true },
|
||||
});
|
||||
return account;
|
||||
}
|
||||
/**
|
||||
* Upsert a channel by telegramId. Returns the channel record.
|
||||
* If it already exists, update title and forum status.
|
||||
* New channels default to disabled (isActive: false) so the admin must
|
||||
* explicitly enable them before the worker processes them.
|
||||
* Pass isActive: true for DESTINATION channels that must be active immediately.
|
||||
*/
|
||||
export async function upsertChannel(input) {
|
||||
return db.telegramChannel.upsert({
|
||||
where: { telegramId: input.telegramId },
|
||||
create: {
|
||||
telegramId: input.telegramId,
|
||||
title: input.title,
|
||||
type: input.type,
|
||||
isForum: input.isForum,
|
||||
isActive: input.isActive ?? false,
|
||||
},
|
||||
update: {
|
||||
title: input.title,
|
||||
isForum: input.isForum,
|
||||
},
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Link an account to a channel if not already linked.
|
||||
* Uses a try/catch on unique constraint to make it idempotent.
|
||||
*/
|
||||
export async function ensureAccountChannelLink(accountId, channelId, role) {
|
||||
try {
|
||||
return await db.accountChannelMap.create({
|
||||
data: { accountId, channelId, role },
|
||||
});
|
||||
}
|
||||
catch (err) {
|
||||
// Already linked — ignore unique constraint violation
|
||||
if (err instanceof Error && err.message.includes("Unique constraint")) {
|
||||
return null;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
// ── Forum / Topic progress ──
|
||||
export async function setChannelForum(channelId, isForum) {
|
||||
return db.telegramChannel.update({
|
||||
where: { id: channelId },
|
||||
data: { isForum },
|
||||
});
|
||||
}
|
||||
export async function getTopicProgress(mappingId) {
|
||||
return db.topicProgress.findMany({
|
||||
where: { accountChannelMapId: mappingId },
|
||||
});
|
||||
}
|
||||
export async function upsertTopicProgress(mappingId, topicId, topicName, lastProcessedMessageId) {
|
||||
return db.topicProgress.upsert({
|
||||
where: {
|
||||
accountChannelMapId_topicId: {
|
||||
accountChannelMapId: mappingId,
|
||||
topicId,
|
||||
},
|
||||
},
|
||||
create: {
|
||||
accountChannelMapId: mappingId,
|
||||
topicId,
|
||||
topicName,
|
||||
lastProcessedMessageId,
|
||||
},
|
||||
update: {
|
||||
topicName,
|
||||
lastProcessedMessageId,
|
||||
},
|
||||
});
|
||||
}
|
||||
// ── Channel fetch requests (DB-mediated communication with web app) ──
|
||||
export async function getChannelFetchRequest(requestId) {
|
||||
return db.channelFetchRequest.findUnique({
|
||||
where: { id: requestId },
|
||||
include: { account: true },
|
||||
});
|
||||
}
|
||||
export async function updateFetchRequestStatus(requestId, status, extra) {
|
||||
return db.channelFetchRequest.update({
|
||||
where: { id: requestId },
|
||||
data: {
|
||||
status,
|
||||
resultJson: extra?.resultJson ?? undefined,
|
||||
error: extra?.error ?? undefined,
|
||||
},
|
||||
});
|
||||
}
|
||||
export async function getAccountLinkedChannelIds(accountId) {
|
||||
const links = await db.accountChannelMap.findMany({
|
||||
where: { accountId },
|
||||
select: { channel: { select: { telegramId: true } } },
|
||||
});
|
||||
return new Set(links.map((l) => l.channel.telegramId.toString()));
|
||||
}
|
||||
export async function getExistingChannelsByTelegramId() {
|
||||
const channels = await db.telegramChannel.findMany({
|
||||
select: { id: true, telegramId: true },
|
||||
});
|
||||
const map = new Map();
|
||||
for (const ch of channels) {
|
||||
map.set(ch.telegramId.toString(), ch.id);
|
||||
}
|
||||
return map;
|
||||
}
|
||||
export async function getAccountById(accountId) {
|
||||
return db.telegramAccount.findUnique({ where: { id: accountId } });
|
||||
}
|
||||
//# sourceMappingURL=queries.js.map
|
||||
1
worker/dist/db/queries.js.map
vendored
1
worker/dist/db/queries.js.map
vendored
File diff suppressed because one or more lines are too long
11
worker/dist/fetch-listener.d.ts
vendored
11
worker/dist/fetch-listener.d.ts
vendored
@@ -1,11 +0,0 @@
|
||||
/**
|
||||
* Start listening for pg_notify signals from the web app.
|
||||
*
|
||||
* Channels:
|
||||
* - `channel_fetch` — payload = requestId → fetch channels for an account
|
||||
* - `generate_invite` — payload = channelId → generate invite link for destination
|
||||
* - `create_destination` — payload = JSON { requestId, title } → create supergroup via TDLib
|
||||
* - `ingestion_trigger` — trigger an immediate ingestion cycle
|
||||
*/
|
||||
export declare function startFetchListener(): Promise<void>;
|
||||
export declare function stopFetchListener(): void;
|
||||
195
worker/dist/fetch-listener.js
vendored
195
worker/dist/fetch-listener.js
vendored
@@ -1,195 +0,0 @@
|
||||
import { pool } from "./db/client.js";
|
||||
import { childLogger } from "./util/logger.js";
|
||||
import { withTdlibMutex } from "./util/mutex.js";
|
||||
import { processFetchRequest } from "./worker.js";
|
||||
import { generateInviteLink, createSupergroup } from "./tdlib/chats.js";
|
||||
import { createTdlibClient, closeTdlibClient } from "./tdlib/client.js";
|
||||
import { triggerImmediateCycle } from "./scheduler.js";
|
||||
import { getGlobalDestinationChannel, setGlobalSetting, getActiveAccounts, upsertChannel, ensureAccountChannelLink, } from "./db/queries.js";
|
||||
const log = childLogger("fetch-listener");
|
||||
let pgClient = null;
|
||||
/**
|
||||
* Start listening for pg_notify signals from the web app.
|
||||
*
|
||||
* Channels:
|
||||
* - `channel_fetch` — payload = requestId → fetch channels for an account
|
||||
* - `generate_invite` — payload = channelId → generate invite link for destination
|
||||
* - `create_destination` — payload = JSON { requestId, title } → create supergroup via TDLib
|
||||
* - `ingestion_trigger` — trigger an immediate ingestion cycle
|
||||
*/
|
||||
export async function startFetchListener() {
|
||||
pgClient = await pool.connect();
|
||||
await pgClient.query("LISTEN channel_fetch");
|
||||
await pgClient.query("LISTEN generate_invite");
|
||||
await pgClient.query("LISTEN create_destination");
|
||||
await pgClient.query("LISTEN ingestion_trigger");
|
||||
pgClient.on("notification", (msg) => {
|
||||
if (msg.channel === "channel_fetch" && msg.payload) {
|
||||
handleChannelFetch(msg.payload);
|
||||
}
|
||||
else if (msg.channel === "generate_invite" && msg.payload) {
|
||||
handleGenerateInvite(msg.payload);
|
||||
}
|
||||
else if (msg.channel === "create_destination" && msg.payload) {
|
||||
handleCreateDestination(msg.payload);
|
||||
}
|
||||
else if (msg.channel === "ingestion_trigger") {
|
||||
handleIngestionTrigger();
|
||||
}
|
||||
});
|
||||
log.info("Fetch listener started (channel_fetch, generate_invite, create_destination, ingestion_trigger)");
|
||||
}
|
||||
export function stopFetchListener() {
|
||||
if (pgClient) {
|
||||
pgClient.release();
|
||||
pgClient = null;
|
||||
}
|
||||
log.info("Fetch listener stopped");
|
||||
}
|
||||
// ── Channel fetch handler ──
|
||||
// Chain promises to ensure sequential execution
|
||||
let fetchQueue = Promise.resolve();
|
||||
function handleChannelFetch(requestId) {
|
||||
fetchQueue = fetchQueue.then(async () => {
|
||||
try {
|
||||
await withTdlibMutex("fetch-channels", () => processFetchRequest(requestId));
|
||||
}
|
||||
catch (err) {
|
||||
log.error({ err, requestId }, "Failed to process fetch request");
|
||||
}
|
||||
});
|
||||
}
|
||||
// ── Invite link generation handler ──
|
||||
function handleGenerateInvite(channelId) {
|
||||
fetchQueue = fetchQueue.then(async () => {
|
||||
try {
|
||||
await withTdlibMutex("generate-invite", async () => {
|
||||
const destChannel = await getGlobalDestinationChannel();
|
||||
if (!destChannel || destChannel.id !== channelId) {
|
||||
log.warn({ channelId }, "Destination channel mismatch, skipping invite generation");
|
||||
return;
|
||||
}
|
||||
// Use the first available authenticated account to generate the link
|
||||
const accounts = await getActiveAccounts();
|
||||
if (accounts.length === 0) {
|
||||
log.warn("No authenticated accounts to generate invite link");
|
||||
return;
|
||||
}
|
||||
const account = accounts[0];
|
||||
const client = await createTdlibClient({ id: account.id, phone: account.phone });
|
||||
try {
|
||||
const link = await generateInviteLink(client, destChannel.telegramId);
|
||||
await setGlobalSetting("destination_invite_link", link);
|
||||
log.info({ link }, "Invite link generated and saved");
|
||||
}
|
||||
finally {
|
||||
await closeTdlibClient(client);
|
||||
}
|
||||
});
|
||||
}
|
||||
catch (err) {
|
||||
log.error({ err, channelId }, "Failed to generate invite link");
|
||||
}
|
||||
});
|
||||
}
|
||||
// ── Create destination supergroup handler ──
|
||||
function handleCreateDestination(payload) {
|
||||
fetchQueue = fetchQueue.then(async () => {
|
||||
let requestId;
|
||||
try {
|
||||
const parsed = JSON.parse(payload);
|
||||
requestId = parsed.requestId;
|
||||
await withTdlibMutex("create-destination", async () => {
|
||||
const { db } = await import("./db/client.js");
|
||||
// Mark the request as in-progress
|
||||
await db.channelFetchRequest.update({
|
||||
where: { id: parsed.requestId },
|
||||
data: { status: "IN_PROGRESS" },
|
||||
});
|
||||
// Use the first available authenticated account
|
||||
const accounts = await getActiveAccounts();
|
||||
if (accounts.length === 0) {
|
||||
throw new Error("No authenticated accounts available to create the group");
|
||||
}
|
||||
const account = accounts[0];
|
||||
const client = await createTdlibClient({ id: account.id, phone: account.phone });
|
||||
try {
|
||||
// Create the supergroup via TDLib
|
||||
const result = await createSupergroup(client, parsed.title);
|
||||
log.info({ chatId: result.chatId.toString(), title: result.title }, "Supergroup created");
|
||||
// Upsert it as a DESTINATION channel in the DB (active by default)
|
||||
const channel = await upsertChannel({
|
||||
telegramId: result.chatId,
|
||||
title: result.title,
|
||||
type: "DESTINATION",
|
||||
isForum: false,
|
||||
isActive: true,
|
||||
});
|
||||
// Set as global destination
|
||||
await setGlobalSetting("destination_channel_id", channel.id);
|
||||
// Generate an invite link
|
||||
const link = await generateInviteLink(client, result.chatId);
|
||||
await setGlobalSetting("destination_invite_link", link);
|
||||
log.info({ link }, "Invite link generated for new destination");
|
||||
// Link all authenticated accounts as WRITER
|
||||
for (const acc of accounts) {
|
||||
try {
|
||||
await ensureAccountChannelLink(acc.id, channel.id, "WRITER");
|
||||
}
|
||||
catch {
|
||||
// Already linked
|
||||
}
|
||||
}
|
||||
// Mark fetch request as completed with the channel info
|
||||
await db.channelFetchRequest.update({
|
||||
where: { id: parsed.requestId },
|
||||
data: {
|
||||
status: "COMPLETED",
|
||||
resultJson: JSON.stringify({
|
||||
channelId: channel.id,
|
||||
telegramId: result.chatId.toString(),
|
||||
title: result.title,
|
||||
inviteLink: link,
|
||||
}),
|
||||
},
|
||||
});
|
||||
log.info({ channelId: channel.id, telegramId: result.chatId.toString() }, "Destination channel created and configured");
|
||||
}
|
||||
finally {
|
||||
await closeTdlibClient(client);
|
||||
}
|
||||
});
|
||||
}
|
||||
catch (err) {
|
||||
log.error({ err, payload }, "Failed to create destination channel");
|
||||
if (requestId) {
|
||||
try {
|
||||
const { db } = await import("./db/client.js");
|
||||
await db.channelFetchRequest.update({
|
||||
where: { id: requestId },
|
||||
data: {
|
||||
status: "FAILED",
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
},
|
||||
});
|
||||
}
|
||||
catch {
|
||||
// Best-effort
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
// ── Ingestion trigger handler ──
|
||||
function handleIngestionTrigger() {
|
||||
fetchQueue = fetchQueue.then(async () => {
|
||||
try {
|
||||
log.info("Ingestion trigger received from UI");
|
||||
await triggerImmediateCycle();
|
||||
}
|
||||
catch (err) {
|
||||
log.error({ err }, "Failed to trigger immediate ingestion cycle");
|
||||
}
|
||||
});
|
||||
}
|
||||
//# sourceMappingURL=fetch-listener.js.map
|
||||
1
worker/dist/fetch-listener.js.map
vendored
1
worker/dist/fetch-listener.js.map
vendored
File diff suppressed because one or more lines are too long
1
worker/dist/index.d.ts
vendored
1
worker/dist/index.d.ts
vendored
@@ -1 +0,0 @@
|
||||
export {};
|
||||
50
worker/dist/index.js
vendored
50
worker/dist/index.js
vendored
@@ -1,50 +0,0 @@
|
||||
import { mkdir } from "fs/promises";
|
||||
import { config } from "./util/config.js";
|
||||
import { logger } from "./util/logger.js";
|
||||
import { markStaleRunsAsFailed } from "./db/queries.js";
|
||||
import { cleanupTempDir } from "./worker.js";
|
||||
import { startScheduler, stopScheduler } from "./scheduler.js";
|
||||
import { startFetchListener, stopFetchListener } from "./fetch-listener.js";
|
||||
import { db, pool } from "./db/client.js";
|
||||
const log = logger.child({ module: "main" });
|
||||
async function main() {
|
||||
log.info("DragonsStash Telegram Worker starting");
|
||||
log.info({ config: { ...config, databaseUrl: "***" } }, "Configuration loaded");
|
||||
if (!config.telegramApiId || !config.telegramApiHash) {
|
||||
log.fatal("TELEGRAM_API_ID and TELEGRAM_API_HASH are both required");
|
||||
process.exit(1);
|
||||
}
|
||||
// Ensure temp directory exists
|
||||
await mkdir(config.tempDir, { recursive: true });
|
||||
await mkdir(config.tdlibStateDir, { recursive: true });
|
||||
// Clean up stale state
|
||||
await cleanupTempDir();
|
||||
await markStaleRunsAsFailed();
|
||||
// Start the fetch listener (pg_notify for on-demand channel fetching)
|
||||
await startFetchListener();
|
||||
// Start the scheduler
|
||||
await startScheduler();
|
||||
}
|
||||
// Graceful shutdown
|
||||
function shutdown(signal) {
|
||||
log.info({ signal }, "Shutdown signal received");
|
||||
stopScheduler();
|
||||
stopFetchListener();
|
||||
// Close DB connections
|
||||
Promise.all([db.$disconnect(), pool.end()])
|
||||
.then(() => {
|
||||
log.info("Shutdown complete");
|
||||
process.exit(0);
|
||||
})
|
||||
.catch((err) => {
|
||||
log.error({ err }, "Error during shutdown");
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
process.on("SIGTERM", () => shutdown("SIGTERM"));
|
||||
process.on("SIGINT", () => shutdown("SIGINT"));
|
||||
main().catch((err) => {
|
||||
log.fatal({ err }, "Worker failed to start");
|
||||
process.exit(1);
|
||||
});
|
||||
//# sourceMappingURL=index.js.map
|
||||
1
worker/dist/index.js.map
vendored
1
worker/dist/index.js.map
vendored
@@ -1 +0,0 @@
|
||||
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,OAAO,EAAE,qBAAqB,EAAE,MAAM,iBAAiB,CAAC;AACxD,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/D,OAAO,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAC5E,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAC;AAE1C,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;AAE7C,KAAK,UAAU,IAAI;IACjB,GAAG,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;IAClD,GAAG,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,GAAG,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,EAAE,EAAE,sBAAsB,CAAC,CAAC;IAEhF,IAAI,CAAC,MAAM,CAAC,aAAa,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC;QACrD,GAAG,CAAC,KAAK,CAAC,yDAAyD,CAAC,CAAC;QACrE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,+BAA+B;IAC/B,MAAM,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACjD,MAAM,KAAK,CAAC,MAAM,CAAC,aAAa,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAEvD,uBAAuB;IACvB,MAAM,cAAc,EAAE,CAAC;IACvB,MAAM,qBAAqB,EAAE,CAAC;IAE9B,sEAAsE;IACtE,MAAM,kBAAkB,EAAE,CAAC;IAE3B,sBAAsB;IACtB,MAAM,cAAc,EAAE,CAAC;AACzB,CAAC;AAED,oBAAoB;AACpB,SAAS,QAAQ,CAAC,MAAc;IAC9B,GAAG,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,0BAA0B,CAAC,CAAC;IACjD,aAAa,EAAE,CAAC;IAChB,iBAAiB,EAAE,CAAC;IAEpB,uBAAuB;IACvB,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,WAAW,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;SACxC,IAAI,CAAC,GAAG,EAAE;QACT,GAAG,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;QAC9B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC;SACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;QACb,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,EAAE,uBAAuB,CAAC,CAAC;QAC5C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;AACP,CAAC;AAED,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;AACjD,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;AAE/C,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IACnB,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,EAAE,wBAAwB,CAAC,CAAC;IAC7C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}
|
||||
22
worker/dist/preview/match.d.ts
vendored
22
worker/dist/preview/match.d.ts
vendored
@@ -1,22 +0,0 @@
|
||||
export interface TelegramPhoto {
|
||||
id: bigint;
|
||||
date: Date;
|
||||
/** Caption text on the photo message (if any). */
|
||||
caption: string;
|
||||
/** The smallest photo size available — used as thumbnail. */
|
||||
fileId: string;
|
||||
fileSize: number;
|
||||
}
|
||||
export interface ArchiveRef {
|
||||
baseName: string;
|
||||
firstMessageId: bigint;
|
||||
firstMessageDate: Date;
|
||||
}
|
||||
/**
|
||||
* Try to match a photo message to an archive by:
|
||||
* 1. Caption contains the archive baseName (without extension)
|
||||
* 2. Photo was posted within ±10 messages (time-window: ±6 hours)
|
||||
*
|
||||
* Returns the best match (closest in time), or null.
|
||||
*/
|
||||
export declare function matchPreviewToArchive(photos: TelegramPhoto[], archives: ArchiveRef[]): Map<string, TelegramPhoto>;
|
||||
53
worker/dist/preview/match.js
vendored
53
worker/dist/preview/match.js
vendored
@@ -1,53 +0,0 @@
|
||||
import { childLogger } from "../util/logger.js";
|
||||
const log = childLogger("preview-match");
|
||||
/**
|
||||
* Try to match a photo message to an archive by:
|
||||
* 1. Caption contains the archive baseName (without extension)
|
||||
* 2. Photo was posted within ±10 messages (time-window: ±6 hours)
|
||||
*
|
||||
* Returns the best match (closest in time), or null.
|
||||
*/
|
||||
export function matchPreviewToArchive(photos, archives) {
|
||||
const results = new Map();
|
||||
const TIME_WINDOW_MS = 6 * 60 * 60 * 1000; // 6 hours
|
||||
for (const archive of archives) {
|
||||
// Normalize the archive base name for matching
|
||||
const normalizedBase = normalizeForMatch(archive.baseName);
|
||||
if (!normalizedBase)
|
||||
continue;
|
||||
let bestMatch = null;
|
||||
let bestTimeDiff = Infinity;
|
||||
for (const photo of photos) {
|
||||
const timeDiff = Math.abs(photo.date.getTime() - archive.firstMessageDate.getTime());
|
||||
// Must be within time window
|
||||
if (timeDiff > TIME_WINDOW_MS)
|
||||
continue;
|
||||
// Check if the photo caption contains the archive base name
|
||||
const normalizedCaption = normalizeForMatch(photo.caption);
|
||||
if (!normalizedCaption)
|
||||
continue;
|
||||
const matches = normalizedCaption.includes(normalizedBase) ||
|
||||
normalizedBase.includes(normalizedCaption);
|
||||
if (matches && timeDiff < bestTimeDiff) {
|
||||
bestMatch = photo;
|
||||
bestTimeDiff = timeDiff;
|
||||
}
|
||||
}
|
||||
if (bestMatch) {
|
||||
log.debug({ baseName: archive.baseName, photoId: bestMatch.id.toString() }, "Matched preview photo to archive");
|
||||
results.set(archive.baseName, bestMatch);
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
/**
|
||||
* Strip extension, punctuation, and normalize for fuzzy matching.
|
||||
*/
|
||||
function normalizeForMatch(input) {
|
||||
return input
|
||||
.toLowerCase()
|
||||
.replace(/\.[a-z0-9]{1,5}$/i, "") // strip extension
|
||||
.replace(/[_\-.\s]+/g, " ") // normalize separators
|
||||
.trim();
|
||||
}
|
||||
//# sourceMappingURL=match.js.map
|
||||
1
worker/dist/preview/match.js.map
vendored
1
worker/dist/preview/match.js.map
vendored
@@ -1 +0,0 @@
|
||||
{"version":3,"file":"match.js","sourceRoot":"","sources":["../../src/preview/match.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,MAAM,GAAG,GAAG,WAAW,CAAC,eAAe,CAAC,CAAC;AAkBzC;;;;;;GAMG;AACH,MAAM,UAAU,qBAAqB,CACnC,MAAuB,EACvB,QAAsB;IAEtB,MAAM,OAAO,GAAG,IAAI,GAAG,EAAyB,CAAC;IACjD,MAAM,cAAc,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,UAAU;IAErD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,+CAA+C;QAC/C,MAAM,cAAc,GAAG,iBAAiB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC3D,IAAI,CAAC,cAAc;YAAE,SAAS;QAE9B,IAAI,SAAS,GAAyB,IAAI,CAAC;QAC3C,IAAI,YAAY,GAAG,QAAQ,CAAC;QAE5B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CACvB,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,OAAO,CAAC,gBAAgB,CAAC,OAAO,EAAE,CAC1D,CAAC;YAEF,6BAA6B;YAC7B,IAAI,QAAQ,GAAG,cAAc;gBAAE,SAAS;YAExC,4DAA4D;YAC5D,MAAM,iBAAiB,GAAG,iBAAiB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAC3D,IAAI,CAAC,iBAAiB;gBAAE,SAAS;YAEjC,MAAM,OAAO,GACX,iBAAiB,CAAC,QAAQ,CAAC,cAAc,CAAC;gBAC1C,cAAc,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC;YAE7C,IAAI,OAAO,IAAI,QAAQ,GAAG,YAAY,EAAE,CAAC;gBACvC,SAAS,GAAG,KAAK,CAAC;gBAClB,YAAY,GAAG,QAAQ,CAAC;YAC1B,CAAC;QACH,CAAC;QAED,IAAI,SAAS,EAAE,CAAC;YACd,GAAG,CAAC,KAAK,CACP,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,OAAO,EAAE,SAAS,CAAC,EAAE,CAAC,QAAQ,EAAE,EAAE,EAChE,kCAAkC,CACnC,CAAC;YACF,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;QAC3C,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB,CAAC,KAAa;IACtC,OAAO,KAAK;SACT,WAAW,EAAE;SACb,OAAO,CAAC,mBAAmB,EAAE,EAAE,CAAC,CAAC,kBAAkB;SACnD,OAAO,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC,uBAAuB;SAClD,IAAI,EAAE,CAAC;AACZ,CAAC"}
|
||||
13
worker/dist/scheduler.d.ts
vendored
13
worker/dist/scheduler.d.ts
vendored
@@ -1,13 +0,0 @@
|
||||
/**
|
||||
* Start the scheduler. Runs an immediate first cycle, then schedules subsequent ones.
|
||||
*/
|
||||
export declare function startScheduler(): Promise<void>;
|
||||
/**
|
||||
* Trigger an immediate ingestion cycle (e.g. from the admin UI).
|
||||
* If a cycle is already running, this is a no-op.
|
||||
*/
|
||||
export declare function triggerImmediateCycle(): Promise<void>;
|
||||
/**
|
||||
* Stop the scheduler gracefully.
|
||||
*/
|
||||
export declare function stopScheduler(): void;
|
||||
121
worker/dist/scheduler.js
vendored
121
worker/dist/scheduler.js
vendored
@@ -1,121 +0,0 @@
|
||||
import { config } from "./util/config.js";
|
||||
import { childLogger } from "./util/logger.js";
|
||||
import { withTdlibMutex } from "./util/mutex.js";
|
||||
import { getActiveAccounts, getPendingAccounts } from "./db/queries.js";
|
||||
import { runWorkerForAccount, authenticateAccount } from "./worker.js";
|
||||
const log = childLogger("scheduler");
|
||||
let running = false;
|
||||
let timer = null;
|
||||
let cycleCount = 0;
|
||||
/**
|
||||
* Maximum time for a single ingestion cycle (ms).
|
||||
* After this, new accounts won't be started (in-progress work finishes).
|
||||
* Default: 4 hours. Configurable via WORKER_CYCLE_TIMEOUT_MINUTES.
|
||||
*/
|
||||
const CYCLE_TIMEOUT_MS = (parseInt(process.env.WORKER_CYCLE_TIMEOUT_MINUTES ?? "240", 10)) * 60 * 1000;
|
||||
/**
|
||||
* Run one ingestion cycle:
|
||||
* 1. Authenticate any PENDING accounts (triggers SMS code flow + auto-fetch channels)
|
||||
* 2. Process all active AUTHENTICATED accounts for ingestion
|
||||
*
|
||||
* All TDLib operations are wrapped in the mutex to ensure only one client
|
||||
* runs at a time (also shared with the fetch listener for on-demand requests).
|
||||
*
|
||||
* The cycle has a configurable timeout (WORKER_CYCLE_TIMEOUT_MINUTES, default 4h).
|
||||
* Once the timeout elapses, no new accounts will be started but any in-progress
|
||||
* account processing is allowed to finish its current archive set.
|
||||
*/
|
||||
async function runCycle() {
|
||||
if (running) {
|
||||
log.warn("Previous cycle still running, skipping");
|
||||
return;
|
||||
}
|
||||
running = true;
|
||||
cycleCount++;
|
||||
const cycleStart = Date.now();
|
||||
log.info({ cycle: cycleCount, timeoutMinutes: CYCLE_TIMEOUT_MS / 60_000 }, "Starting ingestion cycle");
|
||||
try {
|
||||
// ── Phase 1: Authenticate pending accounts ──
|
||||
const pendingAccounts = await getPendingAccounts();
|
||||
if (pendingAccounts.length > 0) {
|
||||
log.info({ count: pendingAccounts.length }, "Found pending accounts, starting authentication");
|
||||
for (const account of pendingAccounts) {
|
||||
if (Date.now() - cycleStart > CYCLE_TIMEOUT_MS) {
|
||||
log.warn("Cycle timeout reached during authentication phase, stopping");
|
||||
break;
|
||||
}
|
||||
await withTdlibMutex(`auth:${account.phone}`, () => authenticateAccount(account));
|
||||
}
|
||||
}
|
||||
// ── Phase 2: Ingest for authenticated accounts ──
|
||||
const accounts = await getActiveAccounts();
|
||||
if (accounts.length === 0) {
|
||||
log.info("No active authenticated accounts, nothing to ingest");
|
||||
return;
|
||||
}
|
||||
log.info({ accountCount: accounts.length }, "Processing accounts");
|
||||
for (const account of accounts) {
|
||||
if (Date.now() - cycleStart > CYCLE_TIMEOUT_MS) {
|
||||
log.warn({ elapsed: Math.round((Date.now() - cycleStart) / 60_000), timeoutMinutes: CYCLE_TIMEOUT_MS / 60_000 }, "Cycle timeout reached, skipping remaining accounts");
|
||||
break;
|
||||
}
|
||||
await withTdlibMutex(`ingest:${account.phone}`, () => runWorkerForAccount(account));
|
||||
}
|
||||
log.info({ elapsed: Math.round((Date.now() - cycleStart) / 1000) }, "Ingestion cycle complete");
|
||||
}
|
||||
catch (err) {
|
||||
log.error({ err }, "Ingestion cycle failed");
|
||||
}
|
||||
finally {
|
||||
running = false;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Schedule the next cycle with jitter.
|
||||
*/
|
||||
function scheduleNext() {
|
||||
const intervalMs = config.workerIntervalMinutes * 60 * 1000;
|
||||
const jitterMs = Math.random() * config.jitterMinutes * 60 * 1000;
|
||||
const delay = intervalMs + jitterMs;
|
||||
log.info({ nextRunInMinutes: Math.round(delay / 60000) }, "Next cycle scheduled");
|
||||
timer = setTimeout(async () => {
|
||||
await runCycle();
|
||||
scheduleNext();
|
||||
}, delay);
|
||||
}
|
||||
/**
|
||||
* Start the scheduler. Runs an immediate first cycle, then schedules subsequent ones.
|
||||
*/
|
||||
export async function startScheduler() {
|
||||
log.info({
|
||||
intervalMinutes: config.workerIntervalMinutes,
|
||||
jitterMinutes: config.jitterMinutes,
|
||||
}, "Scheduler starting");
|
||||
// Run immediately on start
|
||||
await runCycle();
|
||||
// Then schedule recurring cycles
|
||||
scheduleNext();
|
||||
}
|
||||
/**
|
||||
* Trigger an immediate ingestion cycle (e.g. from the admin UI).
|
||||
* If a cycle is already running, this is a no-op.
|
||||
*/
|
||||
export async function triggerImmediateCycle() {
|
||||
if (running) {
|
||||
log.info("Cycle already running, ignoring trigger");
|
||||
return;
|
||||
}
|
||||
log.info("Immediate cycle triggered via UI");
|
||||
await runCycle();
|
||||
}
|
||||
/**
|
||||
* Stop the scheduler gracefully.
|
||||
*/
|
||||
export function stopScheduler() {
|
||||
if (timer) {
|
||||
clearTimeout(timer);
|
||||
timer = null;
|
||||
}
|
||||
log.info("Scheduler stopped");
|
||||
}
|
||||
//# sourceMappingURL=scheduler.js.map
|
||||
1
worker/dist/scheduler.js.map
vendored
1
worker/dist/scheduler.js.map
vendored
@@ -1 +0,0 @@
|
||||
{"version":3,"file":"scheduler.js","sourceRoot":"","sources":["../src/scheduler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AACxE,OAAO,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAEvE,MAAM,GAAG,GAAG,WAAW,CAAC,WAAW,CAAC,CAAC;AAErC,IAAI,OAAO,GAAG,KAAK,CAAC;AACpB,IAAI,KAAK,GAAyC,IAAI,CAAC;AACvD,IAAI,UAAU,GAAG,CAAC,CAAC;AAEnB;;;;GAIG;AACH,MAAM,gBAAgB,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,4BAA4B,IAAI,KAAK,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;AAEvG;;;;;;;;;;;GAWG;AACH,KAAK,UAAU,QAAQ;IACrB,IAAI,OAAO,EAAE,CAAC;QACZ,GAAG,CAAC,IAAI,CAAC,wCAAwC,CAAC,CAAC;QACnD,OAAO;IACT,CAAC;IAED,OAAO,GAAG,IAAI,CAAC;IACf,UAAU,EAAE,CAAC;IACb,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC9B,GAAG,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,cAAc,EAAE,gBAAgB,GAAG,MAAM,EAAE,EAAE,0BAA0B,CAAC,CAAC;IAEvG,IAAI,CAAC;QACH,+CAA+C;QAC/C,MAAM,eAAe,GAAG,MAAM,kBAAkB,EAAE,CAAC;QACnD,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/B,GAAG,CAAC,IAAI,CACN,EAAE,KAAK,EAAE,eAAe,CAAC,MAAM,EAAE,EACjC,iDAAiD,CAClD,CAAC;YACF,KAAK,MAAM,OAAO,IAAI,eAAe,EAAE,CAAC;gBACtC,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,GAAG,gBAAgB,EAAE,CAAC;oBAC/C,GAAG,CAAC,IAAI,CAAC,6DAA6D,CAAC,CAAC;oBACxE,MAAM;gBACR,CAAC;gBACD,MAAM,cAAc,CAAC,QAAQ,OAAO,CAAC,KAAK,EAAE,EAAE,GAAG,EAAE,CACjD,mBAAmB,CAAC,OAAO,CAAC,CAC7B,CAAC;YACJ,CAAC;QACH,CAAC;QAED,mDAAmD;QACnD,MAAM,QAAQ,GAAG,MAAM,iBAAiB,EAAE,CAAC;QAE3C,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,GAAG,CAAC,IAAI,CAAC,qDAAqD,CAAC,CAAC;YAChE,OAAO;QACT,CAAC;QAED,GAAG,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,QAAQ,CAAC,MAAM,EAAE,EAAE,qBAAqB,CAAC,CAAC;QAEnE,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,GAAG,gBAAgB,EAAE,CAAC;gBAC/C,GAAG,CAAC,IAAI,CACN,EAAE,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC,GAAG,MAAM,CAAC,EAAE,cAAc,EAAE,gBAAgB,GAAG,MAAM,EAAE,EACtG,oDAAoD,CACrD,CAAC;gBACF,MAAM;YACR,CAAC;YACD,MAAM,cAAc,CAAC,UAAU,OAAO,CAAC,KAAK,EAAE,EAAE,GAAG,EAAE,CACnD,mBAAmB,CAAC,OAAO,CAAC,CAC7B,CAAC;QACJ,CAAC;QAED,GAAG,CAAC,IAAI,CACN,EAAE,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC,GAAG,IAAI,CAAC,EAAE,EACzD,0BAA0B,CAC3B,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,EAAE,wBAAwB,CAAC,CAAC;IAC/C,CAAC;YAAS,CAAC;QACT,OAAO,GAAG,KAAK,CAAC;IAClB,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,YAAY;IACnB,MAAM,UAAU,GAAG,MAAM,CAAC,qBAAqB,GAAG,EAAE,GAAG,IAAI,CAAC;IAC5D,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC,aAAa,GAAG,EAAE,GAAG,IAAI,CAAC;IAClE,MAAM,KAAK,GAAG,UAAU,GAAG,QAAQ,CAAC;IAEpC,GAAG,CAAC,IAAI,CACN,EAAE,gBAAgB,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,EAAE,EAC/C,sBAAsB,CACvB,CAAC;IAEF,KAAK,GAAG,UAAU,CAAC,KAAK,IAAI,EAAE;QAC5B,MAAM,QAAQ,EAAE,CAAC;QACjB,YAAY,EAAE,CAAC;IACjB,CAAC,EAAE,KAAK,CAAC,CAAC;AACZ,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc;IAClC,GAAG,CAAC,IAAI,CACN;QACE,eAAe,EAAE,MAAM,CAAC,qBAAqB;QAC7C,aAAa,EAAE,MAAM,CAAC,aAAa;KACpC,EACD,oBAAoB,CACrB,CAAC;IAEF,2BAA2B;IAC3B,MAAM,QAAQ,EAAE,CAAC;IAEjB,iCAAiC;IACjC,YAAY,EAAE,CAAC;AACjB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB;IACzC,IAAI,OAAO,EAAE,CAAC;QACZ,GAAG,CAAC,IAAI,CAAC,yCAAyC,CAAC,CAAC;QACpD,OAAO;IACT,CAAC;IACD,GAAG,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;IAC7C,MAAM,QAAQ,EAAE,CAAC;AACnB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,aAAa;IAC3B,IAAI,KAAK,EAAE,CAAC;QACV,YAAY,CAAC,KAAK,CAAC,CAAC;QACpB,KAAK,GAAG,IAAI,CAAC;IACf,CAAC;IACD,GAAG,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;AAChC,CAAC"}
|
||||
31
worker/dist/tdlib/chats.d.ts
vendored
31
worker/dist/tdlib/chats.d.ts
vendored
@@ -1,31 +0,0 @@
|
||||
import type { Client } from "tdl";
|
||||
export interface TelegramChatInfo {
|
||||
chatId: bigint;
|
||||
title: string;
|
||||
type: "channel" | "supergroup" | "group" | "private" | "other";
|
||||
isForum: boolean;
|
||||
memberCount?: number;
|
||||
}
|
||||
/**
|
||||
* Fetch all chats the account is a member of.
|
||||
* Uses TDLib's getChats to load the chat list, then getChat for details.
|
||||
* Filters to channels and supergroups only (groups/privates are not useful for ingestion).
|
||||
*/
|
||||
export declare function getAccountChats(client: Client): Promise<TelegramChatInfo[]>;
|
||||
/**
|
||||
* Generate an invite link for a chat. The account must be an admin or have
|
||||
* invite link permissions.
|
||||
*/
|
||||
export declare function generateInviteLink(client: Client, chatId: bigint): Promise<string>;
|
||||
/**
|
||||
* Create a new supergroup (private group) via TDLib.
|
||||
* Returns the chat ID and title.
|
||||
*/
|
||||
export declare function createSupergroup(client: Client, title: string): Promise<{
|
||||
chatId: bigint;
|
||||
title: string;
|
||||
}>;
|
||||
/**
|
||||
* Join a chat using an invite link.
|
||||
*/
|
||||
export declare function joinChatByInviteLink(client: Client, inviteLink: string): Promise<void>;
|
||||
124
worker/dist/tdlib/chats.js
vendored
124
worker/dist/tdlib/chats.js
vendored
@@ -1,124 +0,0 @@
|
||||
import { childLogger } from "../util/logger.js";
|
||||
import { config } from "../util/config.js";
|
||||
const log = childLogger("chats");
|
||||
/**
|
||||
* Fetch all chats the account is a member of.
|
||||
* Uses TDLib's getChats to load the chat list, then getChat for details.
|
||||
* Filters to channels and supergroups only (groups/privates are not useful for ingestion).
|
||||
*/
|
||||
export async function getAccountChats(client) {
|
||||
const chats = [];
|
||||
// Load main chat list — TDLib loads in batches
|
||||
let offsetOrder = "9223372036854775807"; // max int64 as string
|
||||
let offsetChatId = 0;
|
||||
let hasMore = true;
|
||||
while (hasMore) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const result = (await client.invoke({
|
||||
_: "getChats",
|
||||
chat_list: { _: "chatListMain" },
|
||||
limit: 100,
|
||||
}));
|
||||
if (!result.chat_ids || result.chat_ids.length === 0) {
|
||||
break;
|
||||
}
|
||||
for (const chatId of result.chat_ids) {
|
||||
try {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const chat = (await client.invoke({
|
||||
_: "getChat",
|
||||
chat_id: chatId,
|
||||
}));
|
||||
const chatType = chat.type?._;
|
||||
let type = "other";
|
||||
let isForum = false;
|
||||
if (chatType === "chatTypeSupergroup") {
|
||||
// Get supergroup details to check if it's a channel or group
|
||||
try {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const sg = (await client.invoke({
|
||||
_: "getSupergroup",
|
||||
supergroup_id: chat.type.supergroup_id,
|
||||
}));
|
||||
type = sg.is_channel ? "channel" : "supergroup";
|
||||
isForum = sg.is_forum ?? false;
|
||||
}
|
||||
catch {
|
||||
type = "supergroup";
|
||||
}
|
||||
}
|
||||
else if (chatType === "chatTypeBasicGroup") {
|
||||
type = "group";
|
||||
}
|
||||
else if (chatType === "chatTypePrivate" || chatType === "chatTypeSecret") {
|
||||
type = "private";
|
||||
}
|
||||
// Only include channels and supergroups
|
||||
if (type === "channel" || type === "supergroup") {
|
||||
chats.push({
|
||||
chatId: BigInt(chatId),
|
||||
title: chat.title ?? `Chat ${chatId}`,
|
||||
type,
|
||||
isForum,
|
||||
});
|
||||
}
|
||||
}
|
||||
catch (err) {
|
||||
log.warn({ chatId, err }, "Failed to get chat details, skipping");
|
||||
}
|
||||
}
|
||||
// getChats with chatListMain returns all chats at once in newer TDLib versions
|
||||
// So we break after the first batch
|
||||
hasMore = false;
|
||||
await sleep(config.apiDelayMs);
|
||||
}
|
||||
log.info({ total: chats.length }, "Fetched channels/supergroups from Telegram");
|
||||
return chats;
|
||||
}
|
||||
/**
|
||||
* Generate an invite link for a chat. The account must be an admin or have
|
||||
* invite link permissions.
|
||||
*/
|
||||
export async function generateInviteLink(client, chatId) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const result = (await client.invoke({
|
||||
_: "createChatInviteLink",
|
||||
chat_id: Number(chatId),
|
||||
name: "DragonsStash Auto-Join",
|
||||
creates_join_request: false,
|
||||
}));
|
||||
const link = result.invite_link;
|
||||
log.info({ chatId: chatId.toString(), link }, "Generated invite link");
|
||||
return link;
|
||||
}
|
||||
/**
|
||||
* Create a new supergroup (private group) via TDLib.
|
||||
* Returns the chat ID and title.
|
||||
*/
|
||||
export async function createSupergroup(client, title) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const result = (await client.invoke({
|
||||
_: "createNewSupergroupChat",
|
||||
title,
|
||||
is_forum: false,
|
||||
is_channel: false,
|
||||
description: "DragonsStash archive destination — all accounts write here",
|
||||
}));
|
||||
const chatId = BigInt(result.id);
|
||||
log.info({ chatId: chatId.toString(), title }, "Created new supergroup");
|
||||
return { chatId, title: result.title ?? title };
|
||||
}
|
||||
/**
|
||||
* Join a chat using an invite link.
|
||||
*/
|
||||
export async function joinChatByInviteLink(client, inviteLink) {
|
||||
await client.invoke({
|
||||
_: "joinChatByInviteLink",
|
||||
invite_link: inviteLink,
|
||||
});
|
||||
log.info({ inviteLink }, "Joined chat by invite link");
|
||||
}
|
||||
function sleep(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
//# sourceMappingURL=chats.js.map
|
||||
1
worker/dist/tdlib/chats.js.map
vendored
1
worker/dist/tdlib/chats.js.map
vendored
@@ -1 +0,0 @@
|
||||
{"version":3,"file":"chats.js","sourceRoot":"","sources":["../../src/tdlib/chats.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,MAAM,GAAG,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;AAUjC;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,MAAc;IAEd,MAAM,KAAK,GAAuB,EAAE,CAAC;IAErC,+CAA+C;IAC/C,IAAI,WAAW,GAAG,qBAAqB,CAAC,CAAC,sBAAsB;IAC/D,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,OAAO,GAAG,IAAI,CAAC;IAEnB,OAAO,OAAO,EAAE,CAAC;QACf,8DAA8D;QAC9D,MAAM,MAAM,GAAG,CAAC,MAAM,MAAM,CAAC,MAAM,CAAC;YAClC,CAAC,EAAE,UAAU;YACb,SAAS,EAAE,EAAE,CAAC,EAAE,cAAc,EAAE;YAChC,KAAK,EAAE,GAAG;SACX,CAAC,CAA2B,CAAC;QAE9B,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrD,MAAM;QACR,CAAC;QAED,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YACrC,IAAI,CAAC;gBACH,8DAA8D;gBAC9D,MAAM,IAAI,GAAG,CAAC,MAAM,MAAM,CAAC,MAAM,CAAC;oBAChC,CAAC,EAAE,SAAS;oBACZ,OAAO,EAAE,MAAM;iBAChB,CAAC,CAAQ,CAAC;gBAEX,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC9B,IAAI,IAAI,GAA6B,OAAO,CAAC;gBAC7C,IAAI,OAAO,GAAG,KAAK,CAAC;gBAEpB,IAAI,QAAQ,KAAK,oBAAoB,EAAE,CAAC;oBACtC,6DAA6D;oBAC7D,IAAI,CAAC;wBACH,8DAA8D;wBAC9D,MAAM,EAAE,GAAG,CAAC,MAAM,MAAM,CAAC,MAAM,CAAC;4BAC9B,CAAC,EAAE,eAAe;4BAClB,aAAa,EAAE,IAAI,CAAC,IAAI,CAAC,aAAa;yBACvC,CAAC,CAAQ,CAAC;wBAEX,IAAI,GAAG,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC;wBAChD,OAAO,GAAG,EAAE,CAAC,QAAQ,IAAI,KAAK,CAAC;oBACjC,CAAC;oBAAC,MAAM,CAAC;wBACP,IAAI,GAAG,YAAY,CAAC;oBACtB,CAAC;gBACH,CAAC;qBAAM,IAAI,QAAQ,KAAK,oBAAoB,EAAE,CAAC;oBAC7C,IAAI,GAAG,OAAO,CAAC;gBACjB,CAAC;qBAAM,IAAI,QAAQ,KAAK,iBAAiB,IAAI,QAAQ,KAAK,gBAAgB,EAAE,CAAC;oBAC3E,IAAI,GAAG,SAAS,CAAC;gBACnB,CAAC;gBAED,wCAAwC;gBACxC,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,YAAY,EAAE,CAAC;oBAChD,KAAK,CAAC,IAAI,CAAC;wBACT,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC;wBACtB,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,QAAQ,MAAM,EAAE;wBACrC,IAAI;wBACJ,OAAO;qBACR,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,GAAG,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,sCAAsC,CAAC,CAAC;YACpE,CAAC;QACH,CAAC;QAED,+EAA+E;QAC/E,oCAAoC;QACpC,OAAO,GAAG,KAAK,CAAC;QAEhB,MAAM,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IACjC,CAAC;IAED,GAAG,CAAC,IAAI,CACN,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,EACvB,4CAA4C,CAC7C,CAAC;IAEF,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,MAAc,EACd,MAAc;IAEd,8DAA8D;IAC9D,MAAM,MAAM,GAAG,CAAC,MAAM,MAAM,CAAC,MAAM,CAAC;QAClC,CAAC,EAAE,sBAAsB;QACzB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC;QACvB,IAAI,EAAE,wBAAwB;QAC9B,oBAAoB,EAAE,KAAK;KAC5B,CAAC,CAAQ,CAAC;IAEX,MAAM,IAAI,GAAG,MAAM,CAAC,WAAqB,CAAC;IAC1C,GAAG,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,EAAE,uBAAuB,CAAC,CAAC;IACvE,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,MAAc,EACd,KAAa;IAEb,8DAA8D;IAC9D,MAAM,MAAM,GAAG,CAAC,MAAM,MAAM,CAAC,MAAM,CAAC;QAClC,CAAC,EAAE,yBAAyB;QAC5B,KAAK;QACL,QAAQ,EAAE,KAAK;QACf,UAAU,EAAE,KAAK;QACjB,WAAW,EAAE,4DAA4D;KAC1E,CAAC,CAAQ,CAAC;IAEX,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACjC,GAAG,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,EAAE,wBAAwB,CAAC,CAAC;IACzE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,KAAK,EAAE,CAAC;AAClD,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,MAAc,EACd,UAAkB;IAElB,MAAM,MAAM,CAAC,MAAM,CAAC;QAClB,CAAC,EAAE,sBAAsB;QACzB,WAAW,EAAE,UAAU;KACxB,CAAC,CAAC;IACH,GAAG,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,EAAE,4BAA4B,CAAC,CAAC;AACzD,CAAC;AAED,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC"}
|
||||
18
worker/dist/tdlib/client.d.ts
vendored
18
worker/dist/tdlib/client.d.ts
vendored
@@ -1,18 +0,0 @@
|
||||
import { type Client } from "tdl";
|
||||
interface AccountConfig {
|
||||
id: string;
|
||||
phone: string;
|
||||
}
|
||||
/**
|
||||
* Create and authenticate a TDLib client for a Telegram account.
|
||||
* Authentication flow communicates with the admin UI via the database:
|
||||
* - Worker sets authState to AWAITING_CODE when TDLib asks for phone code
|
||||
* - Admin enters the code via UI, which writes it to authCode field
|
||||
* - Worker polls DB for the code and feeds it to TDLib
|
||||
*/
|
||||
export declare function createTdlibClient(account: AccountConfig): Promise<Client>;
|
||||
/**
|
||||
* Close a TDLib client gracefully.
|
||||
*/
|
||||
export declare function closeTdlibClient(client: Client): Promise<void>;
|
||||
export {};
|
||||
96
worker/dist/tdlib/client.js
vendored
96
worker/dist/tdlib/client.js
vendored
@@ -1,96 +0,0 @@
|
||||
import tdl, { createClient } from "tdl";
|
||||
import { getTdjson } from "prebuilt-tdlib";
|
||||
import path from "path";
|
||||
import { config } from "../util/config.js";
|
||||
import { childLogger } from "../util/logger.js";
|
||||
import { updateAccountAuthState, getAccountAuthCode, } from "../db/queries.js";
|
||||
const log = childLogger("tdlib-client");
|
||||
// Configure tdl to use the prebuilt tdjson shared library
|
||||
tdl.configure({ tdjson: getTdjson() });
|
||||
/**
|
||||
* Create and authenticate a TDLib client for a Telegram account.
|
||||
* Authentication flow communicates with the admin UI via the database:
|
||||
* - Worker sets authState to AWAITING_CODE when TDLib asks for phone code
|
||||
* - Admin enters the code via UI, which writes it to authCode field
|
||||
* - Worker polls DB for the code and feeds it to TDLib
|
||||
*/
|
||||
export async function createTdlibClient(account) {
|
||||
const dbPath = path.join(config.tdlibStateDir, account.id);
|
||||
const client = createClient({
|
||||
apiId: config.telegramApiId,
|
||||
apiHash: config.telegramApiHash,
|
||||
databaseDirectory: dbPath,
|
||||
filesDirectory: path.join(dbPath, "files"),
|
||||
});
|
||||
client.on("error", (err) => {
|
||||
log.error({ err, accountId: account.id }, "TDLib client error");
|
||||
});
|
||||
try {
|
||||
await client.login(() => ({
|
||||
getPhoneNumber: async () => {
|
||||
log.info({ accountId: account.id }, "TDLib requesting phone number");
|
||||
return account.phone;
|
||||
},
|
||||
getAuthCode: async () => {
|
||||
log.info({ accountId: account.id }, "TDLib requesting auth code");
|
||||
await updateAccountAuthState(account.id, "AWAITING_CODE");
|
||||
// Poll database for the code entered via admin UI
|
||||
const code = await pollForAuthCode(account.id);
|
||||
if (!code) {
|
||||
throw new Error("Auth code not provided within timeout");
|
||||
}
|
||||
// Clear the code after reading
|
||||
await updateAccountAuthState(account.id, "AUTHENTICATED", null);
|
||||
return code;
|
||||
},
|
||||
getPassword: async () => {
|
||||
log.info({ accountId: account.id }, "TDLib requesting 2FA password");
|
||||
await updateAccountAuthState(account.id, "AWAITING_PASSWORD");
|
||||
// Poll database for the password entered via admin UI
|
||||
const code = await pollForAuthCode(account.id);
|
||||
if (!code) {
|
||||
throw new Error("2FA password not provided within timeout");
|
||||
}
|
||||
await updateAccountAuthState(account.id, "AUTHENTICATED", null);
|
||||
return code;
|
||||
},
|
||||
}));
|
||||
await updateAccountAuthState(account.id, "AUTHENTICATED");
|
||||
log.info({ accountId: account.id }, "TDLib client authenticated");
|
||||
return client;
|
||||
}
|
||||
catch (err) {
|
||||
log.error({ err, accountId: account.id }, "TDLib authentication failed");
|
||||
await updateAccountAuthState(account.id, "EXPIRED");
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Poll the database every 5 seconds for an auth code, up to 5 minutes.
|
||||
*/
|
||||
async function pollForAuthCode(accountId, timeoutMs = 300_000) {
|
||||
const start = Date.now();
|
||||
while (Date.now() - start < timeoutMs) {
|
||||
const result = await getAccountAuthCode(accountId);
|
||||
if (result?.authCode) {
|
||||
return result.authCode;
|
||||
}
|
||||
await sleep(5000);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
function sleep(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
/**
|
||||
* Close a TDLib client gracefully.
|
||||
*/
|
||||
export async function closeTdlibClient(client) {
|
||||
try {
|
||||
await client.close();
|
||||
}
|
||||
catch (err) {
|
||||
log.warn({ err }, "Error closing TDLib client");
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=client.js.map
|
||||
1
worker/dist/tdlib/client.js.map
vendored
1
worker/dist/tdlib/client.js.map
vendored
@@ -1 +0,0 @@
|
||||
{"version":3,"file":"client.js","sourceRoot":"","sources":["../../src/tdlib/client.ts"],"names":[],"mappings":"AAAA,OAAO,GAAG,EAAE,EAAE,YAAY,EAAe,MAAM,KAAK,CAAC;AACrD,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EACL,sBAAsB,EACtB,kBAAkB,GACnB,MAAM,kBAAkB,CAAC;AAE1B,MAAM,GAAG,GAAG,WAAW,CAAC,cAAc,CAAC,CAAC;AAExC,0DAA0D;AAC1D,GAAG,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;AAOvC;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,OAAsB;IAEtB,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC;IAE3D,MAAM,MAAM,GAAG,YAAY,CAAC;QAC1B,KAAK,EAAE,MAAM,CAAC,aAAa;QAC3B,OAAO,EAAE,MAAM,CAAC,eAAe;QAC/B,iBAAiB,EAAE,MAAM;QACzB,cAAc,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC;KAC3C,CAAC,CAAC;IAEH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;QACzB,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,EAAE,oBAAoB,CAAC,CAAC;IAClE,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC;YACxB,cAAc,EAAE,KAAK,IAAI,EAAE;gBACzB,GAAG,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,EAAE,+BAA+B,CAAC,CAAC;gBACrE,OAAO,OAAO,CAAC,KAAK,CAAC;YACvB,CAAC;YACD,WAAW,EAAE,KAAK,IAAI,EAAE;gBACtB,GAAG,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,EAAE,4BAA4B,CAAC,CAAC;gBAClE,MAAM,sBAAsB,CAAC,OAAO,CAAC,EAAE,EAAE,eAAe,CAAC,CAAC;gBAE1D,kDAAkD;gBAClD,MAAM,IAAI,GAAG,MAAM,eAAe,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;gBAC/C,IAAI,CAAC,IAAI,EAAE,CAAC;oBACV,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;gBAC3D,CAAC;gBAED,+BAA+B;gBAC/B,MAAM,sBAAsB,CAAC,OAAO,CAAC,EAAE,EAAE,eAAe,EAAE,IAAI,CAAC,CAAC;gBAChE,OAAO,IAAI,CAAC;YACd,CAAC;YACD,WAAW,EAAE,KAAK,IAAI,EAAE;gBACtB,GAAG,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,EAAE,+BAA+B,CAAC,CAAC;gBACrE,MAAM,sBAAsB,CAAC,OAAO,CAAC,EAAE,EAAE,mBAAmB,CAAC,CAAC;gBAE9D,sDAAsD;gBACtD,MAAM,IAAI,GAAG,MAAM,eAAe,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;gBAC/C,IAAI,CAAC,IAAI,EAAE,CAAC;oBACV,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;gBAC9D,CAAC;gBAED,MAAM,sBAAsB,CAAC,OAAO,CAAC,EAAE,EAAE,eAAe,EAAE,IAAI,CAAC,CAAC;gBAChE,OAAO,IAAI,CAAC;YACd,CAAC;SACF,CAAC,CAAC,CAAC;QAEJ,MAAM,sBAAsB,CAAC,OAAO,CAAC,EAAE,EAAE,eAAe,CAAC,CAAC;QAC1D,GAAG,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,EAAE,4BAA4B,CAAC,CAAC;QAClE,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,EAAE,6BAA6B,CAAC,CAAC;QACzE,MAAM,sBAAsB,CAAC,OAAO,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;QACpD,MAAM,GAAG,CAAC;IACZ,CAAC;AACH,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,eAAe,CAC5B,SAAiB,EACjB,SAAS,GAAG,OAAO;IAEnB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,GAAG,SAAS,EAAE,CAAC;QACtC,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,SAAS,CAAC,CAAC;QACnD,IAAI,MAAM,EAAE,QAAQ,EAAE,CAAC;YACrB,OAAO,MAAM,CAAC,QAAQ,CAAC;QACzB,CAAC;QACD,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,MAAc;IACnD,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,EAAE,4BAA4B,CAAC,CAAC;IAClD,CAAC;AACH,CAAC"}
|
||||
67
worker/dist/tdlib/download.d.ts
vendored
67
worker/dist/tdlib/download.d.ts
vendored
@@ -1,67 +0,0 @@
|
||||
import type { Client } from "tdl";
|
||||
import type { TelegramMessage } from "../archive/multipart.js";
|
||||
import type { TelegramPhoto } from "../preview/match.js";
|
||||
/** Maximum number of pages to scan per channel/topic to prevent infinite loops */
|
||||
export declare const MAX_SCAN_PAGES = 5000;
|
||||
/** Timeout for a single TDLib API call (ms) */
|
||||
export declare const INVOKE_TIMEOUT_MS = 120000;
|
||||
export interface ChannelScanResult {
|
||||
archives: TelegramMessage[];
|
||||
photos: TelegramPhoto[];
|
||||
totalScanned: number;
|
||||
}
|
||||
export type ScanProgressCallback = (messagesScanned: number) => void;
|
||||
/**
|
||||
* Invoke a TDLib method with a timeout to prevent indefinite hangs.
|
||||
* If TDLib does not respond within the timeout, the promise rejects.
|
||||
*/
|
||||
export declare function invokeWithTimeout<T>(client: Client, request: Record<string, any>, timeoutMs?: number): Promise<T>;
|
||||
/**
|
||||
* Fetch messages from a channel, stopping once we've scanned past the
|
||||
* last-processed boundary (with one page of lookback for multipart safety).
|
||||
* Collects both archive attachments AND photo messages (for preview matching).
|
||||
* Returns messages in chronological order (oldest first).
|
||||
*
|
||||
* When `lastProcessedMessageId` is null (first run), scans everything.
|
||||
* The worker applies a post-grouping filter to skip fully-processed sets,
|
||||
* and keeps `packageExistsBySourceMessage` as a safety net.
|
||||
*
|
||||
* Safety features:
|
||||
* - Max page limit to prevent infinite loops
|
||||
* - Stuck detection: breaks if from_message_id stops advancing
|
||||
* - Timeout on each TDLib API call
|
||||
*/
|
||||
export declare function getChannelMessages(client: Client, chatId: bigint, lastProcessedMessageId?: bigint | null, limit?: number, onProgress?: ScanProgressCallback): Promise<ChannelScanResult>;
|
||||
/**
|
||||
* Download a photo thumbnail from Telegram and return its raw bytes.
|
||||
* Uses synchronous download (photos are small, typically < 100KB).
|
||||
* Returns null if download fails (non-critical).
|
||||
*/
|
||||
export declare function downloadPhotoThumbnail(client: Client, fileId: string): Promise<Buffer | null>;
|
||||
export interface DownloadProgress {
|
||||
fileId: string;
|
||||
fileName: string;
|
||||
downloadedBytes: number;
|
||||
totalBytes: number;
|
||||
percent: number;
|
||||
isComplete: boolean;
|
||||
}
|
||||
export type ProgressCallback = (progress: DownloadProgress) => void;
|
||||
/**
|
||||
* Download a file from Telegram to a local path with progress tracking
|
||||
* and integrity verification.
|
||||
*
|
||||
* Progress flow:
|
||||
* 1. Starts async download via TDLib
|
||||
* 2. Listens for `updateFile` events to track download progress
|
||||
* 3. Logs progress at every 10% increment
|
||||
* 4. Once complete, verifies the local file size matches the expected size
|
||||
* 5. Moves the file from TDLib's cache to the destination path
|
||||
*
|
||||
* Verification:
|
||||
* - Compares actual file size on disk to the expected size from Telegram
|
||||
* - Throws on mismatch (partial/corrupt download)
|
||||
* - Throws on timeout (configurable, scales with file size)
|
||||
* - Throws if download stops without completing (network error, etc.)
|
||||
*/
|
||||
export declare function downloadFile(client: Client, fileId: string, destPath: string, expectedSize: bigint, fileName: string, onProgress?: ProgressCallback): Promise<void>;
|
||||
307
worker/dist/tdlib/download.js
vendored
307
worker/dist/tdlib/download.js
vendored
@@ -1,307 +0,0 @@
|
||||
import { readFile, rename, copyFile, unlink, stat } from "fs/promises";
|
||||
import { config } from "../util/config.js";
|
||||
import { childLogger } from "../util/logger.js";
|
||||
import { isArchiveAttachment } from "../archive/detect.js";
|
||||
const log = childLogger("download");
|
||||
/** Maximum number of pages to scan per channel/topic to prevent infinite loops */
|
||||
export const MAX_SCAN_PAGES = 5000;
|
||||
/** Timeout for a single TDLib API call (ms) */
|
||||
export const INVOKE_TIMEOUT_MS = 120_000; // 2 minutes
|
||||
/**
|
||||
* Invoke a TDLib method with a timeout to prevent indefinite hangs.
|
||||
* If TDLib does not respond within the timeout, the promise rejects.
|
||||
*/
|
||||
export async function invokeWithTimeout(client,
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
request, timeoutMs = INVOKE_TIMEOUT_MS) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(() => {
|
||||
reject(new Error(`TDLib invoke timed out after ${timeoutMs}ms for ${request._}`));
|
||||
}, timeoutMs);
|
||||
client.invoke(request)
|
||||
.then((result) => {
|
||||
clearTimeout(timer);
|
||||
resolve(result);
|
||||
})
|
||||
.catch((err) => {
|
||||
clearTimeout(timer);
|
||||
reject(err);
|
||||
});
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Fetch messages from a channel, stopping once we've scanned past the
|
||||
* last-processed boundary (with one page of lookback for multipart safety).
|
||||
* Collects both archive attachments AND photo messages (for preview matching).
|
||||
* Returns messages in chronological order (oldest first).
|
||||
*
|
||||
* When `lastProcessedMessageId` is null (first run), scans everything.
|
||||
* The worker applies a post-grouping filter to skip fully-processed sets,
|
||||
* and keeps `packageExistsBySourceMessage` as a safety net.
|
||||
*
|
||||
* Safety features:
|
||||
* - Max page limit to prevent infinite loops
|
||||
* - Stuck detection: breaks if from_message_id stops advancing
|
||||
* - Timeout on each TDLib API call
|
||||
*/
|
||||
export async function getChannelMessages(client, chatId, lastProcessedMessageId, limit = 100, onProgress) {
|
||||
const archives = [];
|
||||
const photos = [];
|
||||
const boundary = lastProcessedMessageId ? Number(lastProcessedMessageId) : null;
|
||||
let currentFromId = 0;
|
||||
let totalScanned = 0;
|
||||
let pageCount = 0;
|
||||
// eslint-disable-next-line no-constant-condition
|
||||
while (true) {
|
||||
if (pageCount >= MAX_SCAN_PAGES) {
|
||||
log.warn({ chatId: chatId.toString(), pageCount, totalScanned }, "Hit max page limit for channel scan, stopping");
|
||||
break;
|
||||
}
|
||||
pageCount++;
|
||||
const previousFromId = currentFromId;
|
||||
const result = await invokeWithTimeout(client, {
|
||||
_: "getChatHistory",
|
||||
chat_id: Number(chatId),
|
||||
from_message_id: currentFromId,
|
||||
offset: 0,
|
||||
limit: Math.min(limit, 100),
|
||||
only_local: false,
|
||||
});
|
||||
if (!result.messages || result.messages.length === 0)
|
||||
break;
|
||||
totalScanned += result.messages.length;
|
||||
for (const msg of result.messages) {
|
||||
// Check for archive documents
|
||||
const doc = msg.content?.document;
|
||||
if (doc?.file_name && doc.document && isArchiveAttachment(doc.file_name)) {
|
||||
archives.push({
|
||||
id: BigInt(msg.id),
|
||||
fileName: doc.file_name,
|
||||
fileId: String(doc.document.id),
|
||||
fileSize: BigInt(doc.document.size),
|
||||
date: new Date(msg.date * 1000),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
// Check for photo messages (potential previews)
|
||||
const photo = msg.content?.photo;
|
||||
const caption = msg.content?.caption?.text ?? "";
|
||||
if (photo?.sizes && photo.sizes.length > 0) {
|
||||
const smallest = photo.sizes[0];
|
||||
photos.push({
|
||||
id: BigInt(msg.id),
|
||||
date: new Date(msg.date * 1000),
|
||||
caption,
|
||||
fileId: String(smallest.photo.id),
|
||||
fileSize: smallest.photo.size || smallest.photo.expected_size,
|
||||
});
|
||||
}
|
||||
}
|
||||
// Report scanning progress after each page
|
||||
onProgress?.(totalScanned);
|
||||
currentFromId = result.messages[result.messages.length - 1].id;
|
||||
// Stuck detection: if from_message_id didn't advance, break to prevent infinite loop
|
||||
if (currentFromId === previousFromId) {
|
||||
log.warn({ chatId: chatId.toString(), currentFromId, totalScanned }, "Pagination stuck (from_message_id not advancing), breaking");
|
||||
break;
|
||||
}
|
||||
// Stop scanning once we've gone past the boundary (this page is the lookback)
|
||||
if (boundary && currentFromId < boundary)
|
||||
break;
|
||||
if (result.messages.length < Math.min(limit, 100))
|
||||
break;
|
||||
// Rate limit delay
|
||||
await sleep(config.apiDelayMs);
|
||||
}
|
||||
log.info({ chatId: chatId.toString(), archives: archives.length, photos: photos.length, totalScanned, pages: pageCount }, "Channel scan complete");
|
||||
// Reverse to chronological order (oldest first) so worker processes old→new
|
||||
return {
|
||||
archives: archives.reverse(),
|
||||
photos: photos.reverse(),
|
||||
totalScanned,
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Download a photo thumbnail from Telegram and return its raw bytes.
|
||||
* Uses synchronous download (photos are small, typically < 100KB).
|
||||
* Returns null if download fails (non-critical).
|
||||
*/
|
||||
export async function downloadPhotoThumbnail(client, fileId) {
|
||||
const numericId = parseInt(fileId, 10);
|
||||
try {
|
||||
const result = (await client.invoke({
|
||||
_: "downloadFile",
|
||||
file_id: numericId,
|
||||
priority: 1, // Low priority — thumbnails are nice-to-have
|
||||
offset: 0,
|
||||
limit: 0,
|
||||
synchronous: true, // Small file — wait for it
|
||||
}));
|
||||
if (result?.local?.is_downloading_completed && result.local.path) {
|
||||
const data = await readFile(result.local.path);
|
||||
log.debug({ fileId, bytes: data.length }, "Downloaded photo thumbnail");
|
||||
return data;
|
||||
}
|
||||
}
|
||||
catch (err) {
|
||||
log.warn({ fileId, err }, "Failed to download photo thumbnail");
|
||||
}
|
||||
return null;
|
||||
}
|
||||
/**
|
||||
* Download a file from Telegram to a local path with progress tracking
|
||||
* and integrity verification.
|
||||
*
|
||||
* Progress flow:
|
||||
* 1. Starts async download via TDLib
|
||||
* 2. Listens for `updateFile` events to track download progress
|
||||
* 3. Logs progress at every 10% increment
|
||||
* 4. Once complete, verifies the local file size matches the expected size
|
||||
* 5. Moves the file from TDLib's cache to the destination path
|
||||
*
|
||||
* Verification:
|
||||
* - Compares actual file size on disk to the expected size from Telegram
|
||||
* - Throws on mismatch (partial/corrupt download)
|
||||
* - Throws on timeout (configurable, scales with file size)
|
||||
* - Throws if download stops without completing (network error, etc.)
|
||||
*/
|
||||
export async function downloadFile(client, fileId, destPath, expectedSize, fileName, onProgress) {
|
||||
const numericId = parseInt(fileId, 10);
|
||||
const totalBytes = Number(expectedSize);
|
||||
log.info({ fileId, fileName, destPath, totalBytes }, "Starting file download");
|
||||
// Report initial progress
|
||||
onProgress?.({
|
||||
fileId,
|
||||
fileName,
|
||||
downloadedBytes: 0,
|
||||
totalBytes,
|
||||
percent: 0,
|
||||
isComplete: false,
|
||||
});
|
||||
return new Promise((resolve, reject) => {
|
||||
let lastLoggedPercent = 0;
|
||||
let settled = false;
|
||||
// Timeout: 10 minutes per GB, minimum 5 minutes
|
||||
const timeoutMs = Math.max(5 * 60_000, (totalBytes / (1024 * 1024 * 1024)) * 10 * 60_000);
|
||||
const timer = setTimeout(() => {
|
||||
if (!settled) {
|
||||
settled = true;
|
||||
cleanup();
|
||||
reject(new Error(`Download timed out after ${Math.round(timeoutMs / 60_000)}min for ${fileName}`));
|
||||
}
|
||||
}, timeoutMs);
|
||||
// Listen for file update events to track progress
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const handleUpdate = (update) => {
|
||||
if (update?._ !== "updateFile")
|
||||
return;
|
||||
const file = update.file;
|
||||
if (!file || file.id !== numericId)
|
||||
return;
|
||||
const downloaded = file.local.downloaded_size;
|
||||
const percent = totalBytes > 0 ? Math.round((downloaded / totalBytes) * 100) : 0;
|
||||
// Log at every 10% increment
|
||||
if (percent >= lastLoggedPercent + 10) {
|
||||
lastLoggedPercent = percent - (percent % 10);
|
||||
log.info({ fileId, fileName, downloaded, totalBytes, percent: `${percent}%` }, "Download progress");
|
||||
}
|
||||
// Report to callback
|
||||
onProgress?.({
|
||||
fileId,
|
||||
fileName,
|
||||
downloadedBytes: downloaded,
|
||||
totalBytes,
|
||||
percent,
|
||||
isComplete: file.local.is_downloading_completed,
|
||||
});
|
||||
// Download finished
|
||||
if (file.local.is_downloading_completed) {
|
||||
if (!settled) {
|
||||
settled = true;
|
||||
cleanup();
|
||||
verifyAndMove(file.local.path, destPath, totalBytes, fileName, fileId)
|
||||
.then(resolve)
|
||||
.catch(reject);
|
||||
}
|
||||
}
|
||||
// Download stopped without completing (network error, cancelled, etc.)
|
||||
if (!file.local.is_downloading_active &&
|
||||
!file.local.is_downloading_completed) {
|
||||
if (!settled) {
|
||||
settled = true;
|
||||
cleanup();
|
||||
reject(new Error(`Download stopped unexpectedly for ${fileName} ` +
|
||||
`(${downloaded}/${totalBytes} bytes, ${percent}%)`));
|
||||
}
|
||||
}
|
||||
};
|
||||
const cleanup = () => {
|
||||
clearTimeout(timer);
|
||||
client.off("update", handleUpdate);
|
||||
};
|
||||
// Subscribe to updates BEFORE starting download
|
||||
client.on("update", handleUpdate);
|
||||
// Start async download (non-blocking — progress via updateFile events)
|
||||
client
|
||||
.invoke({
|
||||
_: "downloadFile",
|
||||
file_id: numericId,
|
||||
priority: 32,
|
||||
offset: 0,
|
||||
limit: 0,
|
||||
synchronous: false,
|
||||
})
|
||||
.then((result) => {
|
||||
// If the file was already cached locally, invoke returns immediately
|
||||
const file = result;
|
||||
if (file?.local?.is_downloading_completed && !settled) {
|
||||
settled = true;
|
||||
cleanup();
|
||||
verifyAndMove(file.local.path, destPath, totalBytes, fileName, fileId)
|
||||
.then(resolve)
|
||||
.catch(reject);
|
||||
}
|
||||
})
|
||||
.catch((err) => {
|
||||
if (!settled) {
|
||||
settled = true;
|
||||
cleanup();
|
||||
reject(err);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Verify the downloaded file's size matches the expected size,
|
||||
* then move it to the destination path.
|
||||
*/
|
||||
async function verifyAndMove(localPath, destPath, expectedBytes, fileName, fileId) {
|
||||
const stats = await stat(localPath);
|
||||
const actualBytes = stats.size;
|
||||
if (expectedBytes > 0 && actualBytes !== expectedBytes) {
|
||||
log.error({ fileId, fileName, expectedBytes, actualBytes }, "Download size mismatch — file is incomplete or corrupted");
|
||||
throw new Error(`Download verification failed for ${fileName}: ` +
|
||||
`expected ${expectedBytes} bytes, got ${actualBytes} bytes`);
|
||||
}
|
||||
log.info({ fileId, fileName, bytes: actualBytes, destPath }, "File verified and complete");
|
||||
// Move from TDLib's cache to our temp directory.
|
||||
// Use rename first (fast, same filesystem), fall back to copy+delete
|
||||
// when source and destination are on different filesystems (EXDEV).
|
||||
try {
|
||||
await rename(localPath, destPath);
|
||||
}
|
||||
catch (err) {
|
||||
if (err.code === "EXDEV") {
|
||||
log.debug({ fileId, fileName }, "Cross-device rename — falling back to copy + unlink");
|
||||
await copyFile(localPath, destPath);
|
||||
await unlink(localPath);
|
||||
}
|
||||
else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
function sleep(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
//# sourceMappingURL=download.js.map
|
||||
1
worker/dist/tdlib/download.js.map
vendored
1
worker/dist/tdlib/download.js.map
vendored
File diff suppressed because one or more lines are too long
32
worker/dist/tdlib/topics.d.ts
vendored
32
worker/dist/tdlib/topics.d.ts
vendored
@@ -1,32 +0,0 @@
|
||||
import type { Client } from "tdl";
|
||||
import type { ChannelScanResult, ScanProgressCallback } from "./download.js";
|
||||
export interface ForumTopic {
|
||||
topicId: bigint;
|
||||
name: string;
|
||||
}
|
||||
/**
|
||||
* Check if a chat is a forum supergroup (topics enabled).
|
||||
*/
|
||||
export declare function isChatForum(client: Client, chatId: bigint): Promise<boolean>;
|
||||
/**
|
||||
* Get all forum topics in a supergroup.
|
||||
* Includes stuck detection and timeout protection on API calls.
|
||||
*/
|
||||
export declare function getForumTopicList(client: Client, chatId: bigint): Promise<ForumTopic[]>;
|
||||
/**
|
||||
* Fetch messages from a specific forum topic (thread), stopping once
|
||||
* we've scanned past the last-processed boundary (with one page of lookback).
|
||||
* Uses searchChatMessages with message_thread_id to scan within a topic.
|
||||
*
|
||||
* Returns messages in chronological order (oldest first).
|
||||
*
|
||||
* When `lastProcessedMessageId` is null (first run), scans everything.
|
||||
* The worker applies a post-grouping filter to skip fully-processed sets,
|
||||
* and keeps `packageExistsBySourceMessage` as a safety net.
|
||||
*
|
||||
* Safety features:
|
||||
* - Max page limit to prevent infinite loops
|
||||
* - Stuck detection: breaks if from_message_id stops advancing
|
||||
* - Timeout on each TDLib API call
|
||||
*/
|
||||
export declare function getTopicMessages(client: Client, chatId: bigint, topicId: bigint, lastProcessedMessageId?: bigint | null, limit?: number, onProgress?: ScanProgressCallback): Promise<ChannelScanResult>;
|
||||
196
worker/dist/tdlib/topics.js
vendored
196
worker/dist/tdlib/topics.js
vendored
@@ -1,196 +0,0 @@
|
||||
import { config } from "../util/config.js";
|
||||
import { childLogger } from "../util/logger.js";
|
||||
import { isArchiveAttachment } from "../archive/detect.js";
|
||||
import { invokeWithTimeout, MAX_SCAN_PAGES } from "./download.js";
|
||||
const log = childLogger("topics");
|
||||
/**
|
||||
* Check if a chat is a forum supergroup (topics enabled).
|
||||
*/
|
||||
export async function isChatForum(client, chatId) {
|
||||
try {
|
||||
const chat = await invokeWithTimeout(client, {
|
||||
_: "getChat",
|
||||
chat_id: Number(chatId),
|
||||
});
|
||||
if (chat.type?._ === "chatTypeSupergroup" && chat.type.is_forum) {
|
||||
return true;
|
||||
}
|
||||
// Also check via getSupergroup for older TDLib versions
|
||||
if (chat.type?._ === "chatTypeSupergroup" && chat.type.supergroup_id) {
|
||||
const sg = await invokeWithTimeout(client, {
|
||||
_: "getSupergroup",
|
||||
supergroup_id: chat.type.supergroup_id,
|
||||
});
|
||||
return sg.is_forum === true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
catch (err) {
|
||||
log.warn({ err, chatId: chatId.toString() }, "Failed to check if chat is forum");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Get all forum topics in a supergroup.
|
||||
* Includes stuck detection and timeout protection on API calls.
|
||||
*/
|
||||
export async function getForumTopicList(client, chatId) {
|
||||
const topics = [];
|
||||
let offsetDate = 0;
|
||||
let offsetMessageId = 0;
|
||||
let offsetMessageThreadId = 0;
|
||||
let pageCount = 0;
|
||||
// eslint-disable-next-line no-constant-condition
|
||||
while (true) {
|
||||
if (pageCount >= MAX_SCAN_PAGES) {
|
||||
log.warn({ chatId: chatId.toString(), pageCount, topicCount: topics.length }, "Hit max page limit for topic enumeration, stopping");
|
||||
break;
|
||||
}
|
||||
pageCount++;
|
||||
const prevOffsetDate = offsetDate;
|
||||
const prevOffsetMessageId = offsetMessageId;
|
||||
const prevOffsetMessageThreadId = offsetMessageThreadId;
|
||||
const result = await invokeWithTimeout(client, {
|
||||
_: "getForumTopics",
|
||||
chat_id: Number(chatId),
|
||||
query: "",
|
||||
offset_date: offsetDate,
|
||||
offset_message_id: offsetMessageId,
|
||||
offset_message_thread_id: offsetMessageThreadId,
|
||||
limit: 100,
|
||||
});
|
||||
if (!result.topics || result.topics.length === 0)
|
||||
break;
|
||||
for (const t of result.topics) {
|
||||
if (!t.info?.message_thread_id)
|
||||
continue;
|
||||
// Skip the "General" topic — it's not creator-specific
|
||||
if (t.info.is_general)
|
||||
continue;
|
||||
topics.push({
|
||||
topicId: BigInt(t.info.message_thread_id),
|
||||
name: t.info.name ?? "Unnamed",
|
||||
});
|
||||
}
|
||||
// Check if there are more pages
|
||||
if (!result.next_offset_date &&
|
||||
!result.next_offset_message_id &&
|
||||
!result.next_offset_message_thread_id) {
|
||||
break;
|
||||
}
|
||||
offsetDate = result.next_offset_date ?? 0;
|
||||
offsetMessageId = result.next_offset_message_id ?? 0;
|
||||
offsetMessageThreadId = result.next_offset_message_thread_id ?? 0;
|
||||
// Stuck detection: if offsets didn't advance, break
|
||||
if (offsetDate === prevOffsetDate &&
|
||||
offsetMessageId === prevOffsetMessageId &&
|
||||
offsetMessageThreadId === prevOffsetMessageThreadId) {
|
||||
log.warn({ chatId: chatId.toString(), topicCount: topics.length }, "Topic pagination stuck (offsets not advancing), breaking");
|
||||
break;
|
||||
}
|
||||
await sleep(config.apiDelayMs);
|
||||
}
|
||||
log.info({ chatId: chatId.toString(), topicCount: topics.length }, "Enumerated forum topics");
|
||||
return topics;
|
||||
}
|
||||
/**
|
||||
* Fetch messages from a specific forum topic (thread), stopping once
|
||||
* we've scanned past the last-processed boundary (with one page of lookback).
|
||||
* Uses searchChatMessages with message_thread_id to scan within a topic.
|
||||
*
|
||||
* Returns messages in chronological order (oldest first).
|
||||
*
|
||||
* When `lastProcessedMessageId` is null (first run), scans everything.
|
||||
* The worker applies a post-grouping filter to skip fully-processed sets,
|
||||
* and keeps `packageExistsBySourceMessage` as a safety net.
|
||||
*
|
||||
* Safety features:
|
||||
* - Max page limit to prevent infinite loops
|
||||
* - Stuck detection: breaks if from_message_id stops advancing
|
||||
* - Timeout on each TDLib API call
|
||||
*/
|
||||
export async function getTopicMessages(client, chatId, topicId, lastProcessedMessageId, limit = 100, onProgress) {
|
||||
const archives = [];
|
||||
const photos = [];
|
||||
const boundary = lastProcessedMessageId ? Number(lastProcessedMessageId) : null;
|
||||
let currentFromId = 0;
|
||||
let totalScanned = 0;
|
||||
let pageCount = 0;
|
||||
// eslint-disable-next-line no-constant-condition
|
||||
while (true) {
|
||||
if (pageCount >= MAX_SCAN_PAGES) {
|
||||
log.warn({ chatId: chatId.toString(), topicId: topicId.toString(), pageCount, totalScanned }, "Hit max page limit for topic scan, stopping");
|
||||
break;
|
||||
}
|
||||
pageCount++;
|
||||
const previousFromId = currentFromId;
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const result = await invokeWithTimeout(client, {
|
||||
_: "searchChatMessages",
|
||||
chat_id: Number(chatId),
|
||||
query: "",
|
||||
message_thread_id: Number(topicId),
|
||||
from_message_id: currentFromId,
|
||||
offset: 0,
|
||||
limit: Math.min(limit, 100),
|
||||
filter: null,
|
||||
sender_id: null,
|
||||
saved_messages_topic_id: 0,
|
||||
});
|
||||
if (!result.messages || result.messages.length === 0)
|
||||
break;
|
||||
totalScanned += result.messages.length;
|
||||
for (const msg of result.messages) {
|
||||
// Check for archive documents
|
||||
const doc = msg.content?.document;
|
||||
if (doc?.file_name && doc.document && isArchiveAttachment(doc.file_name)) {
|
||||
archives.push({
|
||||
id: BigInt(msg.id),
|
||||
fileName: doc.file_name,
|
||||
fileId: String(doc.document.id),
|
||||
fileSize: BigInt(doc.document.size),
|
||||
date: new Date(msg.date * 1000),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
// Check for photo messages (potential previews)
|
||||
const photo = msg.content?.photo;
|
||||
const caption = msg.content?.caption?.text ?? "";
|
||||
if (photo?.sizes && photo.sizes.length > 0) {
|
||||
const smallest = photo.sizes[0];
|
||||
photos.push({
|
||||
id: BigInt(msg.id),
|
||||
date: new Date(msg.date * 1000),
|
||||
caption,
|
||||
fileId: String(smallest.photo.id),
|
||||
fileSize: smallest.photo.size || smallest.photo.expected_size,
|
||||
});
|
||||
}
|
||||
}
|
||||
// Report scanning progress after each page
|
||||
onProgress?.(totalScanned);
|
||||
currentFromId = result.messages[result.messages.length - 1].id;
|
||||
// Stuck detection: if from_message_id didn't advance, break to prevent infinite loop
|
||||
if (currentFromId === previousFromId) {
|
||||
log.warn({ chatId: chatId.toString(), topicId: topicId.toString(), currentFromId, totalScanned }, "Topic pagination stuck (from_message_id not advancing), breaking");
|
||||
break;
|
||||
}
|
||||
// Stop scanning once we've gone past the boundary (this page is the lookback)
|
||||
if (boundary && currentFromId < boundary)
|
||||
break;
|
||||
if (result.messages.length < Math.min(limit, 100))
|
||||
break;
|
||||
await sleep(config.apiDelayMs);
|
||||
}
|
||||
log.info({ chatId: chatId.toString(), topicId: topicId.toString(), archives: archives.length, photos: photos.length, totalScanned, pages: pageCount }, "Topic scan complete");
|
||||
// Reverse to chronological order (oldest first) so worker processes old→new
|
||||
return {
|
||||
archives: archives.reverse(),
|
||||
photos: photos.reverse(),
|
||||
totalScanned,
|
||||
};
|
||||
}
|
||||
function sleep(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
//# sourceMappingURL=topics.js.map
|
||||
1
worker/dist/tdlib/topics.js.map
vendored
1
worker/dist/tdlib/topics.js.map
vendored
File diff suppressed because one or more lines are too long
16
worker/dist/upload/channel.d.ts
vendored
16
worker/dist/upload/channel.d.ts
vendored
@@ -1,16 +0,0 @@
|
||||
import type { Client } from "tdl";
|
||||
export interface UploadResult {
|
||||
messageId: bigint;
|
||||
}
|
||||
/**
|
||||
* Upload one or more files to a destination Telegram channel.
|
||||
* For multipart archives, each file is sent as a separate message.
|
||||
* Returns the **final** (server-assigned) message ID of the first uploaded message.
|
||||
*
|
||||
* IMPORTANT: `sendMessage` returns a *temporary* message immediately.
|
||||
* The actual file upload happens asynchronously in TDLib. We listen for
|
||||
* `updateMessageSendSucceeded` to get the real server-side message ID and
|
||||
* to make sure the upload is fully committed before we clean up temp files
|
||||
* or close the TDLib client (which would cancel pending uploads).
|
||||
*/
|
||||
export declare function uploadToChannel(client: Client, chatId: bigint, filePaths: string[], caption?: string): Promise<UploadResult>;
|
||||
137
worker/dist/upload/channel.js
vendored
137
worker/dist/upload/channel.js
vendored
@@ -1,137 +0,0 @@
|
||||
import path from "path";
|
||||
import { stat } from "fs/promises";
|
||||
import { config } from "../util/config.js";
|
||||
import { childLogger } from "../util/logger.js";
|
||||
const log = childLogger("upload");
|
||||
/**
|
||||
* Upload one or more files to a destination Telegram channel.
|
||||
* For multipart archives, each file is sent as a separate message.
|
||||
* Returns the **final** (server-assigned) message ID of the first uploaded message.
|
||||
*
|
||||
* IMPORTANT: `sendMessage` returns a *temporary* message immediately.
|
||||
* The actual file upload happens asynchronously in TDLib. We listen for
|
||||
* `updateMessageSendSucceeded` to get the real server-side message ID and
|
||||
* to make sure the upload is fully committed before we clean up temp files
|
||||
* or close the TDLib client (which would cancel pending uploads).
|
||||
*/
|
||||
export async function uploadToChannel(client, chatId, filePaths, caption) {
|
||||
let firstMessageId = null;
|
||||
for (let i = 0; i < filePaths.length; i++) {
|
||||
const filePath = filePaths[i];
|
||||
const fileCaption = i === 0 && caption ? caption : undefined;
|
||||
const fileName = path.basename(filePath);
|
||||
let fileSizeMB = 0;
|
||||
try {
|
||||
const s = await stat(filePath);
|
||||
fileSizeMB = Math.round(s.size / (1024 * 1024));
|
||||
}
|
||||
catch {
|
||||
// Non-critical
|
||||
}
|
||||
log.info({ chatId: Number(chatId), fileName, sizeMB: fileSizeMB, part: i + 1, total: filePaths.length }, "Uploading file to channel");
|
||||
const serverMsgId = await sendAndWaitForUpload(client, chatId, filePath, fileCaption, fileName, fileSizeMB);
|
||||
if (i === 0) {
|
||||
firstMessageId = serverMsgId;
|
||||
}
|
||||
// Rate limit delay between uploads
|
||||
if (i < filePaths.length - 1) {
|
||||
await sleep(config.apiDelayMs);
|
||||
}
|
||||
}
|
||||
if (firstMessageId === null) {
|
||||
throw new Error("Upload failed: no messages sent");
|
||||
}
|
||||
log.info({ chatId: Number(chatId), messageId: Number(firstMessageId), files: filePaths.length }, "All uploads confirmed by Telegram");
|
||||
return { messageId: firstMessageId };
|
||||
}
|
||||
/**
|
||||
* Send a single file message and wait for Telegram to confirm the upload.
|
||||
* Returns the final server-assigned message ID.
|
||||
*/
|
||||
async function sendAndWaitForUpload(client, chatId, filePath, caption, fileName, fileSizeMB) {
|
||||
// Send the message — this returns a temporary message immediately
|
||||
const tempMsg = (await client.invoke({
|
||||
_: "sendMessage",
|
||||
chat_id: Number(chatId),
|
||||
input_message_content: {
|
||||
_: "inputMessageDocument",
|
||||
document: {
|
||||
_: "inputFileLocal",
|
||||
path: filePath,
|
||||
},
|
||||
caption: caption
|
||||
? {
|
||||
_: "formattedText",
|
||||
text: caption,
|
||||
}
|
||||
: undefined,
|
||||
},
|
||||
}));
|
||||
const tempMsgId = tempMsg.id;
|
||||
log.debug({ fileName, tempMsgId }, "Message queued, waiting for upload confirmation");
|
||||
// Wait for the actual upload to complete
|
||||
return new Promise((resolve, reject) => {
|
||||
let settled = false;
|
||||
let lastLoggedPercent = 0;
|
||||
// Timeout: 10 minutes per GB, minimum 10 minutes
|
||||
const timeoutMs = Math.max(10 * 60_000, (fileSizeMB / 1024) * 10 * 60_000);
|
||||
const timer = setTimeout(() => {
|
||||
if (!settled) {
|
||||
settled = true;
|
||||
cleanup();
|
||||
reject(new Error(`Upload timed out after ${Math.round(timeoutMs / 60_000)}min for ${fileName}`));
|
||||
}
|
||||
}, timeoutMs);
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const handleUpdate = (update) => {
|
||||
// Track upload progress via updateFile events
|
||||
if (update?._ === "updateFile") {
|
||||
const file = update.file;
|
||||
if (file?.remote?.is_uploading_active && file.expected_size > 0) {
|
||||
const uploaded = file.remote.uploaded_size ?? 0;
|
||||
const total = file.expected_size;
|
||||
const percent = Math.round((uploaded / total) * 100);
|
||||
if (percent >= lastLoggedPercent + 20) {
|
||||
lastLoggedPercent = percent - (percent % 20);
|
||||
log.info({ fileName, uploaded, total, percent: `${percent}%` }, "Upload progress");
|
||||
}
|
||||
}
|
||||
}
|
||||
// The money event: upload succeeded, we get the final server message ID
|
||||
if (update?._ === "updateMessageSendSucceeded") {
|
||||
const msg = update.message;
|
||||
const oldMsgId = update.old_message_id;
|
||||
if (oldMsgId === tempMsgId) {
|
||||
if (!settled) {
|
||||
settled = true;
|
||||
cleanup();
|
||||
const finalId = BigInt(msg.id);
|
||||
log.info({ fileName, tempMsgId, finalMsgId: Number(finalId) }, "Upload confirmed by Telegram");
|
||||
resolve(finalId);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Upload failed
|
||||
if (update?._ === "updateMessageSendFailed") {
|
||||
const oldMsgId = update.old_message_id;
|
||||
if (oldMsgId === tempMsgId) {
|
||||
if (!settled) {
|
||||
settled = true;
|
||||
cleanup();
|
||||
const errorMsg = update.error?.message ?? "Unknown upload error";
|
||||
reject(new Error(`Upload failed for ${fileName}: ${errorMsg}`));
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
const cleanup = () => {
|
||||
clearTimeout(timer);
|
||||
client.off("update", handleUpdate);
|
||||
};
|
||||
client.on("update", handleUpdate);
|
||||
});
|
||||
}
|
||||
function sleep(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
//# sourceMappingURL=channel.js.map
|
||||
1
worker/dist/upload/channel.js.map
vendored
1
worker/dist/upload/channel.js.map
vendored
@@ -1 +0,0 @@
|
||||
{"version":3,"file":"channel.js","sourceRoot":"","sources":["../../src/upload/channel.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AAEnC,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,MAAM,GAAG,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC;AAMlC;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,MAAc,EACd,MAAc,EACd,SAAmB,EACnB,OAAgB;IAEhB,IAAI,cAAc,GAAkB,IAAI,CAAC;IAEzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;QAC9B,MAAM,WAAW,GACf,CAAC,KAAK,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;QAE3C,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACzC,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,CAAC;YACH,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC/B,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC;QAClD,CAAC;QAAC,MAAM,CAAC;YACP,eAAe;QACjB,CAAC;QAED,GAAG,CAAC,IAAI,CACN,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,SAAS,CAAC,MAAM,EAAE,EAC9F,2BAA2B,CAC5B,CAAC;QAEF,MAAM,WAAW,GAAG,MAAM,oBAAoB,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;QAE5G,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YACZ,cAAc,GAAG,WAAW,CAAC;QAC/B,CAAC;QAED,mCAAmC;QACnC,IAAI,CAAC,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7B,MAAM,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;IAED,IAAI,cAAc,KAAK,IAAI,EAAE,CAAC;QAC5B,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;IACrD,CAAC;IAED,GAAG,CAAC,IAAI,CACN,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,cAAc,CAAC,EAAE,KAAK,EAAE,SAAS,CAAC,MAAM,EAAE,EACtF,mCAAmC,CACpC,CAAC;IAEF,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,CAAC;AACvC,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,oBAAoB,CACjC,MAAc,EACd,MAAc,EACd,QAAgB,EAChB,OAA2B,EAC3B,QAAgB,EAChB,UAAkB;IAElB,kEAAkE;IAClE,MAAM,OAAO,GAAG,CAAC,MAAM,MAAM,CAAC,MAAM,CAAC;QACnC,CAAC,EAAE,aAAa;QAChB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC;QACvB,qBAAqB,EAAE;YACrB,CAAC,EAAE,sBAAsB;YACzB,QAAQ,EAAE;gBACR,CAAC,EAAE,gBAAgB;gBACnB,IAAI,EAAE,QAAQ;aACf;YACD,OAAO,EAAE,OAAO;gBACd,CAAC,CAAC;oBACE,CAAC,EAAE,eAAe;oBAClB,IAAI,EAAE,OAAO;iBACd;gBACH,CAAC,CAAC,SAAS;SACd;KACF,CAAC,CAAmB,CAAC;IAEtB,MAAM,SAAS,GAAG,OAAO,CAAC,EAAE,CAAC;IAE7B,GAAG,CAAC,KAAK,CACP,EAAE,QAAQ,EAAE,SAAS,EAAE,EACvB,iDAAiD,CAClD,CAAC;IAEF,yCAAyC;IACzC,OAAO,IAAI,OAAO,CAAS,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC7C,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,IAAI,iBAAiB,GAAG,CAAC,CAAC;QAE1B,iDAAiD;QACjD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CACxB,EAAE,GAAG,MAAM,EACX,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAClC,CAAC;QAEF,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,OAAO,GAAG,IAAI,CAAC;gBACf,OAAO,EAAE,CAAC;gBACV,MAAM,CACJ,IAAI,KAAK,CACP,0BAA0B,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,MAAM,CAAC,WAAW,QAAQ,EAAE,CAC9E,CACF,CAAC;YACJ,CAAC;QACH,CAAC,EAAE,SAAS,CAAC,CAAC;QAEd,8DAA8D;QAC9D,MAAM,YAAY,GAAG,CAAC,MAAW,EAAE,EAAE;YACnC,8CAA8C;YAC9C,IAAI,MAAM,EAAE,CAAC,KAAK,YAAY,EAAE,CAAC;gBAC/B,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;gBACzB,IAAI,IAAI,EAAE,MAAM,EAAE,mBAAmB,IAAI,IAAI,CAAC,aAAa,GAAG,CAAC,EAAE,CAAC;oBAChE,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa,IAAI,CAAC,CAAC;oBAChD,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC;oBACjC,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC;oBACrD,IAAI,OAAO,IAAI,iBAAiB,GAAG,EAAE,EAAE,CAAC;wBACtC,iBAAiB,GAAG,OAAO,GAAG,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC;wBAC7C,GAAG,CAAC,IAAI,CACN,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,OAAO,GAAG,EAAE,EACrD,iBAAiB,CAClB,CAAC;oBACJ,CAAC;gBACH,CAAC;YACH,CAAC;YAED,wEAAwE;YACxE,IAAI,MAAM,EAAE,CAAC,KAAK,4BAA4B,EAAE,CAAC;gBAC/C,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC;gBAC3B,MAAM,QAAQ,GAAG,MAAM,CAAC,cAAc,CAAC;gBACvC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;oBAC3B,IAAI,CAAC,OAAO,EAAE,CAAC;wBACb,OAAO,GAAG,IAAI,CAAC;wBACf,OAAO,EAAE,CAAC;wBACV,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;wBAC/B,GAAG,CAAC,IAAI,CACN,EAAE,QAAQ,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,CAAC,OAAO,CAAC,EAAE,EACpD,8BAA8B,CAC/B,CAAC;wBACF,OAAO,CAAC,OAAO,CAAC,CAAC;oBACnB,CAAC;gBACH,CAAC;YACH,CAAC;YAED,gBAAgB;YAChB,IAAI,MAAM,EAAE,CAAC,KAAK,yBAAyB,EAAE,CAAC;gBAC5C,MAAM,QAAQ,GAAG,MAAM,CAAC,cAAc,CAAC;gBACvC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;oBAC3B,IAAI,CAAC,OAAO,EAAE,CAAC;wBACb,OAAO,GAAG,IAAI,CAAC;wBACf,OAAO,EAAE,CAAC;wBACV,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,EAAE,OAAO,IAAI,sBAAsB,CAAC;wBACjE,MAAM,CAAC,IAAI,KAAK,CAAC,qBAAqB,QAAQ,KAAK,QAAQ,EAAE,CAAC,CAAC,CAAC;oBAClE,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC,CAAC;QAEF,MAAM,OAAO,GAAG,GAAG,EAAE;YACnB,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;QACrC,CAAC,CAAC;QAEF,MAAM,CAAC,EAAE,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IACpC,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC"}
|
||||
18
worker/dist/util/config.d.ts
vendored
18
worker/dist/util/config.d.ts
vendored
@@ -1,18 +0,0 @@
|
||||
export declare const config: {
|
||||
readonly databaseUrl: string;
|
||||
readonly workerIntervalMinutes: number;
|
||||
readonly tempDir: string;
|
||||
readonly tdlibStateDir: string;
|
||||
readonly maxZipSizeMB: number;
|
||||
readonly logLevel: "debug" | "info" | "warn" | "error";
|
||||
readonly telegramApiId: number;
|
||||
readonly telegramApiHash: string;
|
||||
/** Maximum jitter added to scheduler interval (in minutes) */
|
||||
readonly jitterMinutes: 5;
|
||||
/** Maximum time span for multipart archive parts (in hours). 0 = no limit. */
|
||||
readonly multipartTimeoutHours: number;
|
||||
/** Delay between Telegram API calls (in ms) to avoid rate limits */
|
||||
readonly apiDelayMs: 1000;
|
||||
/** Max retries for rate-limited requests */
|
||||
readonly maxRetries: 5;
|
||||
};
|
||||
19
worker/dist/util/config.js
vendored
19
worker/dist/util/config.js
vendored
@@ -1,19 +0,0 @@
|
||||
export const config = {
|
||||
databaseUrl: process.env.DATABASE_URL ?? "",
|
||||
workerIntervalMinutes: parseInt(process.env.WORKER_INTERVAL_MINUTES ?? "60", 10),
|
||||
tempDir: process.env.WORKER_TEMP_DIR ?? "/tmp/zips",
|
||||
tdlibStateDir: process.env.TDLIB_STATE_DIR ?? "/data/tdlib",
|
||||
maxZipSizeMB: parseInt(process.env.WORKER_MAX_ZIP_SIZE_MB ?? "4096", 10),
|
||||
logLevel: (process.env.LOG_LEVEL ?? "info"),
|
||||
telegramApiId: parseInt(process.env.TELEGRAM_API_ID ?? "0", 10),
|
||||
telegramApiHash: process.env.TELEGRAM_API_HASH ?? "",
|
||||
/** Maximum jitter added to scheduler interval (in minutes) */
|
||||
jitterMinutes: 5,
|
||||
/** Maximum time span for multipart archive parts (in hours). 0 = no limit. */
|
||||
multipartTimeoutHours: parseInt(process.env.MULTIPART_TIMEOUT_HOURS ?? "0", 10),
|
||||
/** Delay between Telegram API calls (in ms) to avoid rate limits */
|
||||
apiDelayMs: 1000,
|
||||
/** Max retries for rate-limited requests */
|
||||
maxRetries: 5,
|
||||
};
|
||||
//# sourceMappingURL=config.js.map
|
||||
1
worker/dist/util/config.js.map
vendored
1
worker/dist/util/config.js.map
vendored
@@ -1 +0,0 @@
|
||||
{"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/util/config.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,MAAM,GAAG;IACpB,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,EAAE;IAC3C,qBAAqB,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB,IAAI,IAAI,EAAE,EAAE,CAAC;IAChF,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,WAAW;IACnD,aAAa,EAAE,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,aAAa;IAC3D,YAAY,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,sBAAsB,IAAI,MAAM,EAAE,EAAE,CAAC;IACxE,QAAQ,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,MAAM,CAAwC;IAClF,aAAa,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,GAAG,EAAE,EAAE,CAAC;IAC/D,eAAe,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,EAAE;IACpD,8DAA8D;IAC9D,aAAa,EAAE,CAAC;IAChB,8EAA8E;IAC9E,qBAAqB,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB,IAAI,GAAG,EAAE,EAAE,CAAC;IAC/E,oEAAoE;IACpE,UAAU,EAAE,IAAI;IAChB,4CAA4C;IAC5C,UAAU,EAAE,CAAC;CACL,CAAC"}
|
||||
3
worker/dist/util/logger.d.ts
vendored
3
worker/dist/util/logger.d.ts
vendored
@@ -1,3 +0,0 @@
|
||||
import pino from "pino";
|
||||
export declare const logger: pino.Logger<never, boolean>;
|
||||
export declare function childLogger(name: string, extra?: Record<string, unknown>): pino.Logger<never, boolean>;
|
||||
12
worker/dist/util/logger.js
vendored
12
worker/dist/util/logger.js
vendored
@@ -1,12 +0,0 @@
|
||||
import pino from "pino";
|
||||
import { config } from "./config.js";
|
||||
export const logger = pino({
|
||||
level: config.logLevel,
|
||||
transport: config.logLevel === "debug"
|
||||
? { target: "pino/file", options: { destination: 1 } }
|
||||
: undefined,
|
||||
});
|
||||
export function childLogger(name, extra) {
|
||||
return logger.child({ module: name, ...extra });
|
||||
}
|
||||
//# sourceMappingURL=logger.js.map
|
||||
1
worker/dist/util/logger.js.map
vendored
1
worker/dist/util/logger.js.map
vendored
@@ -1 +0,0 @@
|
||||
{"version":3,"file":"logger.js","sourceRoot":"","sources":["../../src/util/logger.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC,MAAM,CAAC,MAAM,MAAM,GAAG,IAAI,CAAC;IACzB,KAAK,EAAE,MAAM,CAAC,QAAQ;IACtB,SAAS,EACP,MAAM,CAAC,QAAQ,KAAK,OAAO;QACzB,CAAC,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,EAAE;QACtD,CAAC,CAAC,SAAS;CAChB,CAAC,CAAC;AAEH,MAAM,UAAU,WAAW,CAAC,IAAY,EAAE,KAA+B;IACvE,OAAO,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,KAAK,EAAE,CAAC,CAAC;AAClD,CAAC"}
|
||||
8
worker/dist/util/mutex.d.ts
vendored
8
worker/dist/util/mutex.d.ts
vendored
@@ -1,8 +0,0 @@
|
||||
/**
|
||||
* Ensures only one TDLib client runs at a time across the entire worker process.
|
||||
* Both the scheduler (auth, ingestion) and the fetch listener acquire this
|
||||
* before creating any TDLib client.
|
||||
*
|
||||
* Includes a wait timeout to prevent indefinite blocking if the current holder hangs.
|
||||
*/
|
||||
export declare function withTdlibMutex<T>(label: string, fn: () => Promise<T>): Promise<T>;
|
||||
61
worker/dist/util/mutex.js
vendored
61
worker/dist/util/mutex.js
vendored
@@ -1,61 +0,0 @@
|
||||
import { childLogger } from "./logger.js";
|
||||
const log = childLogger("mutex");
|
||||
let locked = false;
|
||||
let holder = "";
|
||||
const queue = [];
|
||||
/**
|
||||
* Maximum time to wait for the TDLib mutex (ms).
|
||||
* If the mutex is not available within this time, the operation is rejected.
|
||||
* Default: 30 minutes (long enough for large downloads, short enough to detect hangs).
|
||||
*/
|
||||
const MUTEX_WAIT_TIMEOUT_MS = 30 * 60 * 1000;
|
||||
/**
|
||||
* Ensures only one TDLib client runs at a time across the entire worker process.
|
||||
* Both the scheduler (auth, ingestion) and the fetch listener acquire this
|
||||
* before creating any TDLib client.
|
||||
*
|
||||
* Includes a wait timeout to prevent indefinite blocking if the current holder hangs.
|
||||
*/
|
||||
export async function withTdlibMutex(label, fn) {
|
||||
if (locked) {
|
||||
log.info({ waiting: label, holder }, "Waiting for TDLib mutex");
|
||||
await new Promise((resolve, reject) => {
|
||||
const entry = { resolve, reject, label };
|
||||
queue.push(entry);
|
||||
// Timeout: reject if we've been waiting too long
|
||||
const timer = setTimeout(() => {
|
||||
const idx = queue.indexOf(entry);
|
||||
if (idx !== -1) {
|
||||
queue.splice(idx, 1);
|
||||
reject(new Error(`TDLib mutex wait timeout after ${MUTEX_WAIT_TIMEOUT_MS / 60_000}min ` +
|
||||
`(waiting: ${label}, holder: ${holder})`));
|
||||
}
|
||||
}, MUTEX_WAIT_TIMEOUT_MS);
|
||||
// Wrap resolve to clear the timer
|
||||
const origResolve = entry.resolve;
|
||||
entry.resolve = () => {
|
||||
clearTimeout(timer);
|
||||
origResolve();
|
||||
};
|
||||
});
|
||||
}
|
||||
locked = true;
|
||||
holder = label;
|
||||
log.debug({ label }, "TDLib mutex acquired");
|
||||
try {
|
||||
return await fn();
|
||||
}
|
||||
finally {
|
||||
locked = false;
|
||||
holder = "";
|
||||
const next = queue.shift();
|
||||
if (next) {
|
||||
log.debug({ next: next.label }, "TDLib mutex releasing to next waiter");
|
||||
next.resolve();
|
||||
}
|
||||
else {
|
||||
log.debug({ label }, "TDLib mutex released");
|
||||
}
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=mutex.js.map
|
||||
1
worker/dist/util/mutex.js.map
vendored
1
worker/dist/util/mutex.js.map
vendored
@@ -1 +0,0 @@
|
||||
{"version":3,"file":"mutex.js","sourceRoot":"","sources":["../../src/util/mutex.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAE1C,MAAM,GAAG,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;AAEjC,IAAI,MAAM,GAAG,KAAK,CAAC;AACnB,IAAI,MAAM,GAAG,EAAE,CAAC;AAChB,MAAM,KAAK,GAAgF,EAAE,CAAC;AAE9F;;;;GAIG;AACH,MAAM,qBAAqB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAE7C;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,KAAa,EACb,EAAoB;IAEpB,IAAI,MAAM,EAAE,CAAC;QACX,GAAG,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,yBAAyB,CAAC,CAAC;QAChE,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC1C,MAAM,KAAK,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;YACzC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAElB,iDAAiD;YACjD,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;gBAC5B,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;gBACjC,IAAI,GAAG,KAAK,CAAC,CAAC,EAAE,CAAC;oBACf,KAAK,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;oBACrB,MAAM,CAAC,IAAI,KAAK,CACd,kCAAkC,qBAAqB,GAAG,MAAM,MAAM;wBACtE,aAAa,KAAK,aAAa,MAAM,GAAG,CACzC,CAAC,CAAC;gBACL,CAAC;YACH,CAAC,EAAE,qBAAqB,CAAC,CAAC;YAE1B,kCAAkC;YAClC,MAAM,WAAW,GAAG,KAAK,CAAC,OAAO,CAAC;YAClC,KAAK,CAAC,OAAO,GAAG,GAAG,EAAE;gBACnB,YAAY,CAAC,KAAK,CAAC,CAAC;gBACpB,WAAW,EAAE,CAAC;YAChB,CAAC,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IAED,MAAM,GAAG,IAAI,CAAC;IACd,MAAM,GAAG,KAAK,CAAC;IACf,GAAG,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,EAAE,sBAAsB,CAAC,CAAC;IAE7C,IAAI,CAAC;QACH,OAAO,MAAM,EAAE,EAAE,CAAC;IACpB,CAAC;YAAS,CAAC;QACT,MAAM,GAAG,KAAK,CAAC;QACf,MAAM,GAAG,EAAE,CAAC;QACZ,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;QAC3B,IAAI,IAAI,EAAE,CAAC;YACT,GAAG,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,KAAK,EAAE,EAAE,sCAAsC,CAAC,CAAC;YACxE,IAAI,CAAC,OAAO,EAAE,CAAC;QACjB,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,EAAE,sBAAsB,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;AACH,CAAC"}
|
||||
28
worker/dist/worker.d.ts
vendored
28
worker/dist/worker.d.ts
vendored
@@ -1,28 +0,0 @@
|
||||
import type { TelegramAccount } from "@prisma/client";
|
||||
/**
|
||||
* Authenticate a PENDING account by creating a TDLib client.
|
||||
* TDLib will send an SMS code to the phone number, and the client.login()
|
||||
* callbacks set the authState to AWAITING_CODE. Once the admin enters the
|
||||
* code via the UI, pollForAuthCode picks it up and completes the login.
|
||||
*
|
||||
* After successful auth:
|
||||
* 1. Fetches channels from Telegram and writes as a ChannelFetchRequest
|
||||
* (so the admin can select sources in the UI)
|
||||
* 2. Auto-joins the destination group if an invite link is configured
|
||||
*/
|
||||
export declare function authenticateAccount(account: TelegramAccount): Promise<void>;
|
||||
/**
|
||||
* Process a ChannelFetchRequest: fetch channels from Telegram,
|
||||
* enrich with DB state, and write the result JSON.
|
||||
* Called by the fetch listener (pg_notify) and by authenticateAccount.
|
||||
*/
|
||||
export declare function processFetchRequest(requestId: string): Promise<void>;
|
||||
/**
|
||||
* Run a full ingestion cycle for a single Telegram account.
|
||||
* Every step writes live activity to the DB so the admin UI can display it.
|
||||
*/
|
||||
export declare function runWorkerForAccount(account: TelegramAccount): Promise<void>;
|
||||
/**
|
||||
* Clean up any leftover temp files/directories from previous runs.
|
||||
*/
|
||||
export declare function cleanupTempDir(): Promise<void>;
|
||||
745
worker/dist/worker.js
vendored
745
worker/dist/worker.js
vendored
@@ -1,745 +0,0 @@
|
||||
import path from "path";
|
||||
import { unlink, readdir, mkdir, rm } from "fs/promises";
|
||||
import { config } from "./util/config.js";
|
||||
import { childLogger } from "./util/logger.js";
|
||||
import { tryAcquireLock, releaseLock } from "./db/locks.js";
|
||||
import { getSourceChannelMappings, getGlobalDestinationChannel, packageExistsByHash, packageExistsBySourceMessage, createPackageWithFiles, createIngestionRun, completeIngestionRun, failIngestionRun, updateLastProcessedMessage, updateRunActivity, setChannelForum, getTopicProgress, upsertTopicProgress, upsertChannel, ensureAccountChannelLink, getGlobalSetting, getChannelFetchRequest, updateFetchRequestStatus, getAccountLinkedChannelIds, getExistingChannelsByTelegramId, deleteOrphanedPackageByHash, } from "./db/queries.js";
|
||||
import { createTdlibClient, closeTdlibClient } from "./tdlib/client.js";
|
||||
import { getAccountChats, joinChatByInviteLink } from "./tdlib/chats.js";
|
||||
import { getChannelMessages, downloadFile, downloadPhotoThumbnail } from "./tdlib/download.js";
|
||||
import { isChatForum, getForumTopicList, getTopicMessages } from "./tdlib/topics.js";
|
||||
import { matchPreviewToArchive } from "./preview/match.js";
|
||||
import { groupArchiveSets } from "./archive/multipart.js";
|
||||
import { extractCreatorFromFileName } from "./archive/creator.js";
|
||||
import { hashParts } from "./archive/hash.js";
|
||||
import { readZipCentralDirectory } from "./archive/zip-reader.js";
|
||||
import { readRarContents } from "./archive/rar-reader.js";
|
||||
import { byteLevelSplit, concatenateFiles } from "./archive/split.js";
|
||||
import { uploadToChannel } from "./upload/channel.js";
|
||||
const log = childLogger("worker");
|
||||
/**
|
||||
* Authenticate a PENDING account by creating a TDLib client.
|
||||
* TDLib will send an SMS code to the phone number, and the client.login()
|
||||
* callbacks set the authState to AWAITING_CODE. Once the admin enters the
|
||||
* code via the UI, pollForAuthCode picks it up and completes the login.
|
||||
*
|
||||
* After successful auth:
|
||||
* 1. Fetches channels from Telegram and writes as a ChannelFetchRequest
|
||||
* (so the admin can select sources in the UI)
|
||||
* 2. Auto-joins the destination group if an invite link is configured
|
||||
*/
|
||||
export async function authenticateAccount(account) {
|
||||
const aLog = childLogger("auth", { accountId: account.id, phone: account.phone });
|
||||
aLog.info("Starting authentication flow");
|
||||
let client;
|
||||
try {
|
||||
client = await createTdlibClient({
|
||||
id: account.id,
|
||||
phone: account.phone,
|
||||
});
|
||||
aLog.info("Authentication successful");
|
||||
// Auto-fetch channels and create a fetch request result
|
||||
aLog.info("Fetching channels from Telegram...");
|
||||
await createAutoFetchRequest(client, account.id, aLog);
|
||||
// Auto-join the destination group if an invite link exists
|
||||
const inviteLink = await getGlobalSetting("destination_invite_link");
|
||||
if (inviteLink) {
|
||||
aLog.info("Attempting to join destination group via invite link...");
|
||||
try {
|
||||
await joinChatByInviteLink(client, inviteLink);
|
||||
// Link this account as WRITER to the destination channel
|
||||
const destChannel = await getGlobalDestinationChannel();
|
||||
if (destChannel) {
|
||||
await ensureAccountChannelLink(account.id, destChannel.id, "WRITER");
|
||||
aLog.info({ destChannel: destChannel.title }, "Joined destination group and linked as WRITER");
|
||||
}
|
||||
}
|
||||
catch (err) {
|
||||
// May already be a member — that's fine
|
||||
aLog.warn({ err }, "Could not join destination group (may already be a member)");
|
||||
// Still try to link as WRITER
|
||||
const destChannel = await getGlobalDestinationChannel();
|
||||
if (destChannel) {
|
||||
await ensureAccountChannelLink(account.id, destChannel.id, "WRITER");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (err) {
|
||||
aLog.error({ err }, "Authentication failed");
|
||||
}
|
||||
finally {
|
||||
if (client) {
|
||||
await closeTdlibClient(client);
|
||||
}
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Process a ChannelFetchRequest: fetch channels from Telegram,
|
||||
* enrich with DB state, and write the result JSON.
|
||||
* Called by the fetch listener (pg_notify) and by authenticateAccount.
|
||||
*/
|
||||
export async function processFetchRequest(requestId) {
|
||||
const aLog = childLogger("fetch-request", { requestId });
|
||||
const request = await getChannelFetchRequest(requestId);
|
||||
if (!request || request.status !== "PENDING") {
|
||||
aLog.warn("Fetch request not found or not pending, skipping");
|
||||
return;
|
||||
}
|
||||
await updateFetchRequestStatus(requestId, "IN_PROGRESS");
|
||||
aLog.info({ accountId: request.accountId }, "Processing fetch request");
|
||||
const client = await createTdlibClient({
|
||||
id: request.account.id,
|
||||
phone: request.account.phone,
|
||||
});
|
||||
try {
|
||||
const chats = await getAccountChats(client);
|
||||
// Enrich with DB state
|
||||
const linkedTelegramIds = await getAccountLinkedChannelIds(request.accountId);
|
||||
const existingChannels = await getExistingChannelsByTelegramId();
|
||||
const enrichedChats = chats.map((chat) => {
|
||||
const telegramIdStr = chat.chatId.toString();
|
||||
return {
|
||||
chatId: telegramIdStr,
|
||||
title: chat.title,
|
||||
type: chat.type,
|
||||
isForum: chat.isForum,
|
||||
memberCount: chat.memberCount ?? null,
|
||||
alreadyLinked: linkedTelegramIds.has(telegramIdStr),
|
||||
existingChannelId: existingChannels.get(telegramIdStr) ?? null,
|
||||
};
|
||||
});
|
||||
// Also upsert channel metadata while we have the data
|
||||
for (const chat of chats) {
|
||||
try {
|
||||
await upsertChannel({
|
||||
telegramId: chat.chatId,
|
||||
title: chat.title,
|
||||
type: "SOURCE",
|
||||
isForum: chat.isForum,
|
||||
});
|
||||
}
|
||||
catch {
|
||||
// Non-critical — metadata sync can fail silently
|
||||
}
|
||||
}
|
||||
await updateFetchRequestStatus(requestId, "COMPLETED", {
|
||||
resultJson: JSON.stringify(enrichedChats),
|
||||
});
|
||||
aLog.info({ total: chats.length, linked: [...linkedTelegramIds].length }, "Fetch request completed");
|
||||
}
|
||||
catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
aLog.error({ err }, "Fetch request failed");
|
||||
await updateFetchRequestStatus(requestId, "FAILED", { error: message });
|
||||
}
|
||||
finally {
|
||||
await closeTdlibClient(client);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Internal helper called after authentication to auto-create a fetch request
|
||||
* with the channel list (so the UI can show the picker immediately).
|
||||
*/
|
||||
async function createAutoFetchRequest(client, accountId, aLog) {
|
||||
const chats = await getAccountChats(client);
|
||||
const linkedTelegramIds = await getAccountLinkedChannelIds(accountId);
|
||||
const existingChannels = await getExistingChannelsByTelegramId();
|
||||
const enrichedChats = chats.map((chat) => {
|
||||
const telegramIdStr = chat.chatId.toString();
|
||||
return {
|
||||
chatId: telegramIdStr,
|
||||
title: chat.title,
|
||||
type: chat.type,
|
||||
isForum: chat.isForum,
|
||||
memberCount: chat.memberCount ?? null,
|
||||
alreadyLinked: linkedTelegramIds.has(telegramIdStr),
|
||||
existingChannelId: existingChannels.get(telegramIdStr) ?? null,
|
||||
};
|
||||
});
|
||||
// Upsert channel metadata
|
||||
for (const chat of chats) {
|
||||
try {
|
||||
await upsertChannel({
|
||||
telegramId: chat.chatId,
|
||||
title: chat.title,
|
||||
type: "SOURCE",
|
||||
isForum: chat.isForum,
|
||||
});
|
||||
}
|
||||
catch {
|
||||
// Non-critical
|
||||
}
|
||||
}
|
||||
// Create the fetch request record with the result already filled in
|
||||
const { db } = await import("./db/client.js");
|
||||
await db.channelFetchRequest.create({
|
||||
data: {
|
||||
accountId,
|
||||
status: "COMPLETED",
|
||||
resultJson: JSON.stringify(enrichedChats),
|
||||
},
|
||||
});
|
||||
aLog.info({ total: chats.length }, "Auto-fetch request created with channel list");
|
||||
}
|
||||
/**
|
||||
* Throttle DB writes for download progress to avoid hammering the DB.
|
||||
* Only writes if at least 2 seconds have passed since the last write.
|
||||
*/
|
||||
function createThrottledActivityUpdater(runId, minIntervalMs = 2000) {
|
||||
let lastWriteTime = 0;
|
||||
let pendingUpdate = null;
|
||||
let flushTimer = null;
|
||||
const flush = async () => {
|
||||
if (pendingUpdate) {
|
||||
const update = pendingUpdate;
|
||||
pendingUpdate = null;
|
||||
lastWriteTime = Date.now();
|
||||
await updateRunActivity(runId, update).catch(() => { });
|
||||
}
|
||||
};
|
||||
return {
|
||||
update: (activity) => {
|
||||
pendingUpdate = activity;
|
||||
const elapsed = Date.now() - lastWriteTime;
|
||||
if (elapsed >= minIntervalMs) {
|
||||
if (flushTimer)
|
||||
clearTimeout(flushTimer);
|
||||
flush();
|
||||
}
|
||||
else if (!flushTimer) {
|
||||
flushTimer = setTimeout(() => {
|
||||
flushTimer = null;
|
||||
flush();
|
||||
}, minIntervalMs - elapsed);
|
||||
}
|
||||
},
|
||||
flush,
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Run a full ingestion cycle for a single Telegram account.
|
||||
* Every step writes live activity to the DB so the admin UI can display it.
|
||||
*/
|
||||
export async function runWorkerForAccount(account) {
|
||||
const accountLog = childLogger("worker", { accountId: account.id, phone: account.phone });
|
||||
// 1. Acquire advisory lock
|
||||
const acquired = await tryAcquireLock(account.id);
|
||||
if (!acquired) {
|
||||
accountLog.info("Account already locked, skipping");
|
||||
return;
|
||||
}
|
||||
let runId;
|
||||
try {
|
||||
// 2. Create ingestion run
|
||||
const run = await createIngestionRun(account.id);
|
||||
runId = run.id;
|
||||
const activeRunId = runId;
|
||||
accountLog.info({ runId }, "Ingestion run started");
|
||||
const throttled = createThrottledActivityUpdater(activeRunId);
|
||||
// 3. Initialize TDLib client
|
||||
await updateRunActivity(activeRunId, {
|
||||
currentActivity: "Connecting to Telegram",
|
||||
currentStep: "connecting",
|
||||
});
|
||||
const client = await createTdlibClient({
|
||||
id: account.id,
|
||||
phone: account.phone,
|
||||
});
|
||||
const counters = {
|
||||
messagesScanned: 0,
|
||||
zipsFound: 0,
|
||||
zipsDuplicate: 0,
|
||||
zipsIngested: 0,
|
||||
};
|
||||
try {
|
||||
// 4. Get assigned source channels and global destination
|
||||
const channelMappings = await getSourceChannelMappings(account.id);
|
||||
const destChannel = await getGlobalDestinationChannel();
|
||||
if (!destChannel) {
|
||||
throw new Error("No global destination channel configured — set one in the admin UI");
|
||||
}
|
||||
const totalChannels = channelMappings.length;
|
||||
for (let chIdx = 0; chIdx < channelMappings.length; chIdx++) {
|
||||
const mapping = channelMappings[chIdx];
|
||||
const channel = mapping.channel;
|
||||
const channelLabel = totalChannels > 1
|
||||
? `[${chIdx + 1}/${totalChannels}] ${channel.title}`
|
||||
: channel.title;
|
||||
try {
|
||||
// ── Check if channel is a forum ──
|
||||
const forum = await isChatForum(client, channel.telegramId);
|
||||
if (forum !== channel.isForum) {
|
||||
await setChannelForum(channel.id, forum);
|
||||
accountLog.info({ channelId: channel.id, title: channel.title, isForum: forum }, "Updated channel forum status");
|
||||
}
|
||||
const pipelineCtx = {
|
||||
client,
|
||||
runId: activeRunId,
|
||||
channelTitle: channel.title,
|
||||
channel,
|
||||
destChannelTelegramId: destChannel.telegramId,
|
||||
destChannelId: destChannel.id,
|
||||
throttled,
|
||||
counters,
|
||||
topicCreator: null,
|
||||
sourceTopicId: null,
|
||||
accountLog,
|
||||
};
|
||||
if (forum) {
|
||||
// ── Forum channel: scan per-topic ──
|
||||
await updateRunActivity(activeRunId, {
|
||||
currentActivity: `Enumerating topics in "${channelLabel}"`,
|
||||
currentStep: "scanning",
|
||||
currentChannel: channelLabel,
|
||||
currentFile: null,
|
||||
currentFileNum: null,
|
||||
totalFiles: null,
|
||||
downloadedBytes: null,
|
||||
totalBytes: null,
|
||||
downloadPercent: null,
|
||||
messagesScanned: counters.messagesScanned,
|
||||
});
|
||||
const topics = await getForumTopicList(client, channel.telegramId);
|
||||
const topicProgressList = await getTopicProgress(mapping.id);
|
||||
accountLog.info({ channelId: channel.id, title: channel.title, topicCount: topics.length }, "Scanning forum channel by topic");
|
||||
for (let tIdx = 0; tIdx < topics.length; tIdx++) {
|
||||
const topic = topics[tIdx];
|
||||
try {
|
||||
const progress = topicProgressList.find((tp) => tp.topicId === topic.topicId);
|
||||
const topicLabel = `${channel.title} › ${topic.name}`;
|
||||
const topicProgress = topics.length > 1
|
||||
? ` (topic ${tIdx + 1}/${topics.length})`
|
||||
: "";
|
||||
await updateRunActivity(activeRunId, {
|
||||
currentActivity: `Scanning "${topicLabel}"${topicProgress}`,
|
||||
currentStep: "scanning",
|
||||
currentChannel: channelLabel,
|
||||
currentFile: null,
|
||||
currentFileNum: null,
|
||||
totalFiles: null,
|
||||
downloadedBytes: null,
|
||||
totalBytes: null,
|
||||
downloadPercent: null,
|
||||
messagesScanned: counters.messagesScanned,
|
||||
});
|
||||
const scanResult = await getTopicMessages(client, channel.telegramId, topic.topicId, progress?.lastProcessedMessageId, 100, (scanned) => {
|
||||
throttled.update({
|
||||
currentActivity: `Scanning "${topicLabel}"${topicProgress} — ${scanned} messages scanned`,
|
||||
currentStep: "scanning",
|
||||
currentChannel: channelLabel,
|
||||
messagesScanned: counters.messagesScanned + scanned,
|
||||
});
|
||||
});
|
||||
// Add scanned messages to global counter
|
||||
counters.messagesScanned += scanResult.totalScanned;
|
||||
if (scanResult.archives.length === 0) {
|
||||
accountLog.debug({ channelId: channel.id, topic: topic.name }, "No new archives in topic");
|
||||
continue;
|
||||
}
|
||||
accountLog.info({ topic: topic.name, archives: scanResult.archives.length, photos: scanResult.photos.length }, "Found messages in topic");
|
||||
// Process archives with topic creator
|
||||
pipelineCtx.topicCreator = topic.name;
|
||||
pipelineCtx.sourceTopicId = topic.topicId;
|
||||
pipelineCtx.channelTitle = `${channel.title} › ${topic.name}`;
|
||||
const maxProcessedId = await processArchiveSets(pipelineCtx, scanResult, run.id, progress?.lastProcessedMessageId);
|
||||
// Only advance progress to the highest successfully processed message
|
||||
if (maxProcessedId) {
|
||||
await upsertTopicProgress(mapping.id, topic.topicId, topic.name, maxProcessedId);
|
||||
}
|
||||
}
|
||||
catch (topicErr) {
|
||||
accountLog.warn({ err: topicErr, channelId: channel.id, topic: topic.name, topicId: topic.topicId.toString() }, "Failed to process topic, skipping");
|
||||
}
|
||||
}
|
||||
}
|
||||
else {
|
||||
// ── Non-forum channel: flat scan (existing behavior) ──
|
||||
await updateRunActivity(activeRunId, {
|
||||
currentActivity: `Scanning "${channelLabel}" for new archives`,
|
||||
currentStep: "scanning",
|
||||
currentChannel: channelLabel,
|
||||
currentFile: null,
|
||||
currentFileNum: null,
|
||||
totalFiles: null,
|
||||
downloadedBytes: null,
|
||||
totalBytes: null,
|
||||
downloadPercent: null,
|
||||
messagesScanned: counters.messagesScanned,
|
||||
});
|
||||
accountLog.info({ channelId: channel.id, title: channel.title }, "Processing source channel");
|
||||
const scanResult = await getChannelMessages(client, channel.telegramId, mapping.lastProcessedMessageId, 100, (scanned) => {
|
||||
throttled.update({
|
||||
currentActivity: `Scanning "${channelLabel}" — ${scanned} messages scanned`,
|
||||
currentStep: "scanning",
|
||||
currentChannel: channelLabel,
|
||||
messagesScanned: counters.messagesScanned + scanned,
|
||||
});
|
||||
});
|
||||
// Add scanned messages to global counter
|
||||
counters.messagesScanned += scanResult.totalScanned;
|
||||
if (scanResult.archives.length === 0) {
|
||||
accountLog.debug({ channelId: channel.id }, "No new archives");
|
||||
continue;
|
||||
}
|
||||
accountLog.info({ archives: scanResult.archives.length, photos: scanResult.photos.length }, "Found messages in channel");
|
||||
// For non-forum, creator comes from filename (set to null, resolved per-archive)
|
||||
pipelineCtx.topicCreator = null;
|
||||
pipelineCtx.sourceTopicId = null;
|
||||
pipelineCtx.channelTitle = channel.title;
|
||||
const maxProcessedId = await processArchiveSets(pipelineCtx, scanResult, run.id, mapping.lastProcessedMessageId);
|
||||
// Only advance progress to the highest successfully processed message
|
||||
if (maxProcessedId) {
|
||||
await updateLastProcessedMessage(mapping.id, maxProcessedId);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (channelErr) {
|
||||
accountLog.warn({ err: channelErr, channelId: channel.id, title: channel.title }, "Failed to process channel, skipping to next");
|
||||
}
|
||||
}
|
||||
// ── Done ──
|
||||
await completeIngestionRun(activeRunId, counters);
|
||||
accountLog.info({ counters }, "Ingestion run completed");
|
||||
}
|
||||
finally {
|
||||
await closeTdlibClient(client);
|
||||
}
|
||||
}
|
||||
catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
accountLog.error({ err }, "Ingestion run failed");
|
||||
if (runId) {
|
||||
await failIngestionRun(runId, message).catch((e) => accountLog.error({ e }, "Failed to mark run as failed"));
|
||||
}
|
||||
}
|
||||
finally {
|
||||
await releaseLock(account.id);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Process a scan result through the archive pipeline:
|
||||
* group → download → hash → dedup → metadata → split → upload → preview → index.
|
||||
*
|
||||
* Returns the highest message ID that was successfully processed (ingested or
|
||||
* confirmed duplicate). The caller should only advance the progress boundary
|
||||
* to this value — never to the max of all scanned messages.
|
||||
*/
|
||||
async function processArchiveSets(ctx, scanResult, ingestionRunId, lastProcessedMessageId) {
|
||||
const { client, runId, channelTitle, channel, throttled, counters, accountLog } = ctx;
|
||||
// Group into archive sets
|
||||
let archiveSets = groupArchiveSets(scanResult.archives);
|
||||
// Filter out sets where ALL parts are at or below the boundary (already processed)
|
||||
if (lastProcessedMessageId) {
|
||||
const totalBefore = archiveSets.length;
|
||||
archiveSets = archiveSets.filter((set) => set.parts.some((p) => p.id > lastProcessedMessageId));
|
||||
const filtered = totalBefore - archiveSets.length;
|
||||
if (filtered > 0) {
|
||||
accountLog.info({ filtered, remaining: archiveSets.length }, "Filtered out already-processed archive sets");
|
||||
}
|
||||
}
|
||||
counters.zipsFound += archiveSets.length;
|
||||
// Match preview photos to archive sets
|
||||
const previewMatches = matchPreviewToArchive(scanResult.photos, archiveSets.map((s) => ({
|
||||
baseName: s.baseName,
|
||||
firstMessageId: s.parts[0].id,
|
||||
firstMessageDate: s.parts[0].date,
|
||||
})));
|
||||
if (previewMatches.size > 0) {
|
||||
accountLog.info({ matched: previewMatches.size, total: archiveSets.length }, "Matched preview photos to archives");
|
||||
}
|
||||
await updateRunActivity(runId, {
|
||||
currentActivity: `Found ${archiveSets.length} archive(s) in "${channelTitle}"`,
|
||||
currentStep: "scanning",
|
||||
currentChannel: channelTitle,
|
||||
totalFiles: archiveSets.length,
|
||||
zipsFound: counters.zipsFound,
|
||||
messagesScanned: counters.messagesScanned,
|
||||
});
|
||||
// Track the highest message ID that was successfully processed
|
||||
let maxProcessedId = null;
|
||||
for (let setIdx = 0; setIdx < archiveSets.length; setIdx++) {
|
||||
try {
|
||||
await processOneArchiveSet(ctx, archiveSets[setIdx], setIdx, archiveSets.length, previewMatches, ingestionRunId);
|
||||
// Set completed (ingested or confirmed duplicate) — advance watermark
|
||||
const setMaxId = archiveSets[setIdx].parts.reduce((max, p) => (p.id > max ? p.id : max), 0n);
|
||||
if (setMaxId > (maxProcessedId ?? 0n)) {
|
||||
maxProcessedId = setMaxId;
|
||||
}
|
||||
}
|
||||
catch (setErr) {
|
||||
// If a set fails, do NOT advance the watermark past it
|
||||
accountLog.warn({ err: setErr, baseName: archiveSets[setIdx].baseName }, "Archive set failed, watermark will not advance past this set");
|
||||
}
|
||||
}
|
||||
return maxProcessedId;
|
||||
}
|
||||
/**
|
||||
* Process a single archive set through the full pipeline.
|
||||
*/
|
||||
async function processOneArchiveSet(ctx, archiveSet, setIdx, totalSets, previewMatches, ingestionRunId) {
|
||||
const { client, runId, channelTitle, channel, destChannelTelegramId, destChannelId, throttled, counters, topicCreator, sourceTopicId, accountLog, } = ctx;
|
||||
const archiveName = archiveSet.parts[0].fileName;
|
||||
// ── Early skip: check if this archive set was already ingested ──
|
||||
// This avoids re-downloading large archives that were processed in a prior run.
|
||||
const alreadyIngested = await packageExistsBySourceMessage(channel.id, archiveSet.parts[0].id);
|
||||
if (alreadyIngested) {
|
||||
counters.zipsDuplicate++;
|
||||
accountLog.debug({ fileName: archiveName, sourceMessageId: Number(archiveSet.parts[0].id) }, "Archive already ingested (by source message), skipping");
|
||||
await updateRunActivity(runId, {
|
||||
currentActivity: `Skipped ${archiveName} (already ingested)`,
|
||||
currentStep: "deduplicating",
|
||||
currentChannel: channelTitle,
|
||||
currentFile: archiveName,
|
||||
currentFileNum: setIdx + 1,
|
||||
totalFiles: totalSets,
|
||||
zipsDuplicate: counters.zipsDuplicate,
|
||||
});
|
||||
return;
|
||||
}
|
||||
const tempPaths = [];
|
||||
let splitPaths = [];
|
||||
// Per-set subdirectory so uploaded files keep their original filenames
|
||||
const setDir = path.join(config.tempDir, `${ingestionRunId}_${archiveSet.parts[0].id}`);
|
||||
await mkdir(setDir, { recursive: true });
|
||||
try {
|
||||
// ── Downloading ──
|
||||
for (let partIdx = 0; partIdx < archiveSet.parts.length; partIdx++) {
|
||||
const part = archiveSet.parts[partIdx];
|
||||
const tempPath = path.join(setDir, part.fileName);
|
||||
const partLabel = archiveSet.parts.length > 1
|
||||
? ` (part ${partIdx + 1}/${archiveSet.parts.length})`
|
||||
: "";
|
||||
await updateRunActivity(runId, {
|
||||
currentActivity: `Downloading ${part.fileName}${partLabel}`,
|
||||
currentStep: "downloading",
|
||||
currentChannel: channelTitle,
|
||||
currentFile: part.fileName,
|
||||
currentFileNum: setIdx + 1,
|
||||
totalFiles: totalSets,
|
||||
downloadedBytes: 0n,
|
||||
totalBytes: part.fileSize,
|
||||
downloadPercent: 0,
|
||||
messagesScanned: counters.messagesScanned,
|
||||
});
|
||||
accountLog.info({
|
||||
fileName: part.fileName,
|
||||
fileSize: Number(part.fileSize),
|
||||
part: partIdx + 1,
|
||||
totalParts: archiveSet.parts.length,
|
||||
}, "Downloading archive part");
|
||||
await downloadFile(client, part.fileId, tempPath, part.fileSize, part.fileName, (progress) => {
|
||||
throttled.update({
|
||||
currentActivity: `Downloading ${part.fileName}${partLabel} — ${progress.percent}%`,
|
||||
currentStep: "downloading",
|
||||
currentChannel: channelTitle,
|
||||
currentFile: part.fileName,
|
||||
currentFileNum: setIdx + 1,
|
||||
totalFiles: totalSets,
|
||||
downloadedBytes: BigInt(progress.downloadedBytes),
|
||||
totalBytes: BigInt(progress.totalBytes),
|
||||
downloadPercent: progress.percent,
|
||||
});
|
||||
});
|
||||
await throttled.flush();
|
||||
tempPaths.push(tempPath);
|
||||
}
|
||||
// ── Hashing ──
|
||||
await updateRunActivity(runId, {
|
||||
currentActivity: `Computing hash for ${archiveName}`,
|
||||
currentStep: "hashing",
|
||||
currentChannel: channelTitle,
|
||||
currentFile: archiveName,
|
||||
currentFileNum: setIdx + 1,
|
||||
totalFiles: totalSets,
|
||||
downloadedBytes: null,
|
||||
totalBytes: null,
|
||||
downloadPercent: null,
|
||||
});
|
||||
const contentHash = await hashParts(tempPaths);
|
||||
// ── Deduplicating ──
|
||||
await updateRunActivity(runId, {
|
||||
currentActivity: `Checking if ${archiveName} is a duplicate`,
|
||||
currentStep: "deduplicating",
|
||||
currentChannel: channelTitle,
|
||||
currentFile: archiveName,
|
||||
currentFileNum: setIdx + 1,
|
||||
totalFiles: totalSets,
|
||||
});
|
||||
const exists = await packageExistsByHash(contentHash);
|
||||
if (exists) {
|
||||
counters.zipsDuplicate++;
|
||||
accountLog.debug({ contentHash }, "Duplicate archive, skipping");
|
||||
await updateRunActivity(runId, {
|
||||
currentActivity: `Skipped ${archiveName} (duplicate)`,
|
||||
currentStep: "deduplicating",
|
||||
currentChannel: channelTitle,
|
||||
currentFile: archiveName,
|
||||
currentFileNum: setIdx + 1,
|
||||
totalFiles: totalSets,
|
||||
zipsDuplicate: counters.zipsDuplicate,
|
||||
});
|
||||
return;
|
||||
}
|
||||
// ── Reading metadata ──
|
||||
await updateRunActivity(runId, {
|
||||
currentActivity: `Reading file list from ${archiveName}`,
|
||||
currentStep: "reading_metadata",
|
||||
currentChannel: channelTitle,
|
||||
currentFile: archiveName,
|
||||
currentFileNum: setIdx + 1,
|
||||
totalFiles: totalSets,
|
||||
});
|
||||
let entries = [];
|
||||
try {
|
||||
if (archiveSet.type === "ZIP") {
|
||||
entries = await readZipCentralDirectory(tempPaths);
|
||||
}
|
||||
else {
|
||||
entries = await readRarContents(tempPaths[0]);
|
||||
}
|
||||
}
|
||||
catch (err) {
|
||||
accountLog.warn({ err, baseName: archiveSet.baseName }, "Failed to read archive metadata, ingesting without file list");
|
||||
}
|
||||
// ── Splitting / Repacking (if needed) ──
|
||||
let uploadPaths = [...tempPaths];
|
||||
const totalSize = archiveSet.parts.reduce((sum, p) => sum + p.fileSize, 0n);
|
||||
const MAX_UPLOAD_SIZE = 2n * 1024n * 1024n * 1024n;
|
||||
const hasOversizedPart = archiveSet.parts.some((p) => p.fileSize > MAX_UPLOAD_SIZE);
|
||||
if (hasOversizedPart) {
|
||||
// Full repack: concatenate all parts → single file → re-split into uniform 2GB chunks
|
||||
await updateRunActivity(runId, {
|
||||
currentActivity: `Repacking ${archiveName} (parts >2GB, concatenating + re-splitting)`,
|
||||
currentStep: "splitting",
|
||||
currentChannel: channelTitle,
|
||||
currentFile: archiveName,
|
||||
currentFileNum: setIdx + 1,
|
||||
totalFiles: totalSets,
|
||||
});
|
||||
const concatPath = path.join(setDir, `${archiveSet.baseName}.concat`);
|
||||
await concatenateFiles(tempPaths, concatPath);
|
||||
splitPaths = await byteLevelSplit(concatPath);
|
||||
uploadPaths = splitPaths;
|
||||
// Clean up the concat intermediate file
|
||||
await unlink(concatPath).catch(() => { });
|
||||
}
|
||||
else if (!archiveSet.isMultipart && totalSize > MAX_UPLOAD_SIZE) {
|
||||
// Single file >2GB: split directly
|
||||
await updateRunActivity(runId, {
|
||||
currentActivity: `Splitting ${archiveName} for upload (>2GB)`,
|
||||
currentStep: "splitting",
|
||||
currentChannel: channelTitle,
|
||||
currentFile: archiveName,
|
||||
currentFileNum: setIdx + 1,
|
||||
totalFiles: totalSets,
|
||||
});
|
||||
splitPaths = await byteLevelSplit(tempPaths[0]);
|
||||
uploadPaths = splitPaths;
|
||||
}
|
||||
// ── Uploading ──
|
||||
const uploadLabel = uploadPaths.length > 1
|
||||
? ` (${uploadPaths.length} parts)`
|
||||
: "";
|
||||
await updateRunActivity(runId, {
|
||||
currentActivity: `Uploading ${archiveName} to archive channel${uploadLabel}`,
|
||||
currentStep: "uploading",
|
||||
currentChannel: channelTitle,
|
||||
currentFile: archiveName,
|
||||
currentFileNum: setIdx + 1,
|
||||
totalFiles: totalSets,
|
||||
});
|
||||
const destResult = await uploadToChannel(client, destChannelTelegramId, uploadPaths);
|
||||
// ── Preview thumbnail ──
|
||||
let previewData = null;
|
||||
let previewMsgId = null;
|
||||
const matchedPhoto = previewMatches.get(archiveSet.baseName);
|
||||
if (matchedPhoto) {
|
||||
await updateRunActivity(runId, {
|
||||
currentActivity: `Downloading preview image for ${archiveName}`,
|
||||
currentStep: "preview",
|
||||
currentChannel: channelTitle,
|
||||
currentFile: archiveName,
|
||||
currentFileNum: setIdx + 1,
|
||||
totalFiles: totalSets,
|
||||
});
|
||||
previewData = await downloadPhotoThumbnail(client, matchedPhoto.fileId);
|
||||
previewMsgId = matchedPhoto.id;
|
||||
}
|
||||
// ── Resolve creator: topic name > filename extraction > null ──
|
||||
const creator = topicCreator ?? extractCreatorFromFileName(archiveName) ?? null;
|
||||
// ── Indexing ──
|
||||
await updateRunActivity(runId, {
|
||||
currentActivity: `Saving metadata for ${archiveName} (${entries.length} files)`,
|
||||
currentStep: "indexing",
|
||||
currentChannel: channelTitle,
|
||||
currentFile: archiveName,
|
||||
currentFileNum: setIdx + 1,
|
||||
totalFiles: totalSets,
|
||||
});
|
||||
// Clean up any orphaned record (same hash but no dest upload) before creating
|
||||
await deleteOrphanedPackageByHash(contentHash);
|
||||
await createPackageWithFiles({
|
||||
contentHash,
|
||||
fileName: archiveName,
|
||||
fileSize: totalSize,
|
||||
archiveType: archiveSet.type,
|
||||
sourceChannelId: channel.id,
|
||||
sourceMessageId: archiveSet.parts[0].id,
|
||||
sourceTopicId,
|
||||
destChannelId,
|
||||
destMessageId: destResult.messageId,
|
||||
isMultipart: archiveSet.parts.length > 1 || uploadPaths.length > 1,
|
||||
partCount: uploadPaths.length,
|
||||
ingestionRunId,
|
||||
creator,
|
||||
previewData,
|
||||
previewMsgId,
|
||||
files: entries,
|
||||
});
|
||||
counters.zipsIngested++;
|
||||
await updateRunActivity(runId, {
|
||||
currentActivity: `Ingested ${archiveName} (${entries.length} files indexed)`,
|
||||
currentStep: "complete",
|
||||
currentChannel: channelTitle,
|
||||
currentFile: archiveName,
|
||||
currentFileNum: setIdx + 1,
|
||||
totalFiles: totalSets,
|
||||
zipsIngested: counters.zipsIngested,
|
||||
});
|
||||
accountLog.info({ fileName: archiveName, contentHash, fileCount: entries.length, creator }, "Archive ingested");
|
||||
}
|
||||
finally {
|
||||
// ALWAYS delete temp files and the set directory
|
||||
await deleteFiles([...tempPaths, ...splitPaths]);
|
||||
await rm(setDir, { recursive: true, force: true }).catch(() => { });
|
||||
}
|
||||
}
|
||||
async function deleteFiles(paths) {
|
||||
for (const p of paths) {
|
||||
try {
|
||||
await unlink(p);
|
||||
}
|
||||
catch {
|
||||
// File may already be deleted or never created
|
||||
}
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Clean up any leftover temp files/directories from previous runs.
|
||||
*/
|
||||
export async function cleanupTempDir() {
|
||||
try {
|
||||
const entries = await readdir(config.tempDir);
|
||||
for (const entry of entries) {
|
||||
await rm(path.join(config.tempDir, entry), { recursive: true, force: true }).catch(() => { });
|
||||
}
|
||||
if (entries.length > 0) {
|
||||
log.info({ count: entries.length }, "Cleaned up stale temp files");
|
||||
}
|
||||
}
|
||||
catch {
|
||||
// Directory might not exist yet
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=worker.js.map
|
||||
1
worker/dist/worker.js.map
vendored
1
worker/dist/worker.js.map
vendored
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user