mirror of
https://github.com/xCyanGrizzly/DragonsStash.git
synced 2026-05-11 14:21:15 +00:00
feat: add Telegram integration with forum topic support and creator tracking
Adds full Telegram ZIP ingestion pipeline: TDLib worker service scans source channels for archive files, deduplicates by content hash, extracts metadata, uploads to archive channel, and indexes in Postgres. Forum supergroups are scanned per-topic with topic names used as creator. Filename-based creator extraction (e.g. "Mammoth Factory - 2026-01.zip") serves as fallback. Includes admin UI for managing accounts/channels, simplified account setup (API credentials via env vars), auth code/password submission dialog, package browser with creator column, and live ingestion activity tracking. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
48
worker/src/archive/split.ts
Normal file
48
worker/src/archive/split.ts
Normal file
@@ -0,0 +1,48 @@
|
||||
import { createReadStream, createWriteStream } from "fs";
|
||||
import { stat } from "fs/promises";
|
||||
import path from "path";
|
||||
import { pipeline } from "stream/promises";
|
||||
import { childLogger } from "../util/logger.js";
|
||||
|
||||
const log = childLogger("split");
|
||||
|
||||
/** 2GB in bytes — Telegram's file size limit */
|
||||
const MAX_PART_SIZE = 2n * 1024n * 1024n * 1024n;
|
||||
|
||||
/**
|
||||
* Split a file into ≤2GB parts using byte-level splitting.
|
||||
* Returns paths to the split parts. If the file is already ≤2GB, returns the original path.
|
||||
*/
|
||||
export async function byteLevelSplit(filePath: string): Promise<string[]> {
|
||||
const stats = await stat(filePath);
|
||||
const fileSize = BigInt(stats.size);
|
||||
|
||||
if (fileSize <= MAX_PART_SIZE) {
|
||||
return [filePath];
|
||||
}
|
||||
|
||||
const dir = path.dirname(filePath);
|
||||
const baseName = path.basename(filePath);
|
||||
const partSize = Number(MAX_PART_SIZE);
|
||||
const totalParts = Math.ceil(Number(fileSize) / partSize);
|
||||
const parts: string[] = [];
|
||||
|
||||
log.info({ filePath, fileSize: Number(fileSize), totalParts }, "Splitting file");
|
||||
|
||||
for (let i = 0; i < totalParts; i++) {
|
||||
const partNum = String(i + 1).padStart(3, "0");
|
||||
const partPath = path.join(dir, `${baseName}.${partNum}`);
|
||||
const start = i * partSize;
|
||||
const end = Math.min(start + partSize - 1, Number(fileSize) - 1);
|
||||
|
||||
await pipeline(
|
||||
createReadStream(filePath, { start, end }),
|
||||
createWriteStream(partPath)
|
||||
);
|
||||
|
||||
parts.push(partPath);
|
||||
}
|
||||
|
||||
log.info({ filePath, parts: parts.length }, "File split complete");
|
||||
return parts;
|
||||
}
|
||||
Reference in New Issue
Block a user