feat: add Telegram integration with forum topic support and creator tracking

Adds full Telegram ZIP ingestion pipeline: TDLib worker service scans source
channels for archive files, deduplicates by content hash, extracts metadata,
uploads to archive channel, and indexes in Postgres. Forum supergroups are
scanned per-topic with topic names used as creator. Filename-based creator
extraction (e.g. "Mammoth Factory - 2026-01.zip") serves as fallback.

Includes admin UI for managing accounts/channels, simplified account setup
(API credentials via env vars), auth code/password submission dialog,
package browser with creator column, and live ingestion activity tracking.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
xCyanGrizzly
2026-02-24 16:02:06 +01:00
parent beb9cfb312
commit b427193d17
70 changed files with 8627 additions and 2 deletions

View File

@@ -11,3 +11,14 @@ AUTH_GITHUB_SECRET=""
# App # App
NEXT_PUBLIC_APP_URL="http://localhost:3000" NEXT_PUBLIC_APP_URL="http://localhost:3000"
# Telegram integration (get from https://my.telegram.org/apps)
TELEGRAM_API_ID=""
TELEGRAM_API_HASH=""
# Worker (only needed when running worker container)
WORKER_INTERVAL_MINUTES=60
WORKER_TEMP_DIR="/tmp/zips"
TDLIB_STATE_DIR="/data/tdlib"
WORKER_MAX_ZIP_SIZE_MB=4096
LOG_LEVEL="info"

5
.gitignore vendored
View File

@@ -1,5 +1,6 @@
# dependencies # dependencies
/node_modules /node_modules
worker/node_modules
/.pnp /.pnp
.pnp.* .pnp.*
.yarn/* .yarn/*
@@ -48,3 +49,7 @@ src/generated
# ide # ide
.idea .idea
.vscode .vscode
# temp files
nul
tmpclaude-*

View File

@@ -15,5 +15,27 @@ services:
timeout: 5s timeout: 5s
retries: 5 retries: 5
worker:
build:
context: .
dockerfile: worker/Dockerfile
environment:
- DATABASE_URL=postgresql://dragons:stash@db:5432/dragonsstash
- WORKER_INTERVAL_MINUTES=5
- WORKER_TEMP_DIR=/tmp/zips
- TDLIB_STATE_DIR=/data/tdlib
- WORKER_MAX_ZIP_SIZE_MB=4096
- LOG_LEVEL=debug
- TELEGRAM_API_ID=${TELEGRAM_API_ID}
- TELEGRAM_API_HASH=${TELEGRAM_API_HASH}
volumes:
- tdlib_dev_state:/data/tdlib
- tmp_dev_zips:/tmp/zips
depends_on:
db:
condition: service_healthy
volumes: volumes:
postgres_dev_data: postgres_dev_data:
tdlib_dev_state:
tmp_dev_zips:

View File

@@ -10,11 +10,37 @@ services:
- AUTH_SECRET=change-me-to-a-random-secret-in-production - AUTH_SECRET=change-me-to-a-random-secret-in-production
- AUTH_TRUST_HOST=true - AUTH_TRUST_HOST=true
- NEXT_PUBLIC_APP_URL=http://localhost:3000 - NEXT_PUBLIC_APP_URL=http://localhost:3000
- TELEGRAM_API_KEY=${TELEGRAM_API_KEY:-}
depends_on: depends_on:
db: db:
condition: service_healthy condition: service_healthy
restart: unless-stopped restart: unless-stopped
worker:
build:
context: .
dockerfile: worker/Dockerfile
environment:
- DATABASE_URL=postgresql://dragons:stash@db:5432/dragonsstash
- WORKER_INTERVAL_MINUTES=60
- WORKER_TEMP_DIR=/tmp/zips
- TDLIB_STATE_DIR=/data/tdlib
- WORKER_MAX_ZIP_SIZE_MB=4096
- LOG_LEVEL=info
volumes:
- tdlib_state:/data/tdlib
- tmp_zips:/tmp/zips
depends_on:
db:
condition: service_healthy
restart: unless-stopped
deploy:
resources:
limits:
memory: 1G
reservations:
memory: 256M
db: db:
image: postgres:16-alpine image: postgres:16-alpine
ports: ports:
@@ -34,3 +60,5 @@ services:
volumes: volumes:
postgres_data: postgres_data:
tdlib_state:
tmp_zips:

View File

@@ -0,0 +1,183 @@
-- CreateEnum
CREATE TYPE "AuthState" AS ENUM ('PENDING', 'AWAITING_CODE', 'AWAITING_PASSWORD', 'AUTHENTICATED', 'EXPIRED');
-- CreateEnum
CREATE TYPE "ChannelType" AS ENUM ('SOURCE', 'DESTINATION');
-- CreateEnum
CREATE TYPE "ChannelRole" AS ENUM ('READER', 'WRITER');
-- CreateEnum
CREATE TYPE "ArchiveType" AS ENUM ('ZIP', 'RAR');
-- CreateEnum
CREATE TYPE "IngestionStatus" AS ENUM ('RUNNING', 'COMPLETED', 'FAILED', 'CANCELLED');
-- CreateTable
CREATE TABLE "telegram_accounts" (
"id" TEXT NOT NULL,
"phone" TEXT NOT NULL,
"displayName" TEXT,
"apiId" INTEGER NOT NULL,
"apiHash" TEXT NOT NULL,
"sessionPath" TEXT NOT NULL,
"isActive" BOOLEAN NOT NULL DEFAULT true,
"authState" "AuthState" NOT NULL DEFAULT 'PENDING',
"authCode" TEXT,
"lastSeenAt" TIMESTAMP(3),
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updatedAt" TIMESTAMP(3) NOT NULL,
CONSTRAINT "telegram_accounts_pkey" PRIMARY KEY ("id")
);
-- CreateTable
CREATE TABLE "telegram_channels" (
"id" TEXT NOT NULL,
"telegramId" BIGINT NOT NULL,
"title" TEXT NOT NULL,
"type" "ChannelType" NOT NULL,
"isActive" BOOLEAN NOT NULL DEFAULT true,
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updatedAt" TIMESTAMP(3) NOT NULL,
CONSTRAINT "telegram_channels_pkey" PRIMARY KEY ("id")
);
-- CreateTable
CREATE TABLE "account_channel_map" (
"id" TEXT NOT NULL,
"accountId" TEXT NOT NULL,
"channelId" TEXT NOT NULL,
"role" "ChannelRole" NOT NULL DEFAULT 'READER',
"lastProcessedMessageId" BIGINT,
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "account_channel_map_pkey" PRIMARY KEY ("id")
);
-- CreateTable
CREATE TABLE "packages" (
"id" TEXT NOT NULL,
"contentHash" TEXT NOT NULL,
"fileName" TEXT NOT NULL,
"fileSize" BIGINT NOT NULL,
"archiveType" "ArchiveType" NOT NULL,
"sourceChannelId" TEXT NOT NULL,
"sourceMessageId" BIGINT NOT NULL,
"destChannelId" TEXT,
"destMessageId" BIGINT,
"isMultipart" BOOLEAN NOT NULL DEFAULT false,
"partCount" INTEGER NOT NULL DEFAULT 1,
"fileCount" INTEGER NOT NULL DEFAULT 0,
"indexedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"ingestionRunId" TEXT,
CONSTRAINT "packages_pkey" PRIMARY KEY ("id")
);
-- CreateTable
CREATE TABLE "package_files" (
"id" TEXT NOT NULL,
"packageId" TEXT NOT NULL,
"path" TEXT NOT NULL,
"fileName" TEXT NOT NULL,
"extension" TEXT,
"compressedSize" BIGINT NOT NULL DEFAULT 0,
"uncompressedSize" BIGINT NOT NULL DEFAULT 0,
"crc32" TEXT,
CONSTRAINT "package_files_pkey" PRIMARY KEY ("id")
);
-- CreateTable
CREATE TABLE "ingestion_runs" (
"id" TEXT NOT NULL,
"accountId" TEXT NOT NULL,
"status" "IngestionStatus" NOT NULL DEFAULT 'RUNNING',
"startedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"finishedAt" TIMESTAMP(3),
"messagesScanned" INTEGER NOT NULL DEFAULT 0,
"zipsFound" INTEGER NOT NULL DEFAULT 0,
"zipsDuplicate" INTEGER NOT NULL DEFAULT 0,
"zipsIngested" INTEGER NOT NULL DEFAULT 0,
"errorMessage" TEXT,
CONSTRAINT "ingestion_runs_pkey" PRIMARY KEY ("id")
);
-- CreateIndex
CREATE UNIQUE INDEX "telegram_accounts_phone_key" ON "telegram_accounts"("phone");
-- CreateIndex
CREATE INDEX "telegram_accounts_isActive_idx" ON "telegram_accounts"("isActive");
-- CreateIndex
CREATE UNIQUE INDEX "telegram_channels_telegramId_key" ON "telegram_channels"("telegramId");
-- CreateIndex
CREATE INDEX "telegram_channels_type_isActive_idx" ON "telegram_channels"("type", "isActive");
-- CreateIndex
CREATE INDEX "account_channel_map_accountId_idx" ON "account_channel_map"("accountId");
-- CreateIndex
CREATE INDEX "account_channel_map_channelId_idx" ON "account_channel_map"("channelId");
-- CreateIndex
CREATE UNIQUE INDEX "account_channel_map_accountId_channelId_key" ON "account_channel_map"("accountId", "channelId");
-- CreateIndex
CREATE UNIQUE INDEX "packages_contentHash_key" ON "packages"("contentHash");
-- CreateIndex
CREATE INDEX "packages_sourceChannelId_idx" ON "packages"("sourceChannelId");
-- CreateIndex
CREATE INDEX "packages_destChannelId_idx" ON "packages"("destChannelId");
-- CreateIndex
CREATE INDEX "packages_fileName_idx" ON "packages"("fileName");
-- CreateIndex
CREATE INDEX "packages_indexedAt_idx" ON "packages"("indexedAt");
-- CreateIndex
CREATE INDEX "packages_archiveType_idx" ON "packages"("archiveType");
-- CreateIndex
CREATE INDEX "package_files_packageId_idx" ON "package_files"("packageId");
-- CreateIndex
CREATE INDEX "package_files_extension_idx" ON "package_files"("extension");
-- CreateIndex
CREATE INDEX "package_files_fileName_idx" ON "package_files"("fileName");
-- CreateIndex
CREATE INDEX "ingestion_runs_accountId_idx" ON "ingestion_runs"("accountId");
-- CreateIndex
CREATE INDEX "ingestion_runs_status_idx" ON "ingestion_runs"("status");
-- CreateIndex
CREATE INDEX "ingestion_runs_startedAt_idx" ON "ingestion_runs"("startedAt");
-- AddForeignKey
ALTER TABLE "account_channel_map" ADD CONSTRAINT "account_channel_map_accountId_fkey" FOREIGN KEY ("accountId") REFERENCES "telegram_accounts"("id") ON DELETE CASCADE ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "account_channel_map" ADD CONSTRAINT "account_channel_map_channelId_fkey" FOREIGN KEY ("channelId") REFERENCES "telegram_channels"("id") ON DELETE CASCADE ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "packages" ADD CONSTRAINT "packages_sourceChannelId_fkey" FOREIGN KEY ("sourceChannelId") REFERENCES "telegram_channels"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "packages" ADD CONSTRAINT "packages_ingestionRunId_fkey" FOREIGN KEY ("ingestionRunId") REFERENCES "ingestion_runs"("id") ON DELETE SET NULL ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "package_files" ADD CONSTRAINT "package_files_packageId_fkey" FOREIGN KEY ("packageId") REFERENCES "packages"("id") ON DELETE CASCADE ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "ingestion_runs" ADD CONSTRAINT "ingestion_runs_accountId_fkey" FOREIGN KEY ("accountId") REFERENCES "telegram_accounts"("id") ON DELETE RESTRICT ON UPDATE CASCADE;

View File

@@ -0,0 +1,11 @@
-- AlterTable
ALTER TABLE "ingestion_runs" ADD COLUMN "currentActivity" TEXT,
ADD COLUMN "currentChannel" TEXT,
ADD COLUMN "currentFile" TEXT,
ADD COLUMN "currentFileNum" INTEGER,
ADD COLUMN "currentStep" TEXT,
ADD COLUMN "downloadPercent" INTEGER,
ADD COLUMN "downloadedBytes" BIGINT,
ADD COLUMN "lastActivityAt" TIMESTAMP(3),
ADD COLUMN "totalBytes" BIGINT,
ADD COLUMN "totalFiles" INTEGER;

View File

@@ -0,0 +1,3 @@
-- AlterTable
ALTER TABLE "packages" ADD COLUMN "previewData" BYTEA,
ADD COLUMN "previewMsgId" BIGINT;

View File

@@ -0,0 +1,12 @@
/*
Warnings:
- You are about to drop the column `apiHash` on the `telegram_accounts` table. All the data in the column will be lost.
- You are about to drop the column `apiId` on the `telegram_accounts` table. All the data in the column will be lost.
- You are about to drop the column `sessionPath` on the `telegram_accounts` table. All the data in the column will be lost.
*/
-- AlterTable
ALTER TABLE "telegram_accounts" DROP COLUMN "apiHash",
DROP COLUMN "apiId",
DROP COLUMN "sessionPath";

View File

@@ -0,0 +1,29 @@
-- AlterTable
ALTER TABLE "packages" ADD COLUMN "creator" TEXT,
ADD COLUMN "sourceTopicId" BIGINT;
-- AlterTable
ALTER TABLE "telegram_channels" ADD COLUMN "isForum" BOOLEAN NOT NULL DEFAULT false;
-- CreateTable
CREATE TABLE "topic_progress" (
"id" TEXT NOT NULL,
"accountChannelMapId" TEXT NOT NULL,
"topicId" BIGINT NOT NULL,
"topicName" TEXT,
"lastProcessedMessageId" BIGINT,
CONSTRAINT "topic_progress_pkey" PRIMARY KEY ("id")
);
-- CreateIndex
CREATE INDEX "topic_progress_accountChannelMapId_idx" ON "topic_progress"("accountChannelMapId");
-- CreateIndex
CREATE UNIQUE INDEX "topic_progress_accountChannelMapId_topicId_key" ON "topic_progress"("accountChannelMapId", "topicId");
-- CreateIndex
CREATE INDEX "packages_creator_idx" ON "packages"("creator");
-- AddForeignKey
ALTER TABLE "topic_progress" ADD CONSTRAINT "topic_progress_accountChannelMapId_fkey" FOREIGN KEY ("accountChannelMapId") REFERENCES "account_channel_map"("id") ON DELETE CASCADE ON UPDATE CASCADE;

View File

@@ -349,3 +349,189 @@ model UserSettings {
user User @relation(fields: [userId], references: [id], onDelete: Cascade) user User @relation(fields: [userId], references: [id], onDelete: Cascade)
} }
// ───────────────────────────────────────
// Telegram ingestion models
// ───────────────────────────────────────
enum AuthState {
PENDING
AWAITING_CODE
AWAITING_PASSWORD
AUTHENTICATED
EXPIRED
}
enum ChannelType {
SOURCE
DESTINATION
}
enum ChannelRole {
READER
WRITER
}
enum ArchiveType {
ZIP
RAR
}
enum IngestionStatus {
RUNNING
COMPLETED
FAILED
CANCELLED
}
model TelegramAccount {
id String @id @default(cuid())
phone String @unique
displayName String?
isActive Boolean @default(true)
authState AuthState @default(PENDING)
authCode String?
lastSeenAt DateTime?
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
channelMaps AccountChannelMap[]
ingestionRuns IngestionRun[]
@@index([isActive])
@@map("telegram_accounts")
}
model TelegramChannel {
id String @id @default(cuid())
telegramId BigInt @unique
title String
type ChannelType
isForum Boolean @default(false)
isActive Boolean @default(true)
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
accountMaps AccountChannelMap[]
packages Package[]
@@index([type, isActive])
@@map("telegram_channels")
}
model AccountChannelMap {
id String @id @default(cuid())
accountId String
channelId String
role ChannelRole @default(READER)
lastProcessedMessageId BigInt?
createdAt DateTime @default(now())
account TelegramAccount @relation(fields: [accountId], references: [id], onDelete: Cascade)
channel TelegramChannel @relation(fields: [channelId], references: [id], onDelete: Cascade)
topicProgress TopicProgress[]
@@unique([accountId, channelId])
@@index([accountId])
@@index([channelId])
@@map("account_channel_map")
}
model Package {
id String @id @default(cuid())
contentHash String @unique
fileName String
fileSize BigInt
archiveType ArchiveType
creator String?
sourceChannelId String
sourceMessageId BigInt
sourceTopicId BigInt?
destChannelId String?
destMessageId BigInt?
isMultipart Boolean @default(false)
partCount Int @default(1)
fileCount Int @default(0)
previewData Bytes? // JPEG thumbnail from nearby Telegram photo (stored as raw bytes)
previewMsgId BigInt? // Telegram message ID of the matched photo
indexedAt DateTime @default(now())
createdAt DateTime @default(now())
sourceChannel TelegramChannel @relation(fields: [sourceChannelId], references: [id])
files PackageFile[]
ingestionRun IngestionRun? @relation(fields: [ingestionRunId], references: [id])
ingestionRunId String?
@@index([sourceChannelId])
@@index([destChannelId])
@@index([fileName])
@@index([indexedAt])
@@index([archiveType])
@@index([creator])
@@map("packages")
}
model PackageFile {
id String @id @default(cuid())
packageId String
path String
fileName String
extension String?
compressedSize BigInt @default(0)
uncompressedSize BigInt @default(0)
crc32 String?
package Package @relation(fields: [packageId], references: [id], onDelete: Cascade)
@@index([packageId])
@@index([extension])
@@index([fileName])
@@map("package_files")
}
model IngestionRun {
id String @id @default(cuid())
accountId String
status IngestionStatus @default(RUNNING)
startedAt DateTime @default(now())
finishedAt DateTime?
messagesScanned Int @default(0)
zipsFound Int @default(0)
zipsDuplicate Int @default(0)
zipsIngested Int @default(0)
errorMessage String?
// Live activity tracking — written by worker in real-time
currentActivity String? // Human-readable: "Downloading pack.zip (part 2/5)"
currentStep String? // Machine-readable step key
currentChannel String? // Channel title being processed
currentFile String? // File currently being processed
currentFileNum Int? // Which archive set (1-indexed)
totalFiles Int? // Total archive sets found
downloadedBytes BigInt? // Current download progress in bytes
totalBytes BigInt? // Total size of current download
downloadPercent Int? // 0-100
lastActivityAt DateTime? // When activity was last updated
account TelegramAccount @relation(fields: [accountId], references: [id])
packages Package[]
@@index([accountId])
@@index([status])
@@index([startedAt])
@@map("ingestion_runs")
}
model TopicProgress {
id String @id @default(cuid())
accountChannelMapId String
topicId BigInt
topicName String?
lastProcessedMessageId BigInt?
accountChannelMap AccountChannelMap @relation(fields: [accountChannelMapId], references: [id], onDelete: Cascade)
@@unique([accountChannelMapId, topicId])
@@index([accountChannelMapId])
@@map("topic_progress")
}

View File

@@ -0,0 +1,149 @@
"use client";
import { useEffect, useState } from "react";
import { Loader2, CheckCircle2, XCircle, CloudOff } from "lucide-react";
import { cn } from "@/lib/utils";
import type { IngestionAccountStatus } from "@/lib/telegram/types";
interface IngestionStatusProps {
initialStatus: IngestionAccountStatus[];
}
/**
* Polls /api/ingestion/status every 3 seconds while a run is active,
* or every 30 seconds when idle. Shows a compact status banner with
* a spinning throbber when ingestion is running.
*/
export function IngestionStatus({ initialStatus }: IngestionStatusProps) {
const [accounts, setAccounts] = useState(initialStatus);
const [error, setError] = useState(false);
// Determine if any account is currently running
const activeRun = accounts.find((a) => a.currentRun);
const isRunning = !!activeRun;
useEffect(() => {
let timer: ReturnType<typeof setTimeout>;
let mounted = true;
const poll = async () => {
try {
const res = await fetch("/api/ingestion/status");
if (!res.ok) throw new Error("fetch failed");
const data = await res.json();
if (mounted) {
setAccounts(data.accounts ?? []);
setError(false);
}
} catch {
if (mounted) setError(true);
}
if (mounted) {
// Poll fast while running, slow when idle
const interval = accounts.some((a) => a.currentRun) ? 3_000 : 30_000;
timer = setTimeout(poll, interval);
}
};
// Start polling after a short delay to avoid double-fetching on mount
timer = setTimeout(poll, 3_000);
return () => {
mounted = false;
clearTimeout(timer);
};
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [isRunning]);
// Nothing to show if no accounts configured
if (accounts.length === 0 && !error) return null;
// If we can't reach the API, show a muted offline badge
if (error) {
return (
<div className="flex items-center gap-2 rounded-lg border border-border bg-card px-3 py-2 text-xs text-muted-foreground">
<CloudOff className="h-3.5 w-3.5" />
<span>Sync status unavailable</span>
</div>
);
}
// Active run — show throbber with live activity
if (activeRun?.currentRun) {
const run = activeRun.currentRun;
return (
<div className="flex items-center gap-3 rounded-lg border border-primary/20 bg-primary/5 px-3 py-2">
<Loader2 className="h-4 w-4 shrink-0 animate-spin text-primary" />
<div className="min-w-0 flex-1">
<p className="truncate text-sm font-medium text-primary">
{run.currentActivity ?? "Syncing..."}
</p>
{run.downloadPercent != null && run.downloadPercent > 0 && (
<div className="mt-1 flex items-center gap-2">
<div className="h-1.5 w-24 rounded-full bg-primary/20">
<div
className="h-full rounded-full bg-primary transition-all duration-300"
style={{ width: `${Math.min(100, run.downloadPercent)}%` }}
/>
</div>
<span className="text-[10px] text-primary/70">{run.downloadPercent}%</span>
</div>
)}
</div>
{run.totalFiles != null && run.currentFileNum != null && (
<span className="shrink-0 text-[10px] text-primary/60">
{run.currentFileNum}/{run.totalFiles}
</span>
)}
</div>
);
}
// All idle — show last run summary
const lastCompleted = accounts
.filter((a) => a.lastRun)
.sort(
(a, b) =>
new Date(b.lastRun!.finishedAt ?? b.lastRun!.startedAt).getTime() -
new Date(a.lastRun!.finishedAt ?? a.lastRun!.startedAt).getTime()
)[0];
if (!lastCompleted?.lastRun) return null;
const last = lastCompleted.lastRun;
const isFailed = last.status === "FAILED";
const timeAgo = getTimeAgo(last.finishedAt ?? last.startedAt);
return (
<div
className={cn(
"flex items-center gap-2 rounded-lg border px-3 py-2 text-xs",
isFailed
? "border-red-500/20 bg-red-500/5 text-red-400"
: "border-border bg-card text-muted-foreground"
)}
>
{isFailed ? (
<XCircle className="h-3.5 w-3.5 shrink-0" />
) : (
<CheckCircle2 className="h-3.5 w-3.5 shrink-0 text-emerald-400" />
)}
<span className="truncate">
{isFailed
? `Last sync failed ${timeAgo}`
: `Last sync ${timeAgo}${last.zipsIngested} new, ${last.zipsDuplicate} skipped`}
</span>
</div>
);
}
function getTimeAgo(dateStr: string): string {
const diff = Date.now() - new Date(dateStr).getTime();
const mins = Math.floor(diff / 60_000);
if (mins < 1) return "just now";
if (mins < 60) return `${mins}m ago`;
const hours = Math.floor(mins / 60);
if (hours < 24) return `${hours}h ago`;
const days = Math.floor(hours / 24);
return `${days}d ago`;
}

View File

@@ -0,0 +1,154 @@
"use client";
import { type ColumnDef } from "@tanstack/react-table";
import { FileArchive, Eye, ImageIcon } from "lucide-react";
import { DataTableColumnHeader } from "@/components/shared/data-table-column-header";
import { Badge } from "@/components/ui/badge";
import { Button } from "@/components/ui/button";
export interface PackageRow {
id: string;
fileName: string;
fileSize: string;
contentHash: string;
archiveType: "ZIP" | "RAR";
fileCount: number;
isMultipart: boolean;
hasPreview: boolean;
creator: string | null;
indexedAt: string;
sourceChannel: {
id: string;
title: string;
};
}
interface PackageColumnsProps {
onViewFiles: (pkg: PackageRow) => void;
}
function formatBytes(bytesStr: string): string {
const bytes = Number(bytesStr);
if (bytes === 0) return "0 B";
const k = 1024;
const sizes = ["B", "KB", "MB", "GB", "TB"];
const i = Math.floor(Math.log(bytes) / Math.log(k));
return `${parseFloat((bytes / Math.pow(k, i)).toFixed(1))} ${sizes[i]}`;
}
function PreviewCell({ pkg }: { pkg: PackageRow }) {
if (pkg.hasPreview) {
return (
<img
src={`/api/zips/${pkg.id}/preview`}
alt=""
className="h-9 w-9 rounded-md object-cover bg-muted"
loading="lazy"
/>
);
}
return (
<div className="flex h-9 w-9 items-center justify-center rounded-md bg-muted">
<FileArchive className="h-4 w-4 text-muted-foreground" />
</div>
);
}
export function getPackageColumns({
onViewFiles,
}: PackageColumnsProps): ColumnDef<PackageRow, unknown>[] {
return [
{
id: "preview",
header: "",
cell: ({ row }) => <PreviewCell pkg={row.original} />,
enableHiding: false,
enableSorting: false,
size: 52,
},
{
accessorKey: "fileName",
header: ({ column }) => <DataTableColumnHeader column={column} title="File Name" />,
cell: ({ row }) => (
<div className="flex items-center gap-2 min-w-0">
<span className="font-medium truncate max-w-[300px]">{row.original.fileName}</span>
{row.original.isMultipart && (
<Badge variant="outline" className="text-[10px] shrink-0">
Multi
</Badge>
)}
</div>
),
enableHiding: false,
},
{
accessorKey: "archiveType",
header: ({ column }) => <DataTableColumnHeader column={column} title="Type" />,
cell: ({ row }) => (
<Badge variant="secondary" className="text-[10px]">
{row.original.archiveType}
</Badge>
),
},
{
accessorKey: "fileSize",
header: ({ column }) => <DataTableColumnHeader column={column} title="Size" />,
cell: ({ row }) => (
<span className="text-sm text-muted-foreground">
{formatBytes(row.original.fileSize)}
</span>
),
},
{
accessorKey: "fileCount",
header: ({ column }) => <DataTableColumnHeader column={column} title="Files" />,
cell: ({ row }) => (
<span className="text-sm">
{row.original.fileCount.toLocaleString()}
</span>
),
},
{
accessorKey: "creator",
header: ({ column }) => <DataTableColumnHeader column={column} title="Creator" />,
cell: ({ row }) => (
<span className="text-sm text-muted-foreground truncate max-w-[160px] block">
{row.original.creator ?? "\u2014"}
</span>
),
},
{
id: "channel",
header: ({ column }) => <DataTableColumnHeader column={column} title="Source" />,
cell: ({ row }) => (
<span className="text-sm text-muted-foreground truncate max-w-[160px] block">
{row.original.sourceChannel.title}
</span>
),
accessorFn: (row) => row.sourceChannel.title,
},
{
accessorKey: "indexedAt",
header: ({ column }) => <DataTableColumnHeader column={column} title="Indexed" />,
cell: ({ row }) => (
<span className="text-sm text-muted-foreground">
{new Date(row.original.indexedAt).toLocaleDateString()}
</span>
),
},
{
id: "actions",
cell: ({ row }) => (
<Button
variant="ghost"
size="icon"
className="h-8 w-8"
onClick={() => onViewFiles(row.original)}
>
<Eye className="h-4 w-4" />
</Button>
),
enableHiding: false,
},
];
}

View File

@@ -0,0 +1,412 @@
"use client";
import { useEffect, useState, useCallback, useMemo } from "react";
import {
FileText,
Folder,
FolderOpen,
Loader2,
Search,
ChevronDown,
ChevronRight,
} from "lucide-react";
import {
Dialog,
DialogContent,
DialogHeader,
DialogTitle,
DialogDescription,
} from "@/components/ui/dialog";
import { ScrollArea } from "@/components/ui/scroll-area";
import { Badge } from "@/components/ui/badge";
import { Input } from "@/components/ui/input";
import { Button } from "@/components/ui/button";
import { cn } from "@/lib/utils";
import type { PackageRow } from "./package-columns";
interface FileItem {
id: string;
path: string;
fileName: string;
extension: string | null;
compressedSize: string;
uncompressedSize: string;
crc32: string | null;
}
interface TreeNode {
name: string;
isFolder: boolean;
children: Map<string, TreeNode>;
file?: FileItem;
}
interface PackageFilesDrawerProps {
pkg: PackageRow | null;
open: boolean;
onOpenChange: (open: boolean) => void;
}
function formatBytes(bytesStr: string): string {
const bytes = Number(bytesStr);
if (bytes === 0) return "0 B";
const k = 1024;
const sizes = ["B", "KB", "MB", "GB", "TB"];
const i = Math.floor(Math.log(bytes) / Math.log(k));
return `${parseFloat((bytes / Math.pow(k, i)).toFixed(1))} ${sizes[i]}`;
}
const EXTENSION_COLORS: Record<string, string> = {
stl: "bg-blue-500/15 text-blue-400 border-blue-500/30",
obj: "bg-violet-500/15 text-violet-400 border-violet-500/30",
"3mf": "bg-cyan-500/15 text-cyan-400 border-cyan-500/30",
gcode: "bg-amber-500/15 text-amber-400 border-amber-500/30",
png: "bg-emerald-500/15 text-emerald-400 border-emerald-500/30",
jpg: "bg-emerald-500/15 text-emerald-400 border-emerald-500/30",
jpeg: "bg-emerald-500/15 text-emerald-400 border-emerald-500/30",
pdf: "bg-red-500/15 text-red-400 border-red-500/30",
txt: "bg-zinc-500/15 text-zinc-400 border-zinc-500/30",
lys: "bg-pink-500/15 text-pink-400 border-pink-500/30",
};
function getExtBadgeClass(ext: string | null): string {
if (!ext) return "bg-zinc-500/15 text-zinc-400 border-zinc-500/30";
return EXTENSION_COLORS[ext.toLowerCase()] ?? "bg-zinc-500/15 text-zinc-400 border-zinc-500/30";
}
/**
* Build a tree structure from flat file paths.
*/
function buildFileTree(files: FileItem[]): TreeNode {
const root: TreeNode = { name: "", isFolder: true, children: new Map() };
for (const file of files) {
// Normalize path separators (Windows RAR archives may use backslashes)
const parts = file.path.replace(/\\/g, "/").split("/").filter(Boolean);
let current = root;
for (let i = 0; i < parts.length; i++) {
const part = parts[i];
const isLast = i === parts.length - 1;
if (!current.children.has(part)) {
current.children.set(part, {
name: part,
isFolder: !isLast,
children: new Map(),
file: isLast ? file : undefined,
});
}
current = current.children.get(part)!;
}
}
return root;
}
/**
* Recursively renders a file tree node with indentation.
*/
function TreeNodeView({
node,
depth,
search,
defaultOpen,
}: {
node: TreeNode;
depth: number;
search: string;
defaultOpen: boolean;
}) {
const [open, setOpen] = useState(defaultOpen);
// Sort children: folders first, then files, alphabetical within each group
const sortedChildren = useMemo(() => {
const arr = Array.from(node.children.values());
return arr.sort((a, b) => {
if (a.isFolder !== b.isFolder) return a.isFolder ? -1 : 1;
return a.name.localeCompare(b.name);
});
}, [node.children]);
// If searching, force all open
useEffect(() => {
if (search) setOpen(true);
}, [search]);
if (node.isFolder && node.children.size > 0) {
return (
<div>
{/* Don't render a row for the root node */}
{depth >= 0 && (
<button
type="button"
className="flex w-full items-center gap-1.5 rounded-md px-1 py-1 text-sm hover:bg-muted/50 transition-colors"
style={{ paddingLeft: `${Math.max(0, depth) * 16 + 4}px` }}
onClick={() => setOpen(!open)}
>
{open ? (
<ChevronDown className="h-3 w-3 shrink-0 text-muted-foreground" />
) : (
<ChevronRight className="h-3 w-3 shrink-0 text-muted-foreground" />
)}
{open ? (
<FolderOpen className="h-3.5 w-3.5 shrink-0 text-primary/70" />
) : (
<Folder className="h-3.5 w-3.5 shrink-0 text-primary/70" />
)}
<span className="text-sm font-medium truncate">{node.name}</span>
<span className="text-[10px] text-muted-foreground ml-auto shrink-0">
{countFiles(node)}
</span>
</button>
)}
{open &&
sortedChildren.map((child) => (
<TreeNodeView
key={child.name}
node={child}
depth={depth + 1}
search={search}
defaultOpen={depth < 1} // Auto-expand first 2 levels
/>
))}
</div>
);
}
// File node
if (node.file) {
return (
<div
className="flex items-center gap-2 rounded-md px-1 py-1 hover:bg-muted/50 transition-colors"
style={{ paddingLeft: `${Math.max(0, depth) * 16 + 4}px` }}
>
<FileText className="h-3.5 w-3.5 shrink-0 text-muted-foreground" />
<span className="text-sm truncate flex-1 min-w-0" title={node.file.path}>
{node.name}
</span>
{node.file.extension && (
<Badge
variant="outline"
className={`text-[10px] shrink-0 ${getExtBadgeClass(node.file.extension)}`}
>
.{node.file.extension}
</Badge>
)}
<span className="text-[11px] text-muted-foreground shrink-0 tabular-nums">
{formatBytes(node.file.uncompressedSize)}
</span>
</div>
);
}
return null;
}
function countFiles(node: TreeNode): number {
if (!node.isFolder) return 1;
let count = 0;
for (const child of node.children.values()) {
count += countFiles(child);
}
return count;
}
const PAGE_SIZE = 100;
export function PackageFilesDrawer({ pkg, open, onOpenChange }: PackageFilesDrawerProps) {
const [files, setFiles] = useState<FileItem[]>([]);
const [total, setTotal] = useState(0);
const [loading, setLoading] = useState(false);
const [loadingMore, setLoadingMore] = useState(false);
const [search, setSearch] = useState("");
const [page, setPage] = useState(1);
const fetchFiles = useCallback(
async (pageNum: number, append: boolean) => {
if (!pkg) return;
if (pageNum === 1) setLoading(true);
else setLoadingMore(true);
try {
const params = new URLSearchParams({
page: String(pageNum),
limit: String(PAGE_SIZE),
});
const res = await fetch(`/api/zips/${pkg.id}/files?${params}`);
if (!res.ok) throw new Error("fetch failed");
const data = await res.json();
setFiles((prev) => (append ? [...prev, ...data.items] : data.items));
setTotal(data.pagination.total);
} catch {
// Silently handle
} finally {
setLoading(false);
setLoadingMore(false);
}
},
[pkg]
);
// Reset and fetch when package changes
useEffect(() => {
if (open && pkg) {
setFiles([]);
setTotal(0);
setSearch("");
setPage(1);
fetchFiles(1, false);
}
}, [open, pkg, fetchFiles]);
const loadMore = () => {
const nextPage = page + 1;
setPage(nextPage);
fetchFiles(nextPage, true);
};
const hasMore = files.length < total;
// Client-side search filter (over loaded files)
const filtered = search
? files.filter(
(f) =>
f.fileName.toLowerCase().includes(search.toLowerCase()) ||
f.path.toLowerCase().includes(search.toLowerCase())
)
: files;
// Build tree from filtered files
const tree = useMemo(() => buildFileTree(filtered), [filtered]);
// If all files are in root (no folders), skip the tree and show flat list
const hasNesting = useMemo(() => {
return filtered.some((f) => f.path.replace(/\\/g, "/").includes("/"));
}, [filtered]);
return (
<Dialog open={open} onOpenChange={onOpenChange}>
<DialogContent className="sm:max-w-2xl max-h-[80vh] flex flex-col gap-0 p-0">
<DialogHeader className="px-6 pt-6 pb-4 border-b border-border space-y-3">
{/* Preview image + title row */}
<div className="flex gap-4">
{pkg?.hasPreview && (
<img
src={`/api/zips/${pkg.id}/preview`}
alt=""
className="h-20 w-20 rounded-lg object-cover bg-muted shrink-0"
/>
)}
<div className="min-w-0 flex-1">
<DialogTitle className="truncate pr-8">
{pkg?.fileName ?? "Package Files"}
</DialogTitle>
<DialogDescription className="mt-1">
{total.toLocaleString()} file{total !== 1 ? "s" : ""} in archive
</DialogDescription>
</div>
</div>
{/* Search within file list */}
{files.length > 0 && (
<div className="relative">
<Search className="absolute left-2.5 top-2.5 h-4 w-4 text-muted-foreground" />
<Input
placeholder="Filter files..."
value={search}
onChange={(e) => setSearch(e.target.value)}
className="pl-9 h-9"
/>
</div>
)}
</DialogHeader>
<ScrollArea className="flex-1 min-h-0">
<div className="px-4 py-3">
{loading ? (
<div className="flex flex-col items-center justify-center gap-2 py-12">
<Loader2 className="h-5 w-5 animate-spin text-muted-foreground" />
<span className="text-sm text-muted-foreground">Loading files...</span>
</div>
) : filtered.length === 0 ? (
<div className="flex flex-col items-center justify-center gap-2 py-12">
<FileText className="h-6 w-6 text-muted-foreground/50" />
<span className="text-sm text-muted-foreground">
{search ? "No matching files" : "No files indexed"}
</span>
</div>
) : hasNesting ? (
<>
{/* Render as folder tree */}
{Array.from(tree.children.values())
.sort((a, b) => {
if (a.isFolder !== b.isFolder) return a.isFolder ? -1 : 1;
return a.name.localeCompare(b.name);
})
.map((child) => (
<TreeNodeView
key={child.name}
node={child}
depth={0}
search={search}
defaultOpen={true}
/>
))}
</>
) : (
<>
{/* Flat list for archives without folders */}
{filtered.map((file) => (
<div
key={file.id}
className="flex items-center gap-3 rounded-md px-2 py-1.5 hover:bg-muted/50 transition-colors"
>
<FileText className="h-3.5 w-3.5 shrink-0 text-muted-foreground" />
<div className="min-w-0 flex-1">
<p className="text-sm truncate" title={file.path}>
{file.fileName}
</p>
</div>
{file.extension && (
<Badge
variant="outline"
className={`text-[10px] shrink-0 ${getExtBadgeClass(file.extension)}`}
>
.{file.extension}
</Badge>
)}
<span className="text-[11px] text-muted-foreground shrink-0 tabular-nums">
{formatBytes(file.uncompressedSize)}
</span>
</div>
))}
</>
)}
{/* Load more button */}
{hasMore && !search && (
<div className="flex justify-center pt-3 pb-1">
<Button
variant="outline"
size="sm"
onClick={loadMore}
disabled={loadingMore}
className="gap-1"
>
{loadingMore ? (
<Loader2 className="h-3.5 w-3.5 animate-spin" />
) : (
<ChevronDown className="h-3.5 w-3.5" />
)}
Load more ({files.length} of {total.toLocaleString()})
</Button>
</div>
)}
</div>
</ScrollArea>
</DialogContent>
</Dialog>
);
}

View File

@@ -0,0 +1,95 @@
"use client";
import { useState, useCallback } from "react";
import { useRouter, usePathname, useSearchParams } from "next/navigation";
import { Search, FileBox } from "lucide-react";
import { useDataTable } from "@/hooks/use-data-table";
import { getPackageColumns, type PackageRow } from "./package-columns";
import { PackageFilesDrawer } from "./package-files-drawer";
import { IngestionStatus } from "./ingestion-status";
import { DataTable } from "@/components/shared/data-table";
import { DataTablePagination } from "@/components/shared/data-table-pagination";
import { DataTableViewOptions } from "@/components/shared/data-table-view-options";
import { PageHeader } from "@/components/shared/page-header";
import { Input } from "@/components/ui/input";
import type { IngestionAccountStatus } from "@/lib/telegram/types";
interface StlTableProps {
data: PackageRow[];
pageCount: number;
totalCount: number;
ingestionStatus: IngestionAccountStatus[];
}
export function StlTable({
data,
pageCount,
totalCount,
ingestionStatus,
}: StlTableProps) {
const router = useRouter();
const pathname = usePathname();
const searchParams = useSearchParams();
const [searchValue, setSearchValue] = useState(searchParams.get("search") ?? "");
const [viewPkg, setViewPkg] = useState<PackageRow | null>(null);
const updateSearch = useCallback(
(value: string) => {
setSearchValue(value);
const params = new URLSearchParams(searchParams.toString());
if (value) {
params.set("search", value);
params.set("page", "1");
} else {
params.delete("search");
}
router.push(`${pathname}?${params.toString()}`, { scroll: false });
},
[router, pathname, searchParams]
);
const columns = getPackageColumns({
onViewFiles: (pkg) => setViewPkg(pkg),
});
const { table } = useDataTable({ data, columns, pageCount });
return (
<div className="space-y-4">
<PageHeader
title="STL Files"
description="Browse indexed archive packages from Telegram channels"
>
<IngestionStatus initialStatus={ingestionStatus} />
</PageHeader>
<div className="flex flex-wrap items-center gap-2">
<div className="relative flex-1 min-w-[200px] max-w-sm">
<Search className="absolute left-2.5 top-2.5 h-4 w-4 text-muted-foreground" />
<Input
placeholder="Search packages or files..."
value={searchValue}
onChange={(e) => updateSearch(e.target.value)}
className="pl-9 h-9"
/>
</div>
<DataTableViewOptions table={table} />
</div>
<DataTable
table={table}
emptyMessage="No packages found. Archives will appear here after ingestion."
/>
<DataTablePagination table={table} totalCount={totalCount} />
<PackageFilesDrawer
pkg={viewPkg}
open={!!viewPkg}
onOpenChange={(open) => {
if (!open) setViewPkg(null);
}}
/>
</div>
);
}

View File

@@ -0,0 +1,50 @@
import { auth } from "@/lib/auth";
import { redirect } from "next/navigation";
import { listPackages, searchPackages, getIngestionStatus } from "@/lib/telegram/queries";
import { StlTable } from "./_components/stl-table";
interface Props {
searchParams: Promise<Record<string, string | string[] | undefined>>;
}
export default async function StlFilesPage({ searchParams }: Props) {
const session = await auth();
if (!session?.user?.id) redirect("/login");
const params = await searchParams;
const page = Number(params.page) || 1;
const perPage = Number(params.perPage) || 20;
const sort = (params.sort as string) ?? "indexedAt";
const order = (params.order as "asc" | "desc") ?? "desc";
const search = (params.search as string) ?? "";
const creator = (params.creator as string) || undefined;
// Fetch packages and ingestion status in parallel
const [result, ingestionStatus] = await Promise.all([
search
? searchPackages({
query: search,
page,
limit: perPage,
searchIn: "both",
})
: listPackages({
page,
limit: perPage,
creator,
sortBy: sort as "indexedAt" | "fileName" | "fileSize",
order,
}),
getIngestionStatus(),
]);
return (
<StlTable
data={result.items}
pageCount={result.pagination.totalPages}
totalCount={result.pagination.total}
ingestionStatus={ingestionStatus}
/>
);
}

View File

@@ -0,0 +1,184 @@
"use client";
import { type ColumnDef } from "@tanstack/react-table";
import {
MoreHorizontal,
Pencil,
Trash2,
Power,
Link2,
Play,
KeyRound,
} from "lucide-react";
import { Badge } from "@/components/ui/badge";
import { Button } from "@/components/ui/button";
import {
DropdownMenu,
DropdownMenuContent,
DropdownMenuItem,
DropdownMenuSeparator,
DropdownMenuTrigger,
} from "@/components/ui/dropdown-menu";
import type { AccountRow } from "@/lib/telegram/admin-queries";
const authStateColors: Record<string, string> = {
PENDING: "bg-yellow-500/10 text-yellow-600 border-yellow-500/20",
AWAITING_CODE: "bg-orange-500/10 text-orange-600 border-orange-500/20",
AWAITING_PASSWORD: "bg-orange-500/10 text-orange-600 border-orange-500/20",
AUTHENTICATED: "bg-green-500/10 text-green-600 border-green-500/20",
EXPIRED: "bg-red-500/10 text-red-600 border-red-500/20",
};
interface AccountColumnsProps {
onEdit: (account: AccountRow) => void;
onToggleActive: (id: string) => void;
onDelete: (id: string) => void;
onViewLinks: (id: string) => void;
onTriggerSync: (id: string) => void;
onEnterCode: (account: AccountRow) => void;
}
export function getAccountColumns({
onEdit,
onToggleActive,
onDelete,
onViewLinks,
onTriggerSync,
onEnterCode,
}: AccountColumnsProps): ColumnDef<AccountRow, unknown>[] {
return [
{
accessorKey: "displayName",
header: "Account",
cell: ({ row }) => (
<div className="flex flex-col">
<span className="font-medium">
{row.original.displayName || row.original.phone}
</span>
{row.original.displayName && (
<span className="text-xs text-muted-foreground">
{row.original.phone}
</span>
)}
</div>
),
enableHiding: false,
},
{
accessorKey: "authState",
header: "Auth State",
cell: ({ row }) => {
const needsCode =
row.original.authState === "AWAITING_CODE" ||
row.original.authState === "AWAITING_PASSWORD";
return (
<div className="flex items-center gap-2">
<Badge
variant="outline"
className={authStateColors[row.original.authState] ?? ""}
>
{row.original.authState.replace(/_/g, " ")}
</Badge>
{needsCode && (
<Button
variant="outline"
size="sm"
className="h-6 gap-1 px-2 text-xs"
onClick={() => onEnterCode(row.original)}
>
<KeyRound className="h-3 w-3" />
Enter Code
</Button>
)}
</div>
);
},
},
{
accessorKey: "isActive",
header: "Status",
cell: ({ row }) => (
<Badge variant={row.original.isActive ? "default" : "secondary"}>
{row.original.isActive ? "Active" : "Disabled"}
</Badge>
),
},
{
id: "channels",
header: "Channels",
cell: ({ row }) => (
<Button
variant="ghost"
size="sm"
className="h-7 gap-1 px-2 text-xs"
onClick={() => onViewLinks(row.original.id)}
>
<Link2 className="h-3 w-3" />
{row.original.channelCount}
</Button>
),
},
{
id: "runs",
header: "Runs",
cell: ({ row }) => (
<span className="text-sm text-muted-foreground">
{row.original.runCount}
</span>
),
},
{
accessorKey: "lastSeenAt",
header: "Last Seen",
cell: ({ row }) =>
row.original.lastSeenAt ? (
<span className="text-sm text-muted-foreground">
{new Date(row.original.lastSeenAt).toLocaleDateString()}
</span>
) : (
<span className="text-sm text-muted-foreground">Never</span>
),
},
{
id: "actions",
cell: ({ row }) => (
<DropdownMenu>
<DropdownMenuTrigger asChild>
<Button variant="ghost" size="icon" className="h-8 w-8">
<MoreHorizontal className="h-4 w-4" />
</Button>
</DropdownMenuTrigger>
<DropdownMenuContent align="end">
<DropdownMenuItem onClick={() => onEdit(row.original)}>
<Pencil className="mr-2 h-3.5 w-3.5" />
Edit
</DropdownMenuItem>
<DropdownMenuItem onClick={() => onViewLinks(row.original.id)}>
<Link2 className="mr-2 h-3.5 w-3.5" />
Manage Channels
</DropdownMenuItem>
<DropdownMenuItem onClick={() => onTriggerSync(row.original.id)}>
<Play className="mr-2 h-3.5 w-3.5" />
Sync Now
</DropdownMenuItem>
<DropdownMenuItem
onClick={() => onToggleActive(row.original.id)}
>
<Power className="mr-2 h-3.5 w-3.5" />
{row.original.isActive ? "Disable" : "Enable"}
</DropdownMenuItem>
<DropdownMenuSeparator />
<DropdownMenuItem
onClick={() => onDelete(row.original.id)}
className="text-destructive focus:text-destructive"
>
<Trash2 className="mr-2 h-3.5 w-3.5" />
Delete
</DropdownMenuItem>
</DropdownMenuContent>
</DropdownMenu>
),
enableHiding: false,
},
];
}

View File

@@ -0,0 +1,102 @@
"use client";
import { useTransition } from "react";
import { useForm } from "react-hook-form";
import { zodResolver } from "@hookform/resolvers/zod";
import { toast } from "sonner";
import {
telegramAccountSchema,
type TelegramAccountInput,
} from "@/schemas/telegram";
import { createAccount, updateAccount } from "../actions";
import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
import {
Form,
FormControl,
FormDescription,
FormField,
FormItem,
FormLabel,
FormMessage,
} from "@/components/ui/form";
import type { AccountRow } from "@/lib/telegram/admin-queries";
interface AccountFormProps {
account?: AccountRow;
onSuccess: () => void;
}
export function AccountForm({ account, onSuccess }: AccountFormProps) {
const [isPending, startTransition] = useTransition();
const isEditing = !!account;
const form = useForm<TelegramAccountInput>({
// eslint-disable-next-line @typescript-eslint/no-explicit-any
resolver: zodResolver(telegramAccountSchema) as any,
defaultValues: {
phone: account?.phone ?? "",
displayName: account?.displayName ?? "",
},
});
function onSubmit(values: TelegramAccountInput) {
startTransition(async () => {
const result = isEditing
? await updateAccount(account!.id, values)
: await createAccount(values);
if (!result.success) {
toast.error(result.error);
return;
}
toast.success(isEditing ? "Account updated" : "Account created");
form.reset();
onSuccess();
});
}
return (
<Form {...form}>
<form onSubmit={form.handleSubmit(onSubmit)} className="space-y-4">
<FormField
control={form.control}
name="phone"
render={({ field }) => (
<FormItem>
<FormLabel>Phone Number</FormLabel>
<FormControl>
<Input placeholder="+31612345678" {...field} />
</FormControl>
<FormDescription>
International format with country code
</FormDescription>
<FormMessage />
</FormItem>
)}
/>
<FormField
control={form.control}
name="displayName"
render={({ field }) => (
<FormItem>
<FormLabel>Display Name</FormLabel>
<FormControl>
<Input placeholder="My Bot Account" {...field} />
</FormControl>
<FormMessage />
</FormItem>
)}
/>
<div className="flex justify-end gap-2">
<Button type="submit" disabled={isPending}>
{isPending ? "Saving..." : isEditing ? "Update" : "Create"}
</Button>
</div>
</form>
</Form>
);
}

View File

@@ -0,0 +1,233 @@
"use client";
import { useState, useEffect, useTransition, useCallback } from "react";
import { Link2Off, Plus } from "lucide-react";
import { toast } from "sonner";
import { linkChannel, unlinkChannel } from "../actions";
import {
Dialog,
DialogContent,
DialogDescription,
DialogHeader,
DialogTitle,
} from "@/components/ui/dialog";
import {
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from "@/components/ui/select";
import { Badge } from "@/components/ui/badge";
import { Button } from "@/components/ui/button";
import { Separator } from "@/components/ui/separator";
interface ChannelLink {
id: string;
channelId: string;
role: string;
lastProcessedMessageId: string | null;
channel: {
id: string;
title: string;
type: string;
telegramId: string;
};
}
interface UnlinkedChannel {
id: string;
title: string;
type: string;
telegramId: string;
}
interface AccountLinksDrawerProps {
accountId: string | null;
open: boolean;
onOpenChange: (open: boolean) => void;
}
export function AccountLinksDrawer({
accountId,
open,
onOpenChange,
}: AccountLinksDrawerProps) {
const [isPending, startTransition] = useTransition();
const [links, setLinks] = useState<ChannelLink[]>([]);
const [unlinked, setUnlinked] = useState<UnlinkedChannel[]>([]);
const [selectedChannelId, setSelectedChannelId] = useState("");
const [selectedRole, setSelectedRole] = useState<"READER" | "WRITER">("READER");
const [loading, setLoading] = useState(false);
const fetchLinks = useCallback(async () => {
if (!accountId) return;
setLoading(true);
try {
const [linksRes, unlinkedRes] = await Promise.all([
fetch(`/api/telegram/accounts/${accountId}/links`),
fetch(`/api/telegram/accounts/${accountId}/unlinked-channels`),
]);
if (linksRes.ok) setLinks(await linksRes.json());
if (unlinkedRes.ok) setUnlinked(await unlinkedRes.json());
} catch {
toast.error("Failed to load channel links");
}
setLoading(false);
}, [accountId]);
useEffect(() => {
if (open && accountId) {
fetchLinks();
}
}, [open, accountId, fetchLinks]);
const handleLink = () => {
if (!accountId || !selectedChannelId) return;
startTransition(async () => {
const result = await linkChannel({
accountId,
channelId: selectedChannelId,
role: selectedRole,
});
if (result.success) {
toast.success("Channel linked");
setSelectedChannelId("");
await fetchLinks();
} else {
toast.error(result.error);
}
});
};
const handleUnlink = (linkId: string) => {
startTransition(async () => {
const result = await unlinkChannel(linkId);
if (result.success) {
toast.success("Channel unlinked");
await fetchLinks();
} else {
toast.error(result.error);
}
});
};
return (
<Dialog open={open} onOpenChange={onOpenChange}>
<DialogContent className="sm:max-w-lg">
<DialogHeader>
<DialogTitle>Manage Channel Links</DialogTitle>
<DialogDescription>
Link channels to this account. The account will read from Source
channels and write to Destination channels.
</DialogDescription>
</DialogHeader>
{/* Add new link */}
{unlinked.length > 0 && (
<div className="space-y-3">
<h4 className="text-sm font-medium">Link a Channel</h4>
<div className="flex items-end gap-2">
<div className="flex-1">
<Select
value={selectedChannelId}
onValueChange={setSelectedChannelId}
>
<SelectTrigger>
<SelectValue placeholder="Select channel" />
</SelectTrigger>
<SelectContent>
{unlinked.map((ch) => (
<SelectItem key={ch.id} value={ch.id}>
{ch.title} ({ch.type})
</SelectItem>
))}
</SelectContent>
</Select>
</div>
<Select
value={selectedRole}
onValueChange={(v) => setSelectedRole(v as "READER" | "WRITER")}
>
<SelectTrigger className="w-28">
<SelectValue />
</SelectTrigger>
<SelectContent>
<SelectItem value="READER">Reader</SelectItem>
<SelectItem value="WRITER">Writer</SelectItem>
</SelectContent>
</Select>
<Button
size="sm"
disabled={!selectedChannelId || isPending}
onClick={handleLink}
>
<Plus className="mr-1 h-3.5 w-3.5" />
Link
</Button>
</div>
<Separator />
</div>
)}
{/* Existing links */}
<div className="space-y-2">
<h4 className="text-sm font-medium">
Linked Channels ({links.length})
</h4>
{loading ? (
<p className="text-sm text-muted-foreground">Loading...</p>
) : links.length === 0 ? (
<p className="text-sm text-muted-foreground">
No channels linked to this account.
</p>
) : (
<div className="space-y-2">
{links.map((link) => (
<div
key={link.id}
className="flex items-center justify-between rounded-md border p-3"
>
<div className="flex flex-col gap-1">
<div className="flex items-center gap-2">
<span className="text-sm font-medium">
{link.channel.title}
</span>
<Badge
variant="outline"
className={
link.channel.type === "SOURCE"
? "bg-blue-500/10 text-blue-600 border-blue-500/20"
: "bg-purple-500/10 text-purple-600 border-purple-500/20"
}
>
{link.channel.type}
</Badge>
<Badge variant="secondary" className="text-[10px]">
{link.role}
</Badge>
</div>
<span className="text-xs text-muted-foreground">
ID: {link.channel.telegramId}
{link.lastProcessedMessageId &&
` | Last msg: ${link.lastProcessedMessageId}`}
</span>
</div>
<Button
variant="ghost"
size="icon"
className="h-8 w-8 text-muted-foreground hover:text-destructive"
disabled={isPending}
onClick={() => handleUnlink(link.id)}
>
<Link2Off className="h-4 w-4" />
</Button>
</div>
))}
</div>
)}
</div>
</DialogContent>
</Dialog>
);
}

View File

@@ -0,0 +1,44 @@
"use client";
import {
Dialog,
DialogContent,
DialogDescription,
DialogHeader,
DialogTitle,
} from "@/components/ui/dialog";
import { AccountForm } from "./account-form";
import type { AccountRow } from "@/lib/telegram/admin-queries";
interface AccountModalProps {
open: boolean;
onOpenChange: (open: boolean) => void;
account?: AccountRow;
}
export function AccountModal({
open,
onOpenChange,
account,
}: AccountModalProps) {
return (
<Dialog open={open} onOpenChange={onOpenChange}>
<DialogContent className="sm:max-w-lg">
<DialogHeader>
<DialogTitle>
{account ? "Edit Account" : "Add Telegram Account"}
</DialogTitle>
<DialogDescription>
{account
? "Update the account details below."
: "Configure a new Telegram account for ingestion. You'll need an API ID and hash from my.telegram.org."}
</DialogDescription>
</DialogHeader>
<AccountForm
account={account}
onSuccess={() => onOpenChange(false)}
/>
</DialogContent>
</Dialog>
);
}

View File

@@ -0,0 +1,140 @@
"use client";
import { useState, useTransition } from "react";
import { Plus, Play } from "lucide-react";
import { toast } from "sonner";
import { getAccountColumns } from "./account-columns";
import { AccountModal } from "./account-modal";
import { AccountLinksDrawer } from "./account-links-drawer";
import { AuthCodeDialog } from "./auth-code-dialog";
import { deleteAccount, toggleAccountActive, triggerIngestion } from "../actions";
import { DataTable } from "@/components/shared/data-table";
import { DeleteDialog } from "@/components/shared/delete-dialog";
import { Button } from "@/components/ui/button";
import type { AccountRow } from "@/lib/telegram/admin-queries";
import { useDataTable } from "@/hooks/use-data-table";
interface AccountsTabProps {
accounts: AccountRow[];
}
export function AccountsTab({ accounts }: AccountsTabProps) {
const [isPending, startTransition] = useTransition();
const [modalOpen, setModalOpen] = useState(false);
const [editAccount, setEditAccount] = useState<AccountRow | undefined>();
const [deleteId, setDeleteId] = useState<string | null>(null);
const [linksAccountId, setLinksAccountId] = useState<string | null>(null);
const [authCodeAccount, setAuthCodeAccount] = useState<AccountRow | null>(null);
const columns = getAccountColumns({
onEdit: (account) => {
setEditAccount(account);
setModalOpen(true);
},
onToggleActive: (id) => {
startTransition(async () => {
const result = await toggleAccountActive(id);
if (result.success) toast.success("Account toggled");
else toast.error(result.error);
});
},
onDelete: (id) => setDeleteId(id),
onViewLinks: (id) => setLinksAccountId(id),
onEnterCode: (account) => setAuthCodeAccount(account),
onTriggerSync: (id) => {
startTransition(async () => {
const result = await triggerIngestion(id);
if (result.success) toast.success("Ingestion triggered");
else toast.error(result.error);
});
},
});
const { table } = useDataTable({
data: accounts,
columns,
pageCount: 1,
});
const handleDelete = () => {
if (!deleteId) return;
startTransition(async () => {
const result = await deleteAccount(deleteId);
if (result.success) {
toast.success("Account deleted");
setDeleteId(null);
} else {
toast.error(result.error);
}
});
};
return (
<div className="space-y-4">
<div className="flex items-center gap-2">
<Button
onClick={() => {
setEditAccount(undefined);
setModalOpen(true);
}}
>
<Plus className="mr-2 h-4 w-4" />
Add Account
</Button>
<Button
variant="outline"
disabled={isPending}
onClick={() => {
startTransition(async () => {
const result = await triggerIngestion();
if (result.success) toast.success("Ingestion triggered for all accounts");
else toast.error(result.error);
});
}}
>
<Play className="mr-2 h-4 w-4" />
Sync All
</Button>
</div>
<DataTable
table={table}
emptyMessage="No accounts configured. Add your first Telegram account."
/>
<AccountModal
open={modalOpen}
onOpenChange={(open) => {
setModalOpen(open);
if (!open) setEditAccount(undefined);
}}
account={editAccount}
/>
<DeleteDialog
open={!!deleteId}
onOpenChange={(open) => !open && setDeleteId(null)}
title="Delete Account"
description="This will permanently delete this Telegram account and all its channel links. Existing packages will NOT be deleted."
onConfirm={handleDelete}
isLoading={isPending}
/>
<AccountLinksDrawer
accountId={linksAccountId}
open={!!linksAccountId}
onOpenChange={(open) => {
if (!open) setLinksAccountId(null);
}}
/>
<AuthCodeDialog
account={authCodeAccount}
open={!!authCodeAccount}
onOpenChange={(open) => {
if (!open) setAuthCodeAccount(null);
}}
/>
</div>
);
}

View File

@@ -0,0 +1,102 @@
"use client";
import { useState, useTransition } from "react";
import { toast } from "sonner";
import { submitAuthCode } from "../actions";
import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
import {
Dialog,
DialogContent,
DialogDescription,
DialogFooter,
DialogHeader,
DialogTitle,
} from "@/components/ui/dialog";
import { Label } from "@/components/ui/label";
import type { AccountRow } from "@/lib/telegram/admin-queries";
interface AuthCodeDialogProps {
account: AccountRow | null;
open: boolean;
onOpenChange: (open: boolean) => void;
}
export function AuthCodeDialog({
account,
open,
onOpenChange,
}: AuthCodeDialogProps) {
const [code, setCode] = useState("");
const [isPending, startTransition] = useTransition();
const isPassword = account?.authState === "AWAITING_PASSWORD";
const title = isPassword ? "Enter 2FA Password" : "Enter Auth Code";
const description = isPassword
? "Your Telegram account requires a two-factor authentication password."
: "Enter the code sent to your Telegram app or SMS.";
const placeholder = isPassword ? "Password" : "12345";
function handleSubmit() {
if (!account || !code.trim()) return;
startTransition(async () => {
const result = await submitAuthCode(account.id, { code: code.trim() });
if (result.success) {
toast.success(isPassword ? "Password submitted" : "Code submitted");
setCode("");
onOpenChange(false);
} else {
toast.error(result.error);
}
});
}
return (
<Dialog
open={open}
onOpenChange={(v) => {
if (!v) setCode("");
onOpenChange(v);
}}
>
<DialogContent className="sm:max-w-md">
<DialogHeader>
<DialogTitle>{title}</DialogTitle>
<DialogDescription>{description}</DialogDescription>
</DialogHeader>
<div className="space-y-2">
<Label htmlFor="auth-code">
{isPassword ? "Password" : "Code"}
</Label>
<Input
id="auth-code"
type={isPassword ? "password" : "text"}
placeholder={placeholder}
value={code}
onChange={(e) => setCode(e.target.value)}
onKeyDown={(e) => {
if (e.key === "Enter") handleSubmit();
}}
autoFocus
/>
</div>
<DialogFooter>
<Button
variant="outline"
onClick={() => onOpenChange(false)}
disabled={isPending}
>
Cancel
</Button>
<Button
onClick={handleSubmit}
disabled={isPending || !code.trim()}
>
{isPending ? "Submitting..." : "Submit"}
</Button>
</DialogFooter>
</DialogContent>
</Dialog>
);
}

View File

@@ -0,0 +1,132 @@
"use client";
import { type ColumnDef } from "@tanstack/react-table";
import {
MoreHorizontal,
Pencil,
Trash2,
Power,
} from "lucide-react";
import { Badge } from "@/components/ui/badge";
import { Button } from "@/components/ui/button";
import {
DropdownMenu,
DropdownMenuContent,
DropdownMenuItem,
DropdownMenuSeparator,
DropdownMenuTrigger,
} from "@/components/ui/dropdown-menu";
import type { ChannelRow } from "@/lib/telegram/admin-queries";
interface ChannelColumnsProps {
onEdit: (channel: ChannelRow) => void;
onToggleActive: (id: string) => void;
onDelete: (id: string) => void;
}
export function getChannelColumns({
onEdit,
onToggleActive,
onDelete,
}: ChannelColumnsProps): ColumnDef<ChannelRow, unknown>[] {
return [
{
accessorKey: "title",
header: "Channel",
cell: ({ row }) => (
<div className="flex flex-col">
<span className="font-medium">{row.original.title}</span>
<span className="text-xs text-muted-foreground">
ID: {row.original.telegramId}
</span>
</div>
),
enableHiding: false,
},
{
accessorKey: "type",
header: "Type",
cell: ({ row }) => (
<Badge
variant="outline"
className={
row.original.type === "SOURCE"
? "bg-blue-500/10 text-blue-600 border-blue-500/20"
: "bg-purple-500/10 text-purple-600 border-purple-500/20"
}
>
{row.original.type}
</Badge>
),
},
{
accessorKey: "isActive",
header: "Status",
cell: ({ row }) => (
<Badge variant={row.original.isActive ? "default" : "secondary"}>
{row.original.isActive ? "Active" : "Disabled"}
</Badge>
),
},
{
id: "accounts",
header: "Accounts",
cell: ({ row }) => (
<span className="text-sm text-muted-foreground">
{row.original.accountCount}
</span>
),
},
{
id: "packages",
header: "Packages",
cell: ({ row }) => (
<span className="text-sm text-muted-foreground">
{row.original.packageCount}
</span>
),
},
{
accessorKey: "createdAt",
header: "Created",
cell: ({ row }) => (
<span className="text-sm text-muted-foreground">
{new Date(row.original.createdAt).toLocaleDateString()}
</span>
),
},
{
id: "actions",
cell: ({ row }) => (
<DropdownMenu>
<DropdownMenuTrigger asChild>
<Button variant="ghost" size="icon" className="h-8 w-8">
<MoreHorizontal className="h-4 w-4" />
</Button>
</DropdownMenuTrigger>
<DropdownMenuContent align="end">
<DropdownMenuItem onClick={() => onEdit(row.original)}>
<Pencil className="mr-2 h-3.5 w-3.5" />
Edit
</DropdownMenuItem>
<DropdownMenuItem
onClick={() => onToggleActive(row.original.id)}
>
<Power className="mr-2 h-3.5 w-3.5" />
{row.original.isActive ? "Disable" : "Enable"}
</DropdownMenuItem>
<DropdownMenuSeparator />
<DropdownMenuItem
onClick={() => onDelete(row.original.id)}
className="text-destructive focus:text-destructive"
>
<Trash2 className="mr-2 h-3.5 w-3.5" />
Delete
</DropdownMenuItem>
</DropdownMenuContent>
</DropdownMenu>
),
enableHiding: false,
},
];
}

View File

@@ -0,0 +1,142 @@
"use client";
import { useTransition } from "react";
import { useForm } from "react-hook-form";
import { zodResolver } from "@hookform/resolvers/zod";
import { toast } from "sonner";
import {
telegramChannelSchema,
type TelegramChannelInput,
} from "@/schemas/telegram";
import { createChannel, updateChannel } from "../actions";
import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
import {
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from "@/components/ui/select";
import {
Form,
FormControl,
FormDescription,
FormField,
FormItem,
FormLabel,
FormMessage,
} from "@/components/ui/form";
import type { ChannelRow } from "@/lib/telegram/admin-queries";
interface ChannelFormProps {
channel?: ChannelRow;
onSuccess: () => void;
}
export function ChannelForm({ channel, onSuccess }: ChannelFormProps) {
const [isPending, startTransition] = useTransition();
const isEditing = !!channel;
const form = useForm<TelegramChannelInput>({
// eslint-disable-next-line @typescript-eslint/no-explicit-any
resolver: zodResolver(telegramChannelSchema) as any,
defaultValues: {
telegramId: channel ? Number(channel.telegramId) : (0 as unknown as number),
title: channel?.title ?? "",
type: channel?.type ?? "SOURCE",
},
});
function onSubmit(values: TelegramChannelInput) {
startTransition(async () => {
const result = isEditing
? await updateChannel(channel!.id, values)
: await createChannel(values);
if (!result.success) {
toast.error(result.error);
return;
}
toast.success(isEditing ? "Channel updated" : "Channel created");
form.reset();
onSuccess();
});
}
return (
<Form {...form}>
<form onSubmit={form.handleSubmit(onSubmit)} className="space-y-4">
<FormField
control={form.control}
name="title"
render={({ field }) => (
<FormItem>
<FormLabel>Title</FormLabel>
<FormControl>
<Input placeholder="Channel name" {...field} />
</FormControl>
<FormMessage />
</FormItem>
)}
/>
<FormField
control={form.control}
name="telegramId"
render={({ field }) => (
<FormItem>
<FormLabel>Telegram ID</FormLabel>
<FormControl>
<Input
type="number"
placeholder="1234567890"
{...field}
value={field.value || ""}
/>
</FormControl>
<FormDescription>
Numeric ID of the Telegram channel or group
</FormDescription>
<FormMessage />
</FormItem>
)}
/>
<FormField
control={form.control}
name="type"
render={({ field }) => (
<FormItem>
<FormLabel>Type</FormLabel>
<Select
onValueChange={field.onChange}
defaultValue={field.value}
>
<FormControl>
<SelectTrigger>
<SelectValue placeholder="Select type" />
</SelectTrigger>
</FormControl>
<SelectContent>
<SelectItem value="SOURCE">Source (read archives)</SelectItem>
<SelectItem value="DESTINATION">
Destination (forward indexed)
</SelectItem>
</SelectContent>
</Select>
<FormMessage />
</FormItem>
)}
/>
<div className="flex justify-end gap-2">
<Button type="submit" disabled={isPending}>
{isPending ? "Saving..." : isEditing ? "Update" : "Create"}
</Button>
</div>
</form>
</Form>
);
}

View File

@@ -0,0 +1,44 @@
"use client";
import {
Dialog,
DialogContent,
DialogDescription,
DialogHeader,
DialogTitle,
} from "@/components/ui/dialog";
import { ChannelForm } from "./channel-form";
import type { ChannelRow } from "@/lib/telegram/admin-queries";
interface ChannelModalProps {
open: boolean;
onOpenChange: (open: boolean) => void;
channel?: ChannelRow;
}
export function ChannelModal({
open,
onOpenChange,
channel,
}: ChannelModalProps) {
return (
<Dialog open={open} onOpenChange={onOpenChange}>
<DialogContent className="sm:max-w-md">
<DialogHeader>
<DialogTitle>
{channel ? "Edit Channel" : "Add Channel"}
</DialogTitle>
<DialogDescription>
{channel
? "Update the channel details below."
: "Add a Telegram channel. Source channels are scanned for archives, destination channels receive indexed files."}
</DialogDescription>
</DialogHeader>
<ChannelForm
channel={channel}
onSuccess={() => onOpenChange(false)}
/>
</DialogContent>
</Dialog>
);
}

View File

@@ -0,0 +1,97 @@
"use client";
import { useState, useTransition } from "react";
import { Plus } from "lucide-react";
import { toast } from "sonner";
import { getChannelColumns } from "./channel-columns";
import { ChannelModal } from "./channel-modal";
import { deleteChannel, toggleChannelActive } from "../actions";
import { DataTable } from "@/components/shared/data-table";
import { DeleteDialog } from "@/components/shared/delete-dialog";
import { Button } from "@/components/ui/button";
import type { ChannelRow } from "@/lib/telegram/admin-queries";
import { useDataTable } from "@/hooks/use-data-table";
interface ChannelsTabProps {
channels: ChannelRow[];
}
export function ChannelsTab({ channels }: ChannelsTabProps) {
const [isPending, startTransition] = useTransition();
const [modalOpen, setModalOpen] = useState(false);
const [editChannel, setEditChannel] = useState<ChannelRow | undefined>();
const [deleteId, setDeleteId] = useState<string | null>(null);
const columns = getChannelColumns({
onEdit: (channel) => {
setEditChannel(channel);
setModalOpen(true);
},
onToggleActive: (id) => {
startTransition(async () => {
const result = await toggleChannelActive(id);
if (result.success) toast.success("Channel toggled");
else toast.error(result.error);
});
},
onDelete: (id) => setDeleteId(id),
});
const { table } = useDataTable({
data: channels,
columns,
pageCount: 1,
});
const handleDelete = () => {
if (!deleteId) return;
startTransition(async () => {
const result = await deleteChannel(deleteId);
if (result.success) {
toast.success("Channel deleted");
setDeleteId(null);
} else {
toast.error(result.error);
}
});
};
return (
<div className="space-y-4">
<div className="flex items-center gap-2">
<Button
onClick={() => {
setEditChannel(undefined);
setModalOpen(true);
}}
>
<Plus className="mr-2 h-4 w-4" />
Add Channel
</Button>
</div>
<DataTable
table={table}
emptyMessage="No channels configured. Add a Telegram channel to start ingesting."
/>
<ChannelModal
open={modalOpen}
onOpenChange={(open) => {
setModalOpen(open);
if (!open) setEditChannel(undefined);
}}
channel={editChannel}
/>
<DeleteDialog
open={!!deleteId}
onOpenChange={(open) => !open && setDeleteId(null)}
title="Delete Channel"
description="This will permanently delete this channel and unlink it from all accounts. Existing packages will NOT be deleted."
onConfirm={handleDelete}
isLoading={isPending}
/>
</div>
);
}

View File

@@ -0,0 +1,41 @@
"use client";
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
import { PageHeader } from "@/components/shared/page-header";
import { AccountsTab } from "./accounts-tab";
import { ChannelsTab } from "./channels-tab";
import type { AccountRow, ChannelRow } from "@/lib/telegram/admin-queries";
interface TelegramAdminProps {
accounts: AccountRow[];
channels: ChannelRow[];
}
export function TelegramAdmin({ accounts, channels }: TelegramAdminProps) {
return (
<div className="space-y-4">
<PageHeader
title="Telegram"
description="Manage Telegram accounts, channels, and ingestion"
/>
<Tabs defaultValue="accounts" className="space-y-4">
<TabsList>
<TabsTrigger value="accounts">
Accounts ({accounts.length})
</TabsTrigger>
<TabsTrigger value="channels">
Channels ({channels.length})
</TabsTrigger>
</TabsList>
<TabsContent value="accounts">
<AccountsTab accounts={accounts} />
</TabsContent>
<TabsContent value="channels">
<ChannelsTab channels={channels} />
</TabsContent>
</Tabs>
</div>
);
}

View File

@@ -0,0 +1,345 @@
"use server";
import { auth } from "@/lib/auth";
import { prisma } from "@/lib/prisma";
import { revalidatePath } from "next/cache";
import type { ActionResult } from "@/types/api.types";
import {
telegramAccountSchema,
telegramChannelSchema,
linkChannelSchema,
submitAuthCodeSchema,
} from "@/schemas/telegram";
const REVALIDATE_PATH = "/telegram";
async function requireAdmin(): Promise<
{ success: true; userId: string } | { success: false; error: string }
> {
const session = await auth();
if (!session?.user?.id) return { success: false, error: "Unauthorized" };
if (session.user.role !== "ADMIN")
return { success: false, error: "Admin access required" };
return { success: true, userId: session.user.id };
}
// ── Account actions ──
export async function createAccount(
input: unknown
): Promise<ActionResult<{ id: string }>> {
const admin = await requireAdmin();
if (!admin.success) return admin;
const parsed = telegramAccountSchema.safeParse(input);
if (!parsed.success) return { success: false, error: "Validation failed" };
try {
const account = await prisma.telegramAccount.create({
data: {
phone: parsed.data.phone.replace(/[\s\-]/g, ""),
displayName: parsed.data.displayName || null,
},
});
revalidatePath(REVALIDATE_PATH);
return { success: true, data: { id: account.id } };
} catch (err: unknown) {
if (
err instanceof Error &&
err.message.includes("Unique constraint failed")
) {
return { success: false, error: "Phone number already registered" };
}
return { success: false, error: "Failed to create account" };
}
}
export async function updateAccount(
id: string,
input: unknown
): Promise<ActionResult> {
const admin = await requireAdmin();
if (!admin.success) return admin;
const parsed = telegramAccountSchema.safeParse(input);
if (!parsed.success) return { success: false, error: "Validation failed" };
const existing = await prisma.telegramAccount.findUnique({ where: { id } });
if (!existing) return { success: false, error: "Account not found" };
try {
await prisma.telegramAccount.update({
where: { id },
data: {
phone: parsed.data.phone.replace(/[\s\-]/g, ""),
displayName: parsed.data.displayName || null,
},
});
revalidatePath(REVALIDATE_PATH);
return { success: true, data: undefined };
} catch (err: unknown) {
if (
err instanceof Error &&
err.message.includes("Unique constraint failed")
) {
return { success: false, error: "Phone number already registered" };
}
return { success: false, error: "Failed to update account" };
}
}
export async function toggleAccountActive(id: string): Promise<ActionResult> {
const admin = await requireAdmin();
if (!admin.success) return admin;
const existing = await prisma.telegramAccount.findUnique({ where: { id } });
if (!existing) return { success: false, error: "Account not found" };
try {
await prisma.telegramAccount.update({
where: { id },
data: { isActive: !existing.isActive },
});
revalidatePath(REVALIDATE_PATH);
return { success: true, data: undefined };
} catch {
return { success: false, error: "Failed to toggle account" };
}
}
export async function deleteAccount(id: string): Promise<ActionResult> {
const admin = await requireAdmin();
if (!admin.success) return admin;
const existing = await prisma.telegramAccount.findUnique({ where: { id } });
if (!existing) return { success: false, error: "Account not found" };
try {
await prisma.telegramAccount.delete({ where: { id } });
revalidatePath(REVALIDATE_PATH);
return { success: true, data: undefined };
} catch {
return { success: false, error: "Failed to delete account" };
}
}
export async function submitAuthCode(
accountId: string,
input: unknown
): Promise<ActionResult> {
const admin = await requireAdmin();
if (!admin.success) return admin;
const parsed = submitAuthCodeSchema.safeParse(input);
if (!parsed.success) return { success: false, error: "Validation failed" };
const existing = await prisma.telegramAccount.findUnique({
where: { id: accountId },
});
if (!existing) return { success: false, error: "Account not found" };
if (
existing.authState !== "AWAITING_CODE" &&
existing.authState !== "AWAITING_PASSWORD"
) {
return { success: false, error: "Account is not waiting for a code" };
}
try {
await prisma.telegramAccount.update({
where: { id: accountId },
data: { authCode: parsed.data.code },
});
revalidatePath(REVALIDATE_PATH);
return { success: true, data: undefined };
} catch {
return { success: false, error: "Failed to submit code" };
}
}
// ── Channel actions ──
export async function createChannel(
input: unknown
): Promise<ActionResult<{ id: string }>> {
const admin = await requireAdmin();
if (!admin.success) return admin;
const parsed = telegramChannelSchema.safeParse(input);
if (!parsed.success) return { success: false, error: "Validation failed" };
try {
const channel = await prisma.telegramChannel.create({
data: {
telegramId: BigInt(parsed.data.telegramId),
title: parsed.data.title,
type: parsed.data.type,
},
});
revalidatePath(REVALIDATE_PATH);
return { success: true, data: { id: channel.id } };
} catch (err: unknown) {
if (
err instanceof Error &&
err.message.includes("Unique constraint failed")
) {
return { success: false, error: "Channel with this Telegram ID already exists" };
}
return { success: false, error: "Failed to create channel" };
}
}
export async function updateChannel(
id: string,
input: unknown
): Promise<ActionResult> {
const admin = await requireAdmin();
if (!admin.success) return admin;
const parsed = telegramChannelSchema.safeParse(input);
if (!parsed.success) return { success: false, error: "Validation failed" };
const existing = await prisma.telegramChannel.findUnique({ where: { id } });
if (!existing) return { success: false, error: "Channel not found" };
try {
await prisma.telegramChannel.update({
where: { id },
data: {
telegramId: BigInt(parsed.data.telegramId),
title: parsed.data.title,
type: parsed.data.type,
},
});
revalidatePath(REVALIDATE_PATH);
return { success: true, data: undefined };
} catch (err: unknown) {
if (
err instanceof Error &&
err.message.includes("Unique constraint failed")
) {
return { success: false, error: "Channel with this Telegram ID already exists" };
}
return { success: false, error: "Failed to update channel" };
}
}
export async function toggleChannelActive(id: string): Promise<ActionResult> {
const admin = await requireAdmin();
if (!admin.success) return admin;
const existing = await prisma.telegramChannel.findUnique({ where: { id } });
if (!existing) return { success: false, error: "Channel not found" };
try {
await prisma.telegramChannel.update({
where: { id },
data: { isActive: !existing.isActive },
});
revalidatePath(REVALIDATE_PATH);
return { success: true, data: undefined };
} catch {
return { success: false, error: "Failed to toggle channel" };
}
}
export async function deleteChannel(id: string): Promise<ActionResult> {
const admin = await requireAdmin();
if (!admin.success) return admin;
const existing = await prisma.telegramChannel.findUnique({ where: { id } });
if (!existing) return { success: false, error: "Channel not found" };
try {
await prisma.telegramChannel.delete({ where: { id } });
revalidatePath(REVALIDATE_PATH);
return { success: true, data: undefined };
} catch {
return { success: false, error: "Failed to delete channel" };
}
}
// ── Account-Channel link actions ──
export async function linkChannel(
input: unknown
): Promise<ActionResult<{ id: string }>> {
const admin = await requireAdmin();
if (!admin.success) return admin;
const parsed = linkChannelSchema.safeParse(input);
if (!parsed.success) return { success: false, error: "Validation failed" };
try {
const link = await prisma.accountChannelMap.create({
data: {
accountId: parsed.data.accountId,
channelId: parsed.data.channelId,
role: parsed.data.role,
},
});
revalidatePath(REVALIDATE_PATH);
return { success: true, data: { id: link.id } };
} catch (err: unknown) {
if (
err instanceof Error &&
err.message.includes("Unique constraint failed")
) {
return { success: false, error: "This channel is already linked to this account" };
}
return { success: false, error: "Failed to link channel" };
}
}
export async function unlinkChannel(id: string): Promise<ActionResult> {
const admin = await requireAdmin();
if (!admin.success) return admin;
const existing = await prisma.accountChannelMap.findUnique({
where: { id },
});
if (!existing) return { success: false, error: "Link not found" };
try {
await prisma.accountChannelMap.delete({ where: { id } });
revalidatePath(REVALIDATE_PATH);
return { success: true, data: undefined };
} catch {
return { success: false, error: "Failed to unlink channel" };
}
}
// ── Ingestion trigger ──
export async function triggerIngestion(
accountId?: string
): Promise<ActionResult> {
const admin = await requireAdmin();
if (!admin.success) return admin;
try {
const res = await fetch(
`${process.env.NEXT_PUBLIC_APP_URL || "http://localhost:3000"}/api/ingestion/trigger`,
{
method: "POST",
headers: {
"Content-Type": "application/json",
"X-API-Key": process.env.INGESTION_API_KEY || "",
},
body: JSON.stringify({ accountId }),
}
);
if (!res.ok) {
const data = await res.json().catch(() => ({}));
return {
success: false,
error: (data as { error?: string }).error || "Failed to trigger ingestion",
};
}
revalidatePath(REVALIDATE_PATH);
return { success: true, data: undefined };
} catch {
return { success: false, error: "Failed to trigger ingestion" };
}
}

View File

@@ -0,0 +1,17 @@
import { auth } from "@/lib/auth";
import { redirect } from "next/navigation";
import { listAccounts, listChannels } from "@/lib/telegram/admin-queries";
import { TelegramAdmin } from "./_components/telegram-admin";
export default async function TelegramPage() {
const session = await auth();
if (!session?.user?.id) redirect("/login");
if (session.user.role !== "ADMIN") redirect("/dashboard");
const [accounts, channels] = await Promise.all([
listAccounts(),
listChannels(),
]);
return <TelegramAdmin accounts={accounts} channels={channels} />;
}

View File

@@ -0,0 +1,13 @@
import { NextResponse } from "next/server";
import { authenticateApiRequest } from "@/lib/telegram/api-auth";
import { getIngestionStatus } from "@/lib/telegram/queries";
export const dynamic = "force-dynamic";
export async function GET(request: Request) {
const authResult = await authenticateApiRequest(request);
if ("error" in authResult) return authResult.error;
const accounts = await getIngestionStatus();
return NextResponse.json({ accounts });
}

View File

@@ -0,0 +1,77 @@
import { NextResponse } from "next/server";
import { authenticateApiRequest } from "@/lib/telegram/api-auth";
import { triggerIngestionSchema } from "@/schemas/telegram";
import { prisma } from "@/lib/prisma";
export const dynamic = "force-dynamic";
export async function POST(request: Request) {
const authResult = await authenticateApiRequest(request, true);
if ("error" in authResult) return authResult.error;
let body: unknown = {};
try {
body = await request.json();
} catch {
// Empty body is fine — triggers all accounts
}
const parsed = triggerIngestionSchema.safeParse(body);
if (!parsed.success) {
return NextResponse.json(
{ error: "Invalid parameters", details: parsed.error.flatten() },
{ status: 400 }
);
}
// Find accounts to trigger
const where: { isActive: boolean; authState: "AUTHENTICATED"; id?: string } = {
isActive: true,
authState: "AUTHENTICATED",
};
if (parsed.data.accountId) {
where.id = parsed.data.accountId;
}
const accounts = await prisma.telegramAccount.findMany({
where,
select: { id: true },
});
if (accounts.length === 0) {
return NextResponse.json(
{ triggered: false, message: "No eligible accounts found" },
{ status: 404 }
);
}
// Create ingestion runs marked as RUNNING — the worker will pick these up
// when it next polls, or we use pg_notify for immediate pickup
for (const account of accounts) {
// Only create if no run is already RUNNING for this account
const existing = await prisma.ingestionRun.findFirst({
where: { accountId: account.id, status: "RUNNING" },
});
if (!existing) {
await prisma.ingestionRun.create({
data: { accountId: account.id, status: "RUNNING" },
});
}
}
// Send pg_notify for immediate worker pickup
try {
await prisma.$queryRawUnsafe(
`SELECT pg_notify('ingestion_trigger', $1)`,
accounts.map((a) => a.id).join(",")
);
} catch {
// pg_notify is best-effort — worker will pick up on next cycle anyway
}
return NextResponse.json({
triggered: true,
accountIds: accounts.map((a) => a.id),
message: `Ingestion queued for ${accounts.length} account(s)`,
});
}

View File

@@ -0,0 +1,17 @@
import { NextResponse } from "next/server";
import { authenticateApiRequest } from "@/lib/telegram/api-auth";
import { listAccountChannelLinks } from "@/lib/telegram/admin-queries";
export const dynamic = "force-dynamic";
export async function GET(
request: Request,
{ params }: { params: Promise<{ accountId: string }> }
) {
const authResult = await authenticateApiRequest(request, true);
if ("error" in authResult) return authResult.error;
const { accountId } = await params;
const links = await listAccountChannelLinks(accountId);
return NextResponse.json(links);
}

View File

@@ -0,0 +1,17 @@
import { NextResponse } from "next/server";
import { authenticateApiRequest } from "@/lib/telegram/api-auth";
import { getUnlinkedChannels } from "@/lib/telegram/admin-queries";
export const dynamic = "force-dynamic";
export async function GET(
request: Request,
{ params }: { params: Promise<{ accountId: string }> }
) {
const authResult = await authenticateApiRequest(request, true);
if ("error" in authResult) return authResult.error;
const { accountId } = await params;
const channels = await getUnlinkedChannels(accountId);
return NextResponse.json(channels);
}

View File

@@ -0,0 +1,32 @@
import { NextResponse } from "next/server";
import { authenticateApiRequest } from "@/lib/telegram/api-auth";
import { listPackageFiles } from "@/lib/telegram/queries";
import { listFilesSchema } from "@/schemas/telegram";
export const dynamic = "force-dynamic";
export async function GET(
request: Request,
{ params }: { params: Promise<{ id: string }> }
) {
const authResult = await authenticateApiRequest(request);
if ("error" in authResult) return authResult.error;
const { id } = await params;
const { searchParams } = new URL(request.url);
const parsed = listFilesSchema.safeParse(Object.fromEntries(searchParams));
if (!parsed.success) {
return NextResponse.json(
{ error: "Invalid parameters", details: parsed.error.flatten() },
{ status: 400 }
);
}
const result = await listPackageFiles({
packageId: id,
...parsed.data,
});
return NextResponse.json(result);
}

View File

@@ -0,0 +1,42 @@
import { NextResponse } from "next/server";
import { prisma } from "@/lib/prisma";
import { authenticateApiRequest } from "@/lib/telegram/api-auth";
/**
* GET /api/zips/:id/preview
* Returns the preview thumbnail image as JPEG binary.
* Cached for 1 hour (immutable once set).
*/
export async function GET(
request: Request,
{ params }: { params: Promise<{ id: string }> }
) {
const authResult = await authenticateApiRequest(request);
if ("error" in authResult) return authResult.error;
const { id } = await params;
const pkg = await prisma.package.findUnique({
where: { id },
select: { previewData: true },
});
if (!pkg || !pkg.previewData) {
return new NextResponse(null, { status: 404 });
}
// previewData is stored as Bytes (Buffer) from Prisma
const buffer =
pkg.previewData instanceof Buffer
? pkg.previewData
: Buffer.from(pkg.previewData);
return new NextResponse(buffer, {
status: 200,
headers: {
"Content-Type": "image/jpeg",
"Content-Length": String(buffer.length),
"Cache-Control": "public, max-age=3600, immutable",
},
});
}

View File

@@ -0,0 +1,22 @@
import { NextResponse } from "next/server";
import { authenticateApiRequest } from "@/lib/telegram/api-auth";
import { getPackageById } from "@/lib/telegram/queries";
export const dynamic = "force-dynamic";
export async function GET(
request: Request,
{ params }: { params: Promise<{ id: string }> }
) {
const authResult = await authenticateApiRequest(request);
if ("error" in authResult) return authResult.error;
const { id } = await params;
const pkg = await getPackageById(id);
if (!pkg) {
return NextResponse.json({ error: "Package not found" }, { status: 404 });
}
return NextResponse.json(pkg);
}

24
src/app/api/zips/route.ts Normal file
View File

@@ -0,0 +1,24 @@
import { NextResponse } from "next/server";
import { authenticateApiRequest } from "@/lib/telegram/api-auth";
import { listPackages } from "@/lib/telegram/queries";
import { listPackagesSchema } from "@/schemas/telegram";
export const dynamic = "force-dynamic";
export async function GET(request: Request) {
const authResult = await authenticateApiRequest(request);
if ("error" in authResult) return authResult.error;
const { searchParams } = new URL(request.url);
const parsed = listPackagesSchema.safeParse(Object.fromEntries(searchParams));
if (!parsed.success) {
return NextResponse.json(
{ error: "Invalid parameters", details: parsed.error.flatten() },
{ status: 400 }
);
}
const result = await listPackages(parsed.data);
return NextResponse.json(result);
}

View File

@@ -0,0 +1,25 @@
import { NextResponse } from "next/server";
import { authenticateApiRequest } from "@/lib/telegram/api-auth";
import { searchPackages } from "@/lib/telegram/queries";
import { searchSchema } from "@/schemas/telegram";
export const dynamic = "force-dynamic";
export async function GET(request: Request) {
const authResult = await authenticateApiRequest(request);
if ("error" in authResult) return authResult.error;
const { searchParams } = new URL(request.url);
const parsed = searchSchema.safeParse(Object.fromEntries(searchParams));
if (!parsed.success) {
return NextResponse.json(
{ error: "Invalid parameters", details: parsed.error.flatten() },
{ status: 400 }
);
}
const { q, ...rest } = parsed.data;
const result = await searchPackages({ query: q, ...rest });
return NextResponse.json(result);
}

View File

@@ -8,6 +8,8 @@ import {
Droplets, Droplets,
Paintbrush, Paintbrush,
Gem, Gem,
FileBox,
Send,
ClipboardList, ClipboardList,
Building2, Building2,
MapPin, MapPin,
@@ -18,7 +20,7 @@ import { cn } from "@/lib/utils";
import { APP_NAME } from "@/lib/constants"; import { APP_NAME } from "@/lib/constants";
import { SheetHeader, SheetTitle } from "@/components/ui/sheet"; import { SheetHeader, SheetTitle } from "@/components/ui/sheet";
const icons = { LayoutDashboard, Cylinder, Droplets, Paintbrush, Gem, ClipboardList, Building2, MapPin, Settings }; const icons = { LayoutDashboard, Cylinder, Droplets, Paintbrush, Gem, FileBox, Send, ClipboardList, Building2, MapPin, Settings };
const navItems = [ const navItems = [
{ label: "Dashboard", href: "/dashboard", icon: "LayoutDashboard" as const }, { label: "Dashboard", href: "/dashboard", icon: "LayoutDashboard" as const },
@@ -26,6 +28,8 @@ const navItems = [
{ label: "Resins", href: "/resins", icon: "Droplets" as const }, { label: "Resins", href: "/resins", icon: "Droplets" as const },
{ label: "Paints", href: "/paints", icon: "Paintbrush" as const }, { label: "Paints", href: "/paints", icon: "Paintbrush" as const },
{ label: "Supplies", href: "/supplies", icon: "Gem" as const }, { label: "Supplies", href: "/supplies", icon: "Gem" as const },
{ label: "STL Files", href: "/stls", icon: "FileBox" as const },
{ label: "Telegram", href: "/telegram", icon: "Send" as const },
{ label: "Usage", href: "/usage", icon: "ClipboardList" as const }, { label: "Usage", href: "/usage", icon: "ClipboardList" as const },
{ label: "Vendors", href: "/vendors", icon: "Building2" as const }, { label: "Vendors", href: "/vendors", icon: "Building2" as const },
{ label: "Locations", href: "/locations", icon: "MapPin" as const }, { label: "Locations", href: "/locations", icon: "MapPin" as const },

View File

@@ -9,6 +9,8 @@ import {
Droplets, Droplets,
Paintbrush, Paintbrush,
Gem, Gem,
FileBox,
Send,
ClipboardList, ClipboardList,
Building2, Building2,
MapPin, MapPin,
@@ -28,6 +30,8 @@ const icons = {
Droplets, Droplets,
Paintbrush, Paintbrush,
Gem, Gem,
FileBox,
Send,
ClipboardList, ClipboardList,
Building2, Building2,
MapPin, MapPin,
@@ -40,6 +44,8 @@ const navItems = [
{ label: "Resins", href: "/resins", icon: "Droplets" as const }, { label: "Resins", href: "/resins", icon: "Droplets" as const },
{ label: "Paints", href: "/paints", icon: "Paintbrush" as const }, { label: "Paints", href: "/paints", icon: "Paintbrush" as const },
{ label: "Supplies", href: "/supplies", icon: "Gem" as const }, { label: "Supplies", href: "/supplies", icon: "Gem" as const },
{ label: "STL Files", href: "/stls", icon: "FileBox" as const },
{ label: "Telegram", href: "/telegram", icon: "Send" as const },
{ label: "Usage", href: "/usage", icon: "ClipboardList" as const }, { label: "Usage", href: "/usage", icon: "ClipboardList" as const },
{ label: "Vendors", href: "/vendors", icon: "Building2" as const }, { label: "Vendors", href: "/vendors", icon: "Building2" as const },
{ label: "Locations", href: "/locations", icon: "MapPin" as const }, { label: "Locations", href: "/locations", icon: "MapPin" as const },

View File

@@ -6,6 +6,8 @@ export const NAV_ITEMS = [
{ label: "Resins", href: "/resins", icon: "Droplets" }, { label: "Resins", href: "/resins", icon: "Droplets" },
{ label: "Paints", href: "/paints", icon: "Paintbrush" }, { label: "Paints", href: "/paints", icon: "Paintbrush" },
{ label: "Supplies", href: "/supplies", icon: "Gem" }, { label: "Supplies", href: "/supplies", icon: "Gem" },
{ label: "STL Files", href: "/stls", icon: "FileBox" },
{ label: "Telegram", href: "/telegram", icon: "Send" },
{ label: "Usage", href: "/usage", icon: "ClipboardList" }, { label: "Usage", href: "/usage", icon: "ClipboardList" },
{ label: "Vendors", href: "/vendors", icon: "Building2" }, { label: "Vendors", href: "/vendors", icon: "Building2" },
{ label: "Locations", href: "/locations", icon: "MapPin" }, { label: "Locations", href: "/locations", icon: "MapPin" },

View File

@@ -0,0 +1,105 @@
import { prisma } from "@/lib/prisma";
// ── Account queries ──
export async function listAccounts() {
const accounts = await prisma.telegramAccount.findMany({
orderBy: { createdAt: "desc" },
include: {
_count: { select: { channelMaps: true, ingestionRuns: true } },
},
});
return accounts.map((a) => ({
id: a.id,
phone: a.phone,
displayName: a.displayName,
isActive: a.isActive,
authState: a.authState,
authCode: a.authCode,
lastSeenAt: a.lastSeenAt?.toISOString() ?? null,
createdAt: a.createdAt.toISOString(),
channelCount: a._count.channelMaps,
runCount: a._count.ingestionRuns,
}));
}
export type AccountRow = Awaited<ReturnType<typeof listAccounts>>[number];
// ── Channel queries ──
export async function listChannels() {
const channels = await prisma.telegramChannel.findMany({
orderBy: { createdAt: "desc" },
include: {
_count: { select: { accountMaps: true, packages: true } },
},
});
return channels.map((c) => ({
id: c.id,
telegramId: c.telegramId.toString(),
title: c.title,
type: c.type,
isActive: c.isActive,
createdAt: c.createdAt.toISOString(),
accountCount: c._count.accountMaps,
packageCount: c._count.packages,
}));
}
export type ChannelRow = Awaited<ReturnType<typeof listChannels>>[number];
// ── Account-Channel link queries ──
export async function listAccountChannelLinks(accountId: string) {
const links = await prisma.accountChannelMap.findMany({
where: { accountId },
include: {
channel: { select: { id: true, title: true, type: true, telegramId: true } },
},
orderBy: { createdAt: "desc" },
});
return links.map((l) => ({
id: l.id,
accountId: l.accountId,
channelId: l.channelId,
role: l.role,
lastProcessedMessageId: l.lastProcessedMessageId?.toString() ?? null,
channel: {
id: l.channel.id,
title: l.channel.title,
type: l.channel.type,
telegramId: l.channel.telegramId.toString(),
},
}));
}
export type AccountChannelLinkRow = Awaited<
ReturnType<typeof listAccountChannelLinks>
>[number];
export async function getUnlinkedChannels(accountId: string) {
const linked = await prisma.accountChannelMap.findMany({
where: { accountId },
select: { channelId: true },
});
const linkedIds = linked.map((l) => l.channelId);
const unlinked = await prisma.telegramChannel.findMany({
where: {
id: { notIn: linkedIds },
isActive: true,
},
orderBy: { title: "asc" },
select: { id: true, title: true, type: true, telegramId: true },
});
return unlinked.map((c) => ({
id: c.id,
title: c.title,
type: c.type,
telegramId: c.telegramId.toString(),
}));
}

View File

@@ -0,0 +1,45 @@
import { auth } from "@/lib/auth";
import { NextResponse } from "next/server";
/**
* Authenticate an API request. Checks:
* 1. X-API-Key header against TELEGRAM_API_KEY env var
* 2. NextAuth session
*
* Returns null if authenticated, or a NextResponse error if not.
*/
export async function authenticateApiRequest(
request: Request,
requireAdmin = false
): Promise<{ error: NextResponse } | { userId: string; role: string }> {
// Check API key first
const apiKey = request.headers.get("X-API-Key");
const envKey = process.env.TELEGRAM_API_KEY;
if (apiKey && envKey && apiKey === envKey) {
// API key auth — treated as admin
return { userId: "api-key", role: "ADMIN" };
}
// Fall back to session auth
const session = await auth();
if (!session?.user?.id) {
return {
error: NextResponse.json(
{ error: "Unauthorized" },
{ status: 401 }
),
};
}
if (requireAdmin && session.user.role !== "ADMIN") {
return {
error: NextResponse.json(
{ error: "Forbidden: admin role required" },
{ status: 403 }
),
};
}
return { userId: session.user.id, role: session.user.role };
}

314
src/lib/telegram/queries.ts Normal file
View File

@@ -0,0 +1,314 @@
import { prisma } from "@/lib/prisma";
import type {
PackageListItem,
PackageDetail,
PackageFileItem,
IngestionAccountStatus,
} from "./types";
export async function listPackages(options: {
page: number;
limit: number;
channelId?: string;
creator?: string;
sortBy: "indexedAt" | "fileName" | "fileSize";
order: "asc" | "desc";
}) {
const where: Record<string, unknown> = {};
if (options.channelId) where.sourceChannelId = options.channelId;
if (options.creator) where.creator = options.creator;
const [items, total] = await Promise.all([
prisma.package.findMany({
where,
orderBy: { [options.sortBy]: options.order },
skip: (options.page - 1) * options.limit,
take: options.limit,
select: {
id: true,
fileName: true,
fileSize: true,
contentHash: true,
archiveType: true,
fileCount: true,
isMultipart: true,
indexedAt: true,
creator: true,
previewMsgId: true, // cheap null check — avoids loading blob
sourceChannel: { select: { id: true, title: true } },
},
}),
prisma.package.count({ where }),
]);
const mapped: PackageListItem[] = items.map((pkg) => ({
id: pkg.id,
fileName: pkg.fileName,
fileSize: pkg.fileSize.toString(),
contentHash: pkg.contentHash,
archiveType: pkg.archiveType,
fileCount: pkg.fileCount,
isMultipart: pkg.isMultipart,
hasPreview: pkg.previewMsgId !== null,
creator: pkg.creator,
indexedAt: pkg.indexedAt.toISOString(),
sourceChannel: pkg.sourceChannel,
}));
return {
items: mapped,
pagination: {
page: options.page,
limit: options.limit,
total,
totalPages: Math.ceil(total / options.limit),
},
};
}
export async function getPackageById(
id: string
): Promise<PackageDetail | null> {
const pkg = await prisma.package.findUnique({
where: { id },
include: {
sourceChannel: { select: { id: true, title: true } },
ingestionRun: { select: { id: true, startedAt: true } },
},
});
if (!pkg) return null;
let destChannel: { id: string; title: string } | null = null;
if (pkg.destChannelId) {
const ch = await prisma.telegramChannel.findUnique({
where: { id: pkg.destChannelId },
select: { id: true, title: true },
});
destChannel = ch;
}
return {
id: pkg.id,
fileName: pkg.fileName,
fileSize: pkg.fileSize.toString(),
contentHash: pkg.contentHash,
archiveType: pkg.archiveType,
fileCount: pkg.fileCount,
isMultipart: pkg.isMultipart,
hasPreview: pkg.previewMsgId !== null,
creator: pkg.creator,
partCount: pkg.partCount,
indexedAt: pkg.indexedAt.toISOString(),
sourceChannel: pkg.sourceChannel,
destChannel,
destMessageId: pkg.destMessageId?.toString() ?? null,
sourceMessageId: pkg.sourceMessageId.toString(),
ingestionRun: pkg.ingestionRun
? {
id: pkg.ingestionRun.id,
startedAt: pkg.ingestionRun.startedAt.toISOString(),
}
: null,
};
}
export async function listPackageFiles(options: {
packageId: string;
page: number;
limit: number;
extension?: string;
}) {
const where: { packageId: string; extension?: string } = {
packageId: options.packageId,
};
if (options.extension) {
where.extension = options.extension;
}
const [items, total] = await Promise.all([
prisma.packageFile.findMany({
where,
orderBy: { path: "asc" },
skip: (options.page - 1) * options.limit,
take: options.limit,
}),
prisma.packageFile.count({ where }),
]);
const mapped: PackageFileItem[] = items.map((f) => ({
id: f.id,
path: f.path,
fileName: f.fileName,
extension: f.extension,
compressedSize: f.compressedSize.toString(),
uncompressedSize: f.uncompressedSize.toString(),
crc32: f.crc32,
}));
return {
items: mapped,
pagination: {
page: options.page,
limit: options.limit,
total,
totalPages: Math.ceil(total / options.limit),
},
};
}
export async function searchPackages(options: {
query: string;
page: number;
limit: number;
searchIn: "packages" | "files" | "both";
}) {
const q = options.query;
if (options.searchIn === "files" || options.searchIn === "both") {
// Search in package files, return parent packages
const fileMatches = await prisma.packageFile.findMany({
where: {
OR: [
{ fileName: { contains: q, mode: "insensitive" } },
{ path: { contains: q, mode: "insensitive" } },
],
},
select: { packageId: true },
distinct: ["packageId"],
});
const packageIds = fileMatches.map((f) => f.packageId);
const packageNameIds =
options.searchIn === "both"
? (
await prisma.package.findMany({
where: { fileName: { contains: q, mode: "insensitive" } },
select: { id: true },
})
).map((p) => p.id)
: [];
const allIds = [...new Set([...packageIds, ...packageNameIds])];
const [items, total] = await Promise.all([
prisma.package.findMany({
where: { id: { in: allIds } },
orderBy: { indexedAt: "desc" },
skip: (options.page - 1) * options.limit,
take: options.limit,
select: {
id: true,
fileName: true,
fileSize: true,
contentHash: true,
archiveType: true,
fileCount: true,
isMultipart: true,
indexedAt: true,
creator: true,
previewMsgId: true,
sourceChannel: { select: { id: true, title: true } },
},
}),
Promise.resolve(allIds.length),
]);
const mapped: PackageListItem[] = items.map((pkg) => ({
id: pkg.id,
fileName: pkg.fileName,
fileSize: pkg.fileSize.toString(),
contentHash: pkg.contentHash,
archiveType: pkg.archiveType,
fileCount: pkg.fileCount,
isMultipart: pkg.isMultipart,
hasPreview: pkg.previewMsgId !== null,
creator: pkg.creator,
indexedAt: pkg.indexedAt.toISOString(),
sourceChannel: pkg.sourceChannel,
}));
return {
items: mapped,
pagination: {
page: options.page,
limit: options.limit,
total,
totalPages: Math.ceil(total / options.limit),
},
};
}
// Search packages only
return listPackages({
page: options.page,
limit: options.limit,
sortBy: "indexedAt",
order: "desc",
});
}
export async function getIngestionStatus(): Promise<IngestionAccountStatus[]> {
const accounts = await prisma.telegramAccount.findMany({
orderBy: { createdAt: "asc" },
});
const statuses: IngestionAccountStatus[] = [];
for (const account of accounts) {
const lastRun = await prisma.ingestionRun.findFirst({
where: { accountId: account.id, status: { not: "RUNNING" } },
orderBy: { startedAt: "desc" },
});
const currentRun = await prisma.ingestionRun.findFirst({
where: { accountId: account.id, status: "RUNNING" },
orderBy: { startedAt: "desc" },
});
statuses.push({
id: account.id,
displayName: account.displayName,
phone: account.phone,
isActive: account.isActive,
authState: account.authState,
lastSeenAt: account.lastSeenAt?.toISOString() ?? null,
lastRun: lastRun
? {
id: lastRun.id,
status: lastRun.status,
startedAt: lastRun.startedAt.toISOString(),
finishedAt: lastRun.finishedAt?.toISOString() ?? null,
messagesScanned: lastRun.messagesScanned,
zipsFound: lastRun.zipsFound,
zipsDuplicate: lastRun.zipsDuplicate,
zipsIngested: lastRun.zipsIngested,
}
: null,
currentRun: currentRun
? {
id: currentRun.id,
startedAt: currentRun.startedAt.toISOString(),
messagesScanned: currentRun.messagesScanned,
zipsFound: currentRun.zipsFound,
zipsDuplicate: currentRun.zipsDuplicate,
zipsIngested: currentRun.zipsIngested,
// Live activity tracking
currentActivity: currentRun.currentActivity,
currentStep: currentRun.currentStep,
currentChannel: currentRun.currentChannel,
currentFile: currentRun.currentFile,
currentFileNum: currentRun.currentFileNum,
totalFiles: currentRun.totalFiles,
downloadedBytes: currentRun.downloadedBytes?.toString() ?? null,
totalBytes: currentRun.totalBytes?.toString() ?? null,
downloadPercent: currentRun.downloadPercent,
lastActivityAt: currentRun.lastActivityAt?.toISOString() ?? null,
}
: null,
});
}
return statuses;
}

88
src/lib/telegram/types.ts Normal file
View File

@@ -0,0 +1,88 @@
export interface PackageListItem {
id: string;
fileName: string;
fileSize: string; // BigInt serialized as string
contentHash: string;
archiveType: "ZIP" | "RAR";
fileCount: number;
isMultipart: boolean;
hasPreview: boolean;
creator: string | null;
indexedAt: string;
sourceChannel: {
id: string;
title: string;
};
}
export interface PackageDetail extends PackageListItem {
partCount: number;
destChannel: {
id: string;
title: string;
} | null;
destMessageId: string | null;
sourceMessageId: string;
ingestionRun: {
id: string;
startedAt: string;
} | null;
}
export interface PackageFileItem {
id: string;
path: string;
fileName: string;
extension: string | null;
compressedSize: string;
uncompressedSize: string;
crc32: string | null;
}
export interface PaginatedResponse<T> {
items: T[];
pagination: {
page: number;
limit: number;
total: number;
totalPages: number;
};
}
export interface IngestionAccountStatus {
id: string;
displayName: string | null;
phone: string;
isActive: boolean;
authState: string;
lastSeenAt: string | null;
lastRun: {
id: string;
status: string;
startedAt: string;
finishedAt: string | null;
messagesScanned: number;
zipsFound: number;
zipsDuplicate: number;
zipsIngested: number;
} | null;
currentRun: {
id: string;
startedAt: string;
messagesScanned: number;
zipsFound: number;
zipsDuplicate: number;
zipsIngested: number;
// Live activity tracking
currentActivity: string | null;
currentStep: string | null;
currentChannel: string | null;
currentFile: string | null;
currentFileNum: number | null;
totalFiles: number | null;
downloadedBytes: string | null; // BigInt serialized as string
totalBytes: string | null; // BigInt serialized as string
downloadPercent: number | null;
lastActivityAt: string | null;
} | null;
}

71
src/schemas/telegram.ts Normal file
View File

@@ -0,0 +1,71 @@
import { z } from "zod/v4";
export const paginationSchema = z.object({
page: z.coerce.number().int().min(1).default(1),
limit: z.coerce.number().int().min(1).max(100).default(25),
});
export const listPackagesSchema = paginationSchema.extend({
channelId: z.string().optional(),
creator: z.string().optional(),
sortBy: z.enum(["indexedAt", "fileName", "fileSize"]).default("indexedAt"),
order: z.enum(["asc", "desc"]).default("desc"),
});
export const listFilesSchema = paginationSchema.extend({
limit: z.coerce.number().int().min(1).max(500).default(50),
extension: z.string().optional(),
});
export const searchSchema = paginationSchema.extend({
q: z.string().min(1),
searchIn: z.enum(["packages", "files", "both"]).default("both"),
});
export const triggerIngestionSchema = z.object({
accountId: z.string().optional(),
});
// ── Account CRUD ──
export const telegramAccountSchema = z.object({
phone: z
.string()
.min(1, "Phone number is required")
.regex(/^\+?\d[\d\s\-]{6,20}$/, "Invalid phone format (e.g. +31612345678)"),
displayName: z.string().max(64).optional().or(z.literal("")),
});
export type TelegramAccountInput = z.infer<typeof telegramAccountSchema>;
export const submitAuthCodeSchema = z.object({
code: z.string().min(3, "Auth code is required").max(10),
});
export type SubmitAuthCodeInput = z.infer<typeof submitAuthCodeSchema>;
export const submitPasswordSchema = z.object({
password: z.string().min(1, "Password is required"),
});
export type SubmitPasswordInput = z.infer<typeof submitPasswordSchema>;
// ── Channel CRUD ──
export const telegramChannelSchema = z.object({
telegramId: z.coerce.number().int().min(1, "Telegram ID is required"),
title: z.string().min(1, "Title is required").max(256),
type: z.enum(["SOURCE", "DESTINATION"]),
});
export type TelegramChannelInput = z.infer<typeof telegramChannelSchema>;
// ── Account-Channel linking ──
export const linkChannelSchema = z.object({
accountId: z.string().min(1),
channelId: z.string().min(1),
role: z.enum(["READER", "WRITER"]).default("READER"),
});
export type LinkChannelInput = z.infer<typeof linkChannelSchema>;

View File

@@ -31,5 +31,5 @@
".next/dev/types/**/*.ts", ".next/dev/types/**/*.ts",
"**/*.mts" "**/*.mts"
], ],
"exclude": ["node_modules", "prisma/seed.ts", "scripts/**"] "exclude": ["node_modules", "prisma/seed.ts", "scripts/**", "worker/**"]
} }

48
worker/Dockerfile Normal file
View File

@@ -0,0 +1,48 @@
# ── Stage 1: Install production deps ─────────────────────────
FROM node:20-bookworm-slim AS deps
RUN sed -i 's/^Components: main$/Components: main non-free/' /etc/apt/sources.list.d/debian.sources && \
apt-get update && apt-get install -y \
libssl-dev zlib1g-dev unrar \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /app
COPY worker/package.json worker/package-lock.json* ./
COPY prisma/ ./prisma/
# Install ALL deps (including devDependencies for tsc) and generate Prisma
RUN npm ci && npx prisma generate
# ── Stage 2: Build TypeScript ─────────────────────────────────
FROM deps AS builder
COPY worker/tsconfig.json ./
COPY worker/src/ ./src/
RUN npx tsc
# ── Stage 3: Production runner ────────────────────────────────
FROM node:20-bookworm-slim AS runner
RUN sed -i 's/^Components: main$/Components: main non-free/' /etc/apt/sources.list.d/debian.sources && \
apt-get update && apt-get install -y \
libssl3 zlib1g unrar \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /app
# Copy only production node_modules (prune devDeps)
COPY --from=builder /app/node_modules ./node_modules
COPY --from=builder /app/prisma ./prisma
COPY --from=builder /app/dist ./dist
# Re-generate Prisma client for production (after pruning isn't needed since we copy all)
RUN npx prisma generate
RUN addgroup --system worker && adduser --system --ingroup worker worker
RUN mkdir -p /data/tdlib /tmp/zips && chown -R worker:worker /data/tdlib /tmp/zips
USER worker
VOLUME ["/data/tdlib", "/tmp/zips"]
CMD ["node", "dist/index.js"]

2140
worker/package-lock.json generated Normal file

File diff suppressed because it is too large Load Diff

28
worker/package.json Normal file
View File

@@ -0,0 +1,28 @@
{
"name": "dragonsstash-worker",
"version": "0.1.0",
"private": true,
"type": "module",
"scripts": {
"build": "tsc",
"start": "node dist/index.js",
"dev": "tsx watch src/index.ts"
},
"dependencies": {
"@prisma/adapter-pg": "^7.4.0",
"@prisma/client": "^7.4.0",
"pg": "^8.18.0",
"pino": "^9.6.0",
"prebuilt-tdlib": "^0.1008050.0",
"tdl": "^8.0.0",
"yauzl": "^3.2.0"
},
"devDependencies": {
"@types/node": "^20",
"@types/pg": "^8.16.0",
"@types/yauzl": "^2.10.3",
"prisma": "^7.4.0",
"tsx": "^4.21.0",
"typescript": "^5"
}
}

View File

@@ -0,0 +1,21 @@
/**
* Extract a creator name from common archive file naming patterns.
*
* Priority in the worker: topic name > filename extraction.
* This is the fallback when no forum topic name is available.
*
* Patterns handled (split on ` - `):
* "Mammoth Factory - 2026-01.zip" → "Mammoth Factory"
* "Artist Name - Pack Title.part01.rar" → "Artist Name"
* "some_random_file.zip" → null
*/
export function extractCreatorFromFileName(fileName: string): string | null {
// Strip archive extensions (.zip, .rar, .part01.rar, .z01, etc.)
const bare = fileName.replace(/(\.(part\d+\.rar|z\d{2}|zip|rar))+$/i, "");
const idx = bare.indexOf(" - ");
if (idx <= 0) return null;
const creator = bare.slice(0, idx).trim();
return creator.length > 0 ? creator : null;
}

View File

@@ -0,0 +1,96 @@
export type ArchiveFormat = "ZIP" | "RAR";
export interface MultipartInfo {
baseName: string;
partNumber: number;
format: ArchiveFormat;
pattern: "ZIP_NUMBERED" | "ZIP_LEGACY" | "RAR_PART" | "RAR_LEGACY" | "SINGLE";
}
const patterns: {
regex: RegExp;
format: ArchiveFormat;
pattern: MultipartInfo["pattern"];
getBaseName: (match: RegExpMatchArray) => string;
getPartNumber: (match: RegExpMatchArray) => number;
}[] = [
// pack.zip.001, pack.zip.002
{
regex: /^(.+\.zip)\.(\d{3,})$/i,
format: "ZIP",
pattern: "ZIP_NUMBERED",
getBaseName: (m) => m[1],
getPartNumber: (m) => parseInt(m[2], 10),
},
// pack.z01, pack.z02 (legacy split — final part is pack.zip)
{
regex: /^(.+)\.z(\d{2,})$/i,
format: "ZIP",
pattern: "ZIP_LEGACY",
getBaseName: (m) => m[1],
getPartNumber: (m) => parseInt(m[2], 10),
},
// pack.part1.rar, pack.part2.rar
{
regex: /^(.+)\.part(\d+)\.rar$/i,
format: "RAR",
pattern: "RAR_PART",
getBaseName: (m) => m[1],
getPartNumber: (m) => parseInt(m[2], 10),
},
// pack.r00, pack.r01 (legacy split — final part is pack.rar)
{
regex: /^(.+)\.r(\d{2,})$/i,
format: "RAR",
pattern: "RAR_LEGACY",
getBaseName: (m) => m[1],
getPartNumber: (m) => parseInt(m[2], 10),
},
];
/**
* Detect if a filename is an archive and extract multipart info.
*/
export function detectArchive(fileName: string): MultipartInfo | null {
// Check multipart patterns first
for (const p of patterns) {
const match = fileName.match(p.regex);
if (match) {
return {
baseName: p.getBaseName(match),
partNumber: p.getPartNumber(match),
format: p.format,
pattern: p.pattern,
};
}
}
// Single .zip file — could be a standalone or the final part of a ZIP_LEGACY set
if (/\.zip$/i.test(fileName)) {
return {
baseName: fileName.replace(/\.zip$/i, ""),
partNumber: -1, // -1 signals "could be single or final legacy part"
format: "ZIP",
pattern: "SINGLE",
};
}
// Single .rar file — could be standalone or final part of RAR_LEGACY set
if (/\.rar$/i.test(fileName)) {
return {
baseName: fileName.replace(/\.rar$/i, ""),
partNumber: -1,
format: "RAR",
pattern: "SINGLE",
};
}
return null;
}
/**
* Check if a filename looks like any archive attachment we should process.
*/
export function isArchiveAttachment(fileName: string): boolean {
return detectArchive(fileName) !== null;
}

View File

@@ -0,0 +1,25 @@
import { createReadStream } from "fs";
import { createHash } from "crypto";
import { pipeline } from "stream/promises";
import { PassThrough } from "stream";
/**
* Compute SHA-256 hash of one or more files by streaming them in order.
* Memory usage: O(1) — reads in 64KB chunks regardless of total size.
* For multipart archives, pass all parts sorted by part number.
*/
export async function hashParts(filePaths: string[]): Promise<string> {
const hash = createHash("sha256");
for (const filePath of filePaths) {
await pipeline(
createReadStream(filePath),
new PassThrough({
transform(chunk, _encoding, callback) {
hash.update(chunk);
callback();
},
})
);
}
return hash.digest("hex");
}

View File

@@ -0,0 +1,100 @@
import { detectArchive, type ArchiveFormat, type MultipartInfo } from "./detect.js";
import { config } from "../util/config.js";
import { childLogger } from "../util/logger.js";
const log = childLogger("multipart");
export interface TelegramMessage {
id: bigint;
fileName: string;
fileId: string;
fileSize: bigint;
date: Date;
}
export interface ArchiveSet {
type: ArchiveFormat;
baseName: string;
parts: TelegramMessage[];
isMultipart: boolean;
}
/**
* Group messages into archive sets (single files + multipart groups).
* Messages should be pre-filtered to only include archive attachments.
*/
export function groupArchiveSets(messages: TelegramMessage[]): ArchiveSet[] {
// Detect and annotate each message
const annotated: { msg: TelegramMessage; info: MultipartInfo }[] = [];
for (const msg of messages) {
const info = detectArchive(msg.fileName);
if (info) {
annotated.push({ msg, info });
}
}
// Group by baseName + format
const groups = new Map<string, { msg: TelegramMessage; info: MultipartInfo }[]>();
for (const item of annotated) {
const key = `${item.info.format}:${item.info.baseName.toLowerCase()}`;
const group = groups.get(key) ?? [];
group.push(item);
groups.set(key, group);
}
const results: ArchiveSet[] = [];
for (const [, group] of groups) {
const format = group[0].info.format;
const baseName = group[0].info.baseName;
// Separate explicit multipart entries from potential singles
const multipartEntries = group.filter((g) => g.info.pattern !== "SINGLE");
const singleEntries = group.filter((g) => g.info.pattern === "SINGLE");
if (multipartEntries.length > 0) {
// This is a multipart set
// Check if any single entry is the "final part" of a legacy split
const allEntries = [...multipartEntries, ...singleEntries];
// Check time span — skip if parts span too long
const dates = allEntries.map((e) => e.msg.date.getTime());
const span = Math.max(...dates) - Math.min(...dates);
const maxSpanMs = config.multipartTimeoutHours * 60 * 60 * 1000;
if (span > maxSpanMs) {
log.warn(
{ baseName, format, span: span / 3600000 },
"Multipart set spans too long, skipping"
);
continue;
}
// Sort by part number (singles get a very high number so they come last — they're the final part)
allEntries.sort((a, b) => {
const aNum = a.info.partNumber === -1 ? 999999 : a.info.partNumber;
const bNum = b.info.partNumber === -1 ? 999999 : b.info.partNumber;
return aNum - bNum;
});
results.push({
type: format,
baseName,
parts: allEntries.map((e) => e.msg),
isMultipart: true,
});
} else {
// All entries are singles — each is its own archive set
for (const entry of singleEntries) {
results.push({
type: format,
baseName: entry.info.baseName,
parts: [entry.msg],
isMultipart: false,
});
}
}
}
return results;
}

View File

@@ -0,0 +1,90 @@
import { execFile } from "child_process";
import { promisify } from "util";
import path from "path";
import { childLogger } from "../util/logger.js";
import type { FileEntry } from "./zip-reader.js";
const execFileAsync = promisify(execFile);
const log = childLogger("rar-reader");
/**
* Parse output of `unrar l -v <file>` to extract file metadata.
* unrar automatically discovers sibling parts when they're co-located.
*/
export async function readRarContents(
firstPartPath: string
): Promise<FileEntry[]> {
try {
const { stdout } = await execFileAsync("unrar", ["l", "-v", firstPartPath], {
timeout: 30000,
maxBuffer: 10 * 1024 * 1024, // 10MB for very large archives
});
return parseUnrarOutput(stdout);
} catch (err) {
log.warn({ err, file: firstPartPath }, "Failed to read RAR contents");
return []; // Fallback: return empty on error
}
}
/**
* Parse the tabular output of `unrar l -v`.
*
* Example output format:
* Archive: test.rar
* Details: RAR 5
*
* Attributes Size Packed Ratio Date Time CRC-32 Name
* ----------- --------- --------- ----- -------- ----- -------- ----
* ...A.... 12345 10234 83% 2024-01-15 10:30 DEADBEEF folder/file.stl
* ----------- --------- --------- ----- -------- ----- -------- ----
*/
function parseUnrarOutput(output: string): FileEntry[] {
const entries: FileEntry[] = [];
const lines = output.split("\n");
let inFileList = false;
let separatorCount = 0;
for (const line of lines) {
const trimmed = line.trim();
// Detect separator lines (------- pattern)
if (/^-{5,}/.test(trimmed)) {
separatorCount++;
if (separatorCount === 1) {
inFileList = true;
} else if (separatorCount >= 2) {
inFileList = false;
}
continue;
}
if (!inFileList) continue;
// Parse file entry line
// Format: Attributes Size Packed Ratio Date Time CRC Name
const match = trimmed.match(
/^\S+\s+(\d+)\s+(\d+)\s+\d+%\s+\S+\s+\S+\s+([0-9A-Fa-f]+)\s+(.+)$/
);
if (match) {
const [, uncompressedStr, compressedStr, crc32, filePath] = match;
// Skip directory entries (typically end with / or have size 0 with dir attributes)
if (filePath.endsWith("/") || filePath.endsWith("\\")) continue;
const ext = path.extname(filePath).toLowerCase();
entries.push({
path: filePath,
fileName: path.basename(filePath),
extension: ext ? ext.slice(1) : null,
compressedSize: BigInt(compressedStr),
uncompressedSize: BigInt(uncompressedStr),
crc32: crc32.toLowerCase(),
});
}
}
return entries;
}

View File

@@ -0,0 +1,48 @@
import { createReadStream, createWriteStream } from "fs";
import { stat } from "fs/promises";
import path from "path";
import { pipeline } from "stream/promises";
import { childLogger } from "../util/logger.js";
const log = childLogger("split");
/** 2GB in bytes — Telegram's file size limit */
const MAX_PART_SIZE = 2n * 1024n * 1024n * 1024n;
/**
* Split a file into ≤2GB parts using byte-level splitting.
* Returns paths to the split parts. If the file is already ≤2GB, returns the original path.
*/
export async function byteLevelSplit(filePath: string): Promise<string[]> {
const stats = await stat(filePath);
const fileSize = BigInt(stats.size);
if (fileSize <= MAX_PART_SIZE) {
return [filePath];
}
const dir = path.dirname(filePath);
const baseName = path.basename(filePath);
const partSize = Number(MAX_PART_SIZE);
const totalParts = Math.ceil(Number(fileSize) / partSize);
const parts: string[] = [];
log.info({ filePath, fileSize: Number(fileSize), totalParts }, "Splitting file");
for (let i = 0; i < totalParts; i++) {
const partNum = String(i + 1).padStart(3, "0");
const partPath = path.join(dir, `${baseName}.${partNum}`);
const start = i * partSize;
const end = Math.min(start + partSize - 1, Number(fileSize) - 1);
await pipeline(
createReadStream(filePath, { start, end }),
createWriteStream(partPath)
);
parts.push(partPath);
}
log.info({ filePath, parts: parts.length }, "File split complete");
return parts;
}

View File

@@ -0,0 +1,61 @@
import yauzl from "yauzl";
import path from "path";
import { childLogger } from "../util/logger.js";
const log = childLogger("zip-reader");
export interface FileEntry {
path: string;
fileName: string;
extension: string | null;
compressedSize: bigint;
uncompressedSize: bigint;
crc32: string | null;
}
/**
* Read the central directory of a ZIP file without extracting any contents.
* For multipart ZIPs, pass the paths sorted by part order.
* We attempt to read from the last part first (central directory is at the end).
*/
export async function readZipCentralDirectory(
filePaths: string[]
): Promise<FileEntry[]> {
// The central directory lives at the end of the last file
const targetFile = filePaths[filePaths.length - 1];
return new Promise((resolve, reject) => {
yauzl.open(targetFile, { lazyEntries: true, autoClose: true }, (err, zipFile) => {
if (err) {
log.warn({ err, file: targetFile }, "Failed to open ZIP for reading");
resolve([]); // Fallback: return empty on error
return;
}
const entries: FileEntry[] = [];
zipFile.readEntry();
zipFile.on("entry", (entry: yauzl.Entry) => {
// Skip directories
if (!entry.fileName.endsWith("/")) {
const ext = path.extname(entry.fileName).toLowerCase();
entries.push({
path: entry.fileName,
fileName: path.basename(entry.fileName),
extension: ext ? ext.slice(1) : null, // Remove leading dot
compressedSize: BigInt(entry.compressedSize),
uncompressedSize: BigInt(entry.uncompressedSize),
crc32: entry.crc32 !== 0 ? entry.crc32.toString(16).padStart(8, "0") : null,
});
}
zipFile.readEntry();
});
zipFile.on("end", () => resolve(entries));
zipFile.on("error", (error) => {
log.warn({ error, file: targetFile }, "Error reading ZIP entries");
resolve(entries); // Return whatever we got
});
});
});
}

14
worker/src/db/client.ts Normal file
View File

@@ -0,0 +1,14 @@
import { PrismaClient } from "@prisma/client";
import { PrismaPg } from "@prisma/adapter-pg";
import pg from "pg";
import { config } from "../util/config.js";
const pool = new pg.Pool({
connectionString: config.databaseUrl,
max: 5,
});
const adapter = new PrismaPg(pool);
export const db = new PrismaClient({ adapter });
export { pool };

56
worker/src/db/locks.ts Normal file
View File

@@ -0,0 +1,56 @@
import { pool } from "./client.js";
import { childLogger } from "../util/logger.js";
const log = childLogger("locks");
/**
* Derive a stable 32-bit integer lock ID from an account ID string.
* PostgreSQL advisory locks use bigint, but we use 32-bit for safety.
*/
function hashToLockId(accountId: string): number {
let hash = 0;
for (let i = 0; i < accountId.length; i++) {
const char = accountId.charCodeAt(i);
hash = (hash << 5) - hash + char;
hash |= 0; // Convert to 32-bit integer
}
return Math.abs(hash);
}
/**
* Try to acquire a PostgreSQL advisory lock for an account.
* Returns true if acquired, false if already held by another session.
*/
export async function tryAcquireLock(accountId: string): Promise<boolean> {
const lockId = hashToLockId(accountId);
const client = await pool.connect();
try {
const result = await client.query<{ pg_try_advisory_lock: boolean }>(
"SELECT pg_try_advisory_lock($1)",
[lockId]
);
const acquired = result.rows[0]?.pg_try_advisory_lock ?? false;
if (acquired) {
log.debug({ accountId, lockId }, "Advisory lock acquired");
} else {
log.debug({ accountId, lockId }, "Advisory lock already held");
}
return acquired;
} finally {
client.release();
}
}
/**
* Release the advisory lock for an account.
*/
export async function releaseLock(accountId: string): Promise<void> {
const lockId = hashToLockId(accountId);
const client = await pool.connect();
try {
await client.query("SELECT pg_advisory_unlock($1)", [lockId]);
log.debug({ accountId, lockId }, "Advisory lock released");
} finally {
client.release();
}
}

270
worker/src/db/queries.ts Normal file
View File

@@ -0,0 +1,270 @@
import { db } from "./client.js";
import type { ArchiveType } from "@prisma/client";
export async function getActiveAccounts() {
return db.telegramAccount.findMany({
where: { isActive: true, authState: "AUTHENTICATED" },
});
}
export async function getSourceChannelMappings(accountId: string) {
return db.accountChannelMap.findMany({
where: {
accountId,
role: "READER",
channel: { type: "SOURCE", isActive: true },
},
include: { channel: true },
});
}
export async function getDestinationChannel(accountId: string) {
const mapping = await db.accountChannelMap.findFirst({
where: {
accountId,
role: "WRITER",
channel: { type: "DESTINATION", isActive: true },
},
include: { channel: true },
});
return mapping?.channel ?? null;
}
export async function packageExistsByHash(contentHash: string) {
const pkg = await db.package.findUnique({
where: { contentHash },
select: { id: true },
});
return pkg !== null;
}
export interface CreatePackageInput {
contentHash: string;
fileName: string;
fileSize: bigint;
archiveType: ArchiveType;
sourceChannelId: string;
sourceMessageId: bigint;
sourceTopicId?: bigint | null;
destChannelId?: string;
destMessageId?: bigint;
isMultipart: boolean;
partCount: number;
ingestionRunId: string;
creator?: string | null;
previewData?: Buffer | null;
previewMsgId?: bigint | null;
files: {
path: string;
fileName: string;
extension: string | null;
compressedSize: bigint;
uncompressedSize: bigint;
crc32: string | null;
}[];
}
export async function createPackageWithFiles(input: CreatePackageInput) {
return db.package.create({
data: {
contentHash: input.contentHash,
fileName: input.fileName,
fileSize: input.fileSize,
archiveType: input.archiveType,
sourceChannelId: input.sourceChannelId,
sourceMessageId: input.sourceMessageId,
sourceTopicId: input.sourceTopicId ?? undefined,
destChannelId: input.destChannelId,
destMessageId: input.destMessageId,
isMultipart: input.isMultipart,
partCount: input.partCount,
fileCount: input.files.length,
ingestionRunId: input.ingestionRunId,
creator: input.creator ?? undefined,
previewData: input.previewData ? new Uint8Array(input.previewData) : undefined,
previewMsgId: input.previewMsgId ?? undefined,
files: {
create: input.files,
},
},
});
}
export async function createIngestionRun(accountId: string) {
return db.ingestionRun.create({
data: {
accountId,
status: "RUNNING",
currentActivity: "Starting ingestion run",
currentStep: "initializing",
lastActivityAt: new Date(),
},
});
}
export interface ActivityUpdate {
currentActivity: string;
currentStep: string;
currentChannel?: string | null;
currentFile?: string | null;
currentFileNum?: number | null;
totalFiles?: number | null;
downloadedBytes?: bigint | null;
totalBytes?: bigint | null;
downloadPercent?: number | null;
messagesScanned?: number;
zipsFound?: number;
zipsDuplicate?: number;
zipsIngested?: number;
}
export async function updateRunActivity(
runId: string,
activity: ActivityUpdate
) {
return db.ingestionRun.update({
where: { id: runId },
data: {
currentActivity: activity.currentActivity,
currentStep: activity.currentStep,
currentChannel: activity.currentChannel ?? undefined,
currentFile: activity.currentFile ?? undefined,
currentFileNum: activity.currentFileNum ?? undefined,
totalFiles: activity.totalFiles ?? undefined,
downloadedBytes: activity.downloadedBytes ?? undefined,
totalBytes: activity.totalBytes ?? undefined,
downloadPercent: activity.downloadPercent ?? undefined,
lastActivityAt: new Date(),
...(activity.messagesScanned !== undefined && { messagesScanned: activity.messagesScanned }),
...(activity.zipsFound !== undefined && { zipsFound: activity.zipsFound }),
...(activity.zipsDuplicate !== undefined && { zipsDuplicate: activity.zipsDuplicate }),
...(activity.zipsIngested !== undefined && { zipsIngested: activity.zipsIngested }),
},
});
}
const CLEAR_ACTIVITY = {
currentActivity: null,
currentStep: null,
currentChannel: null,
currentFile: null,
currentFileNum: null,
totalFiles: null,
downloadedBytes: null,
totalBytes: null,
downloadPercent: null,
lastActivityAt: new Date(),
};
export async function completeIngestionRun(
runId: string,
counters: {
messagesScanned: number;
zipsFound: number;
zipsDuplicate: number;
zipsIngested: number;
}
) {
return db.ingestionRun.update({
where: { id: runId },
data: {
status: "COMPLETED",
finishedAt: new Date(),
...counters,
...CLEAR_ACTIVITY,
},
});
}
export async function failIngestionRun(runId: string, errorMessage: string) {
return db.ingestionRun.update({
where: { id: runId },
data: {
status: "FAILED",
finishedAt: new Date(),
errorMessage,
...CLEAR_ACTIVITY,
},
});
}
export async function updateLastProcessedMessage(
mappingId: string,
messageId: bigint
) {
return db.accountChannelMap.update({
where: { id: mappingId },
data: { lastProcessedMessageId: messageId },
});
}
export async function markStaleRunsAsFailed() {
return db.ingestionRun.updateMany({
where: { status: "RUNNING" },
data: {
status: "FAILED",
finishedAt: new Date(),
errorMessage: "Worker restarted — run was still marked as RUNNING",
},
});
}
export async function updateAccountAuthState(
accountId: string,
authState: "PENDING" | "AWAITING_CODE" | "AWAITING_PASSWORD" | "AUTHENTICATED" | "EXPIRED",
authCode?: string | null
) {
return db.telegramAccount.update({
where: { id: accountId },
data: { authState, authCode, lastSeenAt: authState === "AUTHENTICATED" ? new Date() : undefined },
});
}
export async function getAccountAuthCode(accountId: string) {
const account = await db.telegramAccount.findUnique({
where: { id: accountId },
select: { authCode: true, authState: true },
});
return account;
}
// ── Forum / Topic progress ──
export async function setChannelForum(channelId: string, isForum: boolean) {
return db.telegramChannel.update({
where: { id: channelId },
data: { isForum },
});
}
export async function getTopicProgress(mappingId: string) {
return db.topicProgress.findMany({
where: { accountChannelMapId: mappingId },
});
}
export async function upsertTopicProgress(
mappingId: string,
topicId: bigint,
topicName: string | null,
lastProcessedMessageId: bigint
) {
return db.topicProgress.upsert({
where: {
accountChannelMapId_topicId: {
accountChannelMapId: mappingId,
topicId,
},
},
create: {
accountChannelMapId: mappingId,
topicId,
topicName,
lastProcessedMessageId,
},
update: {
topicName,
lastProcessedMessageId,
},
});
}

50
worker/src/index.ts Normal file
View File

@@ -0,0 +1,50 @@
import { mkdir } from "fs/promises";
import { config } from "./util/config.js";
import { logger } from "./util/logger.js";
import { markStaleRunsAsFailed } from "./db/queries.js";
import { cleanupTempDir } from "./worker.js";
import { startScheduler, stopScheduler } from "./scheduler.js";
import { db, pool } from "./db/client.js";
const log = logger.child({ module: "main" });
async function main(): Promise<void> {
log.info("DragonsStash Telegram Worker starting");
log.info({ config: { ...config, databaseUrl: "***" } }, "Configuration loaded");
// Ensure temp directory exists
await mkdir(config.tempDir, { recursive: true });
await mkdir(config.tdlibStateDir, { recursive: true });
// Clean up stale state
await cleanupTempDir();
await markStaleRunsAsFailed();
// Start the scheduler
await startScheduler();
}
// Graceful shutdown
function shutdown(signal: string): void {
log.info({ signal }, "Shutdown signal received");
stopScheduler();
// Close DB connections
Promise.all([db.$disconnect(), pool.end()])
.then(() => {
log.info("Shutdown complete");
process.exit(0);
})
.catch((err) => {
log.error({ err }, "Error during shutdown");
process.exit(1);
});
}
process.on("SIGTERM", () => shutdown("SIGTERM"));
process.on("SIGINT", () => shutdown("SIGINT"));
main().catch((err) => {
log.fatal({ err }, "Worker failed to start");
process.exit(1);
});

View File

@@ -0,0 +1,86 @@
import { childLogger } from "../util/logger.js";
const log = childLogger("preview-match");
export interface TelegramPhoto {
id: bigint;
date: Date;
/** Caption text on the photo message (if any). */
caption: string;
/** The smallest photo size available — used as thumbnail. */
fileId: string;
fileSize: number;
}
export interface ArchiveRef {
baseName: string;
firstMessageId: bigint;
firstMessageDate: Date;
}
/**
* Try to match a photo message to an archive by:
* 1. Caption contains the archive baseName (without extension)
* 2. Photo was posted within ±10 messages (time-window: ±6 hours)
*
* Returns the best match (closest in time), or null.
*/
export function matchPreviewToArchive(
photos: TelegramPhoto[],
archives: ArchiveRef[]
): Map<string, TelegramPhoto> {
const results = new Map<string, TelegramPhoto>();
const TIME_WINDOW_MS = 6 * 60 * 60 * 1000; // 6 hours
for (const archive of archives) {
// Normalize the archive base name for matching
const normalizedBase = normalizeForMatch(archive.baseName);
if (!normalizedBase) continue;
let bestMatch: TelegramPhoto | null = null;
let bestTimeDiff = Infinity;
for (const photo of photos) {
const timeDiff = Math.abs(
photo.date.getTime() - archive.firstMessageDate.getTime()
);
// Must be within time window
if (timeDiff > TIME_WINDOW_MS) continue;
// Check if the photo caption contains the archive base name
const normalizedCaption = normalizeForMatch(photo.caption);
if (!normalizedCaption) continue;
const matches =
normalizedCaption.includes(normalizedBase) ||
normalizedBase.includes(normalizedCaption);
if (matches && timeDiff < bestTimeDiff) {
bestMatch = photo;
bestTimeDiff = timeDiff;
}
}
if (bestMatch) {
log.debug(
{ baseName: archive.baseName, photoId: bestMatch.id.toString() },
"Matched preview photo to archive"
);
results.set(archive.baseName, bestMatch);
}
}
return results;
}
/**
* Strip extension, punctuation, and normalize for fuzzy matching.
*/
function normalizeForMatch(input: string): string {
return input
.toLowerCase()
.replace(/\.[a-z0-9]{1,5}$/i, "") // strip extension
.replace(/[_\-.\s]+/g, " ") // normalize separators
.trim();
}

92
worker/src/scheduler.ts Normal file
View File

@@ -0,0 +1,92 @@
import { config } from "./util/config.js";
import { childLogger } from "./util/logger.js";
import { getActiveAccounts } from "./db/queries.js";
import { runWorkerForAccount } from "./worker.js";
const log = childLogger("scheduler");
let running = false;
let timer: ReturnType<typeof setTimeout> | null = null;
/**
* Run one ingestion cycle: process all active, authenticated accounts sequentially.
*/
async function runCycle(): Promise<void> {
if (running) {
log.warn("Previous cycle still running, skipping");
return;
}
running = true;
log.info("Starting ingestion cycle");
try {
const accounts = await getActiveAccounts();
if (accounts.length === 0) {
log.info("No active authenticated accounts, nothing to do");
return;
}
log.info({ accountCount: accounts.length }, "Processing accounts");
for (const account of accounts) {
await runWorkerForAccount(account);
}
log.info("Ingestion cycle complete");
} catch (err) {
log.error({ err }, "Ingestion cycle failed");
} finally {
running = false;
}
}
/**
* Schedule the next cycle with jitter.
*/
function scheduleNext(): void {
const intervalMs = config.workerIntervalMinutes * 60 * 1000;
const jitterMs = Math.random() * config.jitterMinutes * 60 * 1000;
const delay = intervalMs + jitterMs;
log.info(
{ nextRunInMinutes: Math.round(delay / 60000) },
"Next cycle scheduled"
);
timer = setTimeout(async () => {
await runCycle();
scheduleNext();
}, delay);
}
/**
* Start the scheduler. Runs an immediate first cycle, then schedules subsequent ones.
*/
export async function startScheduler(): Promise<void> {
log.info(
{
intervalMinutes: config.workerIntervalMinutes,
jitterMinutes: config.jitterMinutes,
},
"Scheduler starting"
);
// Run immediately on start
await runCycle();
// Then schedule recurring cycles
scheduleNext();
}
/**
* Stop the scheduler gracefully.
*/
export function stopScheduler(): void {
if (timer) {
clearTimeout(timer);
timer = null;
}
log.info("Scheduler stopped");
}

120
worker/src/tdlib/client.ts Normal file
View File

@@ -0,0 +1,120 @@
import tdl, { createClient, type Client } from "tdl";
import { getTdjson } from "prebuilt-tdlib";
import path from "path";
import { config } from "../util/config.js";
import { childLogger } from "../util/logger.js";
import {
updateAccountAuthState,
getAccountAuthCode,
} from "../db/queries.js";
const log = childLogger("tdlib-client");
// Configure tdl to use the prebuilt tdjson shared library
tdl.configure({ tdjson: getTdjson() });
interface AccountConfig {
id: string;
phone: string;
}
/**
* Create and authenticate a TDLib client for a Telegram account.
* Authentication flow communicates with the admin UI via the database:
* - Worker sets authState to AWAITING_CODE when TDLib asks for phone code
* - Admin enters the code via UI, which writes it to authCode field
* - Worker polls DB for the code and feeds it to TDLib
*/
export async function createTdlibClient(
account: AccountConfig
): Promise<Client> {
const dbPath = path.join(config.tdlibStateDir, account.id);
const client = createClient({
apiId: config.telegramApiId,
apiHash: config.telegramApiHash,
databaseDirectory: dbPath,
filesDirectory: path.join(dbPath, "files"),
});
client.on("error", (err) => {
log.error({ err, accountId: account.id }, "TDLib client error");
});
try {
await client.login(() => ({
getPhoneNumber: async () => {
log.info({ accountId: account.id }, "TDLib requesting phone number");
return account.phone;
},
getAuthCode: async () => {
log.info({ accountId: account.id }, "TDLib requesting auth code");
await updateAccountAuthState(account.id, "AWAITING_CODE");
// Poll database for the code entered via admin UI
const code = await pollForAuthCode(account.id);
if (!code) {
throw new Error("Auth code not provided within timeout");
}
// Clear the code after reading
await updateAccountAuthState(account.id, "AUTHENTICATED", null);
return code;
},
getPassword: async () => {
log.info({ accountId: account.id }, "TDLib requesting 2FA password");
await updateAccountAuthState(account.id, "AWAITING_PASSWORD");
// Poll database for the password entered via admin UI
const code = await pollForAuthCode(account.id);
if (!code) {
throw new Error("2FA password not provided within timeout");
}
await updateAccountAuthState(account.id, "AUTHENTICATED", null);
return code;
},
}));
await updateAccountAuthState(account.id, "AUTHENTICATED");
log.info({ accountId: account.id }, "TDLib client authenticated");
return client;
} catch (err) {
log.error({ err, accountId: account.id }, "TDLib authentication failed");
await updateAccountAuthState(account.id, "EXPIRED");
throw err;
}
}
/**
* Poll the database every 5 seconds for an auth code, up to 5 minutes.
*/
async function pollForAuthCode(
accountId: string,
timeoutMs = 300_000
): Promise<string | null> {
const start = Date.now();
while (Date.now() - start < timeoutMs) {
const result = await getAccountAuthCode(accountId);
if (result?.authCode) {
return result.authCode;
}
await sleep(5000);
}
return null;
}
function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
/**
* Close a TDLib client gracefully.
*/
export async function closeTdlibClient(client: Client): Promise<void> {
try {
await client.close();
} catch (err) {
log.warn({ err }, "Error closing TDLib client");
}
}

View File

@@ -0,0 +1,389 @@
import type { Client } from "tdl";
import { readFile, rename, stat } from "fs/promises";
import { config } from "../util/config.js";
import { childLogger } from "../util/logger.js";
import { isArchiveAttachment } from "../archive/detect.js";
import type { TelegramMessage } from "../archive/multipart.js";
import type { TelegramPhoto } from "../preview/match.js";
const log = childLogger("download");
interface TdPhotoSize {
type: string;
photo: {
id: number;
size: number;
expected_size: number;
local?: {
path?: string;
is_downloading_active?: boolean;
is_downloading_completed?: boolean;
downloaded_size?: number;
};
};
width: number;
height: number;
}
interface TdMessage {
id: number;
date: number;
content: {
_: string;
document?: {
file_name?: string;
document?: {
id: number;
size: number;
local?: {
path?: string;
is_downloading_completed?: boolean;
};
};
};
photo?: {
sizes?: TdPhotoSize[];
};
caption?: {
text?: string;
};
};
}
interface TdFile {
id: number;
size: number;
expected_size: number;
local: {
path: string;
is_downloading_active: boolean;
is_downloading_completed: boolean;
downloaded_size: number;
download_offset: number;
};
}
export interface ChannelScanResult {
archives: TelegramMessage[];
photos: TelegramPhoto[];
}
/**
* Fetch messages from a channel since a given message ID.
* Collects both archive attachments AND photo messages (for preview matching).
* Returns messages in chronological order (oldest first).
*/
export async function getChannelMessages(
client: Client,
chatId: bigint,
fromMessageId?: bigint | null,
limit = 100
): Promise<ChannelScanResult> {
const archives: TelegramMessage[] = [];
const photos: TelegramPhoto[] = [];
let currentFromId = fromMessageId ? Number(fromMessageId) : 0;
// eslint-disable-next-line no-constant-condition
while (true) {
const result = (await client.invoke({
_: "getChatHistory",
chat_id: Number(chatId),
from_message_id: currentFromId,
offset: 0,
limit: Math.min(limit, 100),
only_local: false,
})) as { messages: TdMessage[] };
if (!result.messages || result.messages.length === 0) break;
for (const msg of result.messages) {
// Check for archive documents
const doc = msg.content?.document;
if (doc?.file_name && doc.document && isArchiveAttachment(doc.file_name)) {
archives.push({
id: BigInt(msg.id),
fileName: doc.file_name,
fileId: String(doc.document.id),
fileSize: BigInt(doc.document.size),
date: new Date(msg.date * 1000),
});
continue;
}
// Check for photo messages (potential previews)
const photo = msg.content?.photo;
const caption = msg.content?.caption?.text ?? "";
if (photo?.sizes && photo.sizes.length > 0) {
// Pick the smallest size for thumbnail (type "s" or "m")
// TDLib photo sizes are ordered from smallest to largest
const smallest = photo.sizes[0];
photos.push({
id: BigInt(msg.id),
date: new Date(msg.date * 1000),
caption,
fileId: String(smallest.photo.id),
fileSize: smallest.photo.size || smallest.photo.expected_size,
});
}
}
currentFromId = result.messages[result.messages.length - 1].id;
if (result.messages.length < 100) break;
// Rate limit delay
await sleep(config.apiDelayMs);
}
// Return in chronological order (oldest first)
return {
archives: archives.reverse(),
photos: photos.reverse(),
};
}
/**
* Download a photo thumbnail from Telegram and return its raw bytes.
* Uses synchronous download (photos are small, typically < 100KB).
* Returns null if download fails (non-critical).
*/
export async function downloadPhotoThumbnail(
client: Client,
fileId: string
): Promise<Buffer | null> {
const numericId = parseInt(fileId, 10);
try {
const result = (await client.invoke({
_: "downloadFile",
file_id: numericId,
priority: 1, // Low priority — thumbnails are nice-to-have
offset: 0,
limit: 0,
synchronous: true, // Small file — wait for it
})) as TdFile;
if (result?.local?.is_downloading_completed && result.local.path) {
const data = await readFile(result.local.path);
log.debug(
{ fileId, bytes: data.length },
"Downloaded photo thumbnail"
);
return data;
}
} catch (err) {
log.warn({ fileId, err }, "Failed to download photo thumbnail");
}
return null;
}
export interface DownloadProgress {
fileId: string;
fileName: string;
downloadedBytes: number;
totalBytes: number;
percent: number;
isComplete: boolean;
}
export type ProgressCallback = (progress: DownloadProgress) => void;
/**
* Download a file from Telegram to a local path with progress tracking
* and integrity verification.
*
* Progress flow:
* 1. Starts async download via TDLib
* 2. Listens for `updateFile` events to track download progress
* 3. Logs progress at every 10% increment
* 4. Once complete, verifies the local file size matches the expected size
* 5. Moves the file from TDLib's cache to the destination path
*
* Verification:
* - Compares actual file size on disk to the expected size from Telegram
* - Throws on mismatch (partial/corrupt download)
* - Throws on timeout (configurable, scales with file size)
* - Throws if download stops without completing (network error, etc.)
*/
export async function downloadFile(
client: Client,
fileId: string,
destPath: string,
expectedSize: bigint,
fileName: string,
onProgress?: ProgressCallback
): Promise<void> {
const numericId = parseInt(fileId, 10);
const totalBytes = Number(expectedSize);
log.info(
{ fileId, fileName, destPath, totalBytes },
"Starting file download"
);
// Report initial progress
onProgress?.({
fileId,
fileName,
downloadedBytes: 0,
totalBytes,
percent: 0,
isComplete: false,
});
return new Promise<void>((resolve, reject) => {
let lastLoggedPercent = 0;
let settled = false;
// Timeout: 10 minutes per GB, minimum 5 minutes
const timeoutMs = Math.max(
5 * 60_000,
(totalBytes / (1024 * 1024 * 1024)) * 10 * 60_000
);
const timer = setTimeout(() => {
if (!settled) {
settled = true;
cleanup();
reject(
new Error(
`Download timed out after ${Math.round(timeoutMs / 60_000)}min for ${fileName}`
)
);
}
}, timeoutMs);
// Listen for file update events to track progress
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const handleUpdate = (update: any) => {
if (update?._ !== "updateFile") return;
const file = update.file as TdFile | undefined;
if (!file || file.id !== numericId) return;
const downloaded = file.local.downloaded_size;
const percent =
totalBytes > 0 ? Math.round((downloaded / totalBytes) * 100) : 0;
// Log at every 10% increment
if (percent >= lastLoggedPercent + 10) {
lastLoggedPercent = percent - (percent % 10);
log.info(
{ fileId, fileName, downloaded, totalBytes, percent: `${percent}%` },
"Download progress"
);
}
// Report to callback
onProgress?.({
fileId,
fileName,
downloadedBytes: downloaded,
totalBytes,
percent,
isComplete: file.local.is_downloading_completed,
});
// Download finished
if (file.local.is_downloading_completed) {
if (!settled) {
settled = true;
cleanup();
verifyAndMove(file.local.path, destPath, totalBytes, fileName, fileId)
.then(resolve)
.catch(reject);
}
}
// Download stopped without completing (network error, cancelled, etc.)
if (
!file.local.is_downloading_active &&
!file.local.is_downloading_completed
) {
if (!settled) {
settled = true;
cleanup();
reject(
new Error(
`Download stopped unexpectedly for ${fileName} ` +
`(${downloaded}/${totalBytes} bytes, ${percent}%)`
)
);
}
}
};
const cleanup = () => {
clearTimeout(timer);
client.off("update", handleUpdate);
};
// Subscribe to updates BEFORE starting download
client.on("update", handleUpdate);
// Start async download (non-blocking — progress via updateFile events)
client
.invoke({
_: "downloadFile",
file_id: numericId,
priority: 32,
offset: 0,
limit: 0,
synchronous: false,
})
.then((result: unknown) => {
// If the file was already cached locally, invoke returns immediately
const file = result as TdFile | undefined;
if (file?.local?.is_downloading_completed && !settled) {
settled = true;
cleanup();
verifyAndMove(file.local.path, destPath, totalBytes, fileName, fileId)
.then(resolve)
.catch(reject);
}
})
.catch((err: unknown) => {
if (!settled) {
settled = true;
cleanup();
reject(err);
}
});
});
}
/**
* Verify the downloaded file's size matches the expected size,
* then move it to the destination path.
*/
async function verifyAndMove(
localPath: string,
destPath: string,
expectedBytes: number,
fileName: string,
fileId: string
): Promise<void> {
const stats = await stat(localPath);
const actualBytes = stats.size;
if (expectedBytes > 0 && actualBytes !== expectedBytes) {
log.error(
{ fileId, fileName, expectedBytes, actualBytes },
"Download size mismatch — file is incomplete or corrupted"
);
throw new Error(
`Download verification failed for ${fileName}: ` +
`expected ${expectedBytes} bytes, got ${actualBytes} bytes`
);
}
log.info(
{ fileId, fileName, bytes: actualBytes, destPath },
"File verified and complete"
);
// Move from TDLib's cache to our temp directory
await rename(localPath, destPath);
}
function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}

222
worker/src/tdlib/topics.ts Normal file
View File

@@ -0,0 +1,222 @@
import type { Client } from "tdl";
import { config } from "../util/config.js";
import { childLogger } from "../util/logger.js";
import { isArchiveAttachment } from "../archive/detect.js";
import type { TelegramMessage } from "../archive/multipart.js";
import type { TelegramPhoto } from "../preview/match.js";
import type { ChannelScanResult } from "./download.js";
const log = childLogger("topics");
export interface ForumTopic {
topicId: bigint;
name: string;
}
/**
* Check if a chat is a forum supergroup (topics enabled).
*/
export async function isChatForum(
client: Client,
chatId: bigint
): Promise<boolean> {
try {
const chat = (await client.invoke({
_: "getChat",
chat_id: Number(chatId),
})) as {
type?: {
_: string;
supergroup_id?: number;
is_forum?: boolean;
};
};
if (chat.type?._ === "chatTypeSupergroup" && chat.type.is_forum) {
return true;
}
// Also check via getSupergroup for older TDLib versions
if (chat.type?._ === "chatTypeSupergroup" && chat.type.supergroup_id) {
const sg = (await client.invoke({
_: "getSupergroup",
supergroup_id: chat.type.supergroup_id,
})) as { is_forum?: boolean };
return sg.is_forum === true;
}
return false;
} catch (err) {
log.warn({ err, chatId: chatId.toString() }, "Failed to check if chat is forum");
return false;
}
}
/**
* Get all forum topics in a supergroup.
*/
export async function getForumTopicList(
client: Client,
chatId: bigint
): Promise<ForumTopic[]> {
const topics: ForumTopic[] = [];
let offsetDate = 0;
let offsetMessageId = 0;
let offsetMessageThreadId = 0;
// eslint-disable-next-line no-constant-condition
while (true) {
const result = (await client.invoke({
_: "getForumTopics",
chat_id: Number(chatId),
query: "",
offset_date: offsetDate,
offset_message_id: offsetMessageId,
offset_message_thread_id: offsetMessageThreadId,
limit: 100,
})) as {
topics?: {
info?: {
message_thread_id?: number;
name?: string;
is_general?: boolean;
};
}[];
next_offset_date?: number;
next_offset_message_id?: number;
next_offset_message_thread_id?: number;
};
if (!result.topics || result.topics.length === 0) break;
for (const t of result.topics) {
if (!t.info?.message_thread_id) continue;
// Skip the "General" topic — it's not creator-specific
if (t.info.is_general) continue;
topics.push({
topicId: BigInt(t.info.message_thread_id),
name: t.info.name ?? "Unnamed",
});
}
// Check if there are more pages
if (
!result.next_offset_date &&
!result.next_offset_message_id &&
!result.next_offset_message_thread_id
) {
break;
}
offsetDate = result.next_offset_date ?? 0;
offsetMessageId = result.next_offset_message_id ?? 0;
offsetMessageThreadId = result.next_offset_message_thread_id ?? 0;
await sleep(config.apiDelayMs);
}
log.info(
{ chatId: chatId.toString(), topicCount: topics.length },
"Enumerated forum topics"
);
return topics;
}
/**
* Fetch messages from a specific forum topic (thread).
* Uses getMessageThreadHistory to scan within a topic.
*/
export async function getTopicMessages(
client: Client,
chatId: bigint,
topicId: bigint,
fromMessageId?: bigint | null,
limit = 100
): Promise<ChannelScanResult> {
const archives: TelegramMessage[] = [];
const photos: TelegramPhoto[] = [];
let currentFromId = fromMessageId ? Number(fromMessageId) : 0;
// eslint-disable-next-line no-constant-condition
while (true) {
const result = (await client.invoke({
_: "getMessageThreadHistory",
chat_id: Number(chatId),
message_id: Number(topicId),
from_message_id: currentFromId,
offset: 0,
limit: Math.min(limit, 100),
})) as {
messages?: {
id: number;
date: number;
content: {
_: string;
document?: {
file_name?: string;
document?: {
id: number;
size: number;
};
};
photo?: {
sizes?: {
type: string;
photo: { id: number; size: number; expected_size: number };
width: number;
height: number;
}[];
};
caption?: { text?: string };
};
}[];
};
if (!result.messages || result.messages.length === 0) break;
for (const msg of result.messages) {
// Check for archive documents
const doc = msg.content?.document;
if (doc?.file_name && doc.document && isArchiveAttachment(doc.file_name)) {
archives.push({
id: BigInt(msg.id),
fileName: doc.file_name,
fileId: String(doc.document.id),
fileSize: BigInt(doc.document.size),
date: new Date(msg.date * 1000),
});
continue;
}
// Check for photo messages (potential previews)
const photo = msg.content?.photo;
const caption = msg.content?.caption?.text ?? "";
if (photo?.sizes && photo.sizes.length > 0) {
const smallest = photo.sizes[0];
photos.push({
id: BigInt(msg.id),
date: new Date(msg.date * 1000),
caption,
fileId: String(smallest.photo.id),
fileSize: smallest.photo.size || smallest.photo.expected_size,
});
}
}
currentFromId = result.messages[result.messages.length - 1].id;
if (result.messages.length < 100) break;
await sleep(config.apiDelayMs);
}
return {
archives: archives.reverse(),
photos: photos.reverse(),
};
}
function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}

View File

@@ -0,0 +1,76 @@
import type { Client } from "tdl";
import { config } from "../util/config.js";
import { childLogger } from "../util/logger.js";
const log = childLogger("upload");
export interface UploadResult {
messageId: bigint;
}
/**
* Upload one or more files to a destination Telegram channel.
* For multipart archives, each file is sent as a separate message.
* Returns the message ID of the first uploaded message.
*/
export async function uploadToChannel(
client: Client,
chatId: bigint,
filePaths: string[],
caption?: string
): Promise<UploadResult> {
let firstMessageId: bigint | null = null;
for (let i = 0; i < filePaths.length; i++) {
const filePath = filePaths[i];
const fileCaption =
i === 0 && caption ? caption : undefined;
log.debug(
{ chatId: Number(chatId), filePath, part: i + 1, total: filePaths.length },
"Uploading file to channel"
);
const result = (await client.invoke({
_: "sendMessage",
chat_id: Number(chatId),
input_message_content: {
_: "inputMessageDocument",
document: {
_: "inputFileLocal",
path: filePath,
},
caption: fileCaption
? {
_: "formattedText",
text: fileCaption,
}
: undefined,
},
})) as { id: number };
if (i === 0) {
firstMessageId = BigInt(result.id);
}
// Rate limit delay between uploads
if (i < filePaths.length - 1) {
await sleep(config.apiDelayMs);
}
}
if (firstMessageId === null) {
throw new Error("Upload failed: no messages sent");
}
log.info(
{ chatId: Number(chatId), messageId: Number(firstMessageId), files: filePaths.length },
"Upload complete"
);
return { messageId: firstMessageId };
}
function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}

18
worker/src/util/config.ts Normal file
View File

@@ -0,0 +1,18 @@
export const config = {
databaseUrl: process.env.DATABASE_URL ?? "",
workerIntervalMinutes: parseInt(process.env.WORKER_INTERVAL_MINUTES ?? "60", 10),
tempDir: process.env.WORKER_TEMP_DIR ?? "/tmp/zips",
tdlibStateDir: process.env.TDLIB_STATE_DIR ?? "/data/tdlib",
maxZipSizeMB: parseInt(process.env.WORKER_MAX_ZIP_SIZE_MB ?? "4096", 10),
logLevel: (process.env.LOG_LEVEL ?? "info") as "debug" | "info" | "warn" | "error",
telegramApiId: parseInt(process.env.TELEGRAM_API_ID ?? "0", 10),
telegramApiHash: process.env.TELEGRAM_API_HASH ?? "",
/** Maximum jitter added to scheduler interval (in minutes) */
jitterMinutes: 5,
/** Maximum time between multipart archive parts (in hours) */
multipartTimeoutHours: 24,
/** Delay between Telegram API calls (in ms) to avoid rate limits */
apiDelayMs: 1000,
/** Max retries for rate-limited requests */
maxRetries: 5,
} as const;

14
worker/src/util/logger.ts Normal file
View File

@@ -0,0 +1,14 @@
import pino from "pino";
import { config } from "./config.js";
export const logger = pino({
level: config.logLevel,
transport:
config.logLevel === "debug"
? { target: "pino/file", options: { destination: 1 } }
: undefined,
});
export function childLogger(name: string, extra?: Record<string, unknown>) {
return logger.child({ module: name, ...extra });
}

665
worker/src/worker.ts Normal file
View File

@@ -0,0 +1,665 @@
import path from "path";
import { unlink, readdir } from "fs/promises";
import { config } from "./util/config.js";
import { childLogger } from "./util/logger.js";
import { tryAcquireLock, releaseLock } from "./db/locks.js";
import {
getSourceChannelMappings,
getDestinationChannel,
packageExistsByHash,
createPackageWithFiles,
createIngestionRun,
completeIngestionRun,
failIngestionRun,
updateLastProcessedMessage,
updateRunActivity,
setChannelForum,
getTopicProgress,
upsertTopicProgress,
} from "./db/queries.js";
import type { ActivityUpdate } from "./db/queries.js";
import { createTdlibClient, closeTdlibClient } from "./tdlib/client.js";
import { getChannelMessages, downloadFile, downloadPhotoThumbnail } from "./tdlib/download.js";
import type { DownloadProgress, ChannelScanResult } from "./tdlib/download.js";
import { isChatForum, getForumTopicList, getTopicMessages } from "./tdlib/topics.js";
import { matchPreviewToArchive } from "./preview/match.js";
import { groupArchiveSets } from "./archive/multipart.js";
import type { ArchiveSet } from "./archive/multipart.js";
import { extractCreatorFromFileName } from "./archive/creator.js";
import { hashParts } from "./archive/hash.js";
import { readZipCentralDirectory } from "./archive/zip-reader.js";
import { readRarContents } from "./archive/rar-reader.js";
import { byteLevelSplit } from "./archive/split.js";
import { uploadToChannel } from "./upload/channel.js";
import type { TelegramAccount, TelegramChannel } from "@prisma/client";
import type { Client } from "tdl";
const log = childLogger("worker");
/**
* Throttle DB writes for download progress to avoid hammering the DB.
* Only writes if at least 2 seconds have passed since the last write.
*/
function createThrottledActivityUpdater(runId: string, minIntervalMs = 2000) {
let lastWriteTime = 0;
let pendingUpdate: ActivityUpdate | null = null;
let flushTimer: ReturnType<typeof setTimeout> | null = null;
const flush = async () => {
if (pendingUpdate) {
const update = pendingUpdate;
pendingUpdate = null;
lastWriteTime = Date.now();
await updateRunActivity(runId, update).catch(() => {});
}
};
return {
update: (activity: ActivityUpdate) => {
pendingUpdate = activity;
const elapsed = Date.now() - lastWriteTime;
if (elapsed >= minIntervalMs) {
if (flushTimer) clearTimeout(flushTimer);
flush();
} else if (!flushTimer) {
flushTimer = setTimeout(() => {
flushTimer = null;
flush();
}, minIntervalMs - elapsed);
}
},
flush,
};
}
/** Shared context passed to the archive processing pipeline. */
interface PipelineContext {
client: Client;
runId: string;
channelTitle: string;
channel: TelegramChannel;
destChannelTelegramId: bigint;
destChannelId: string;
throttled: ReturnType<typeof createThrottledActivityUpdater>;
counters: {
messagesScanned: number;
zipsFound: number;
zipsDuplicate: number;
zipsIngested: number;
};
/** Creator from forum topic name (null for non-forum). */
topicCreator: string | null;
/** Forum topic ID (null for non-forum). */
sourceTopicId: bigint | null;
accountLog: ReturnType<typeof childLogger>;
}
/**
* Run a full ingestion cycle for a single Telegram account.
* Every step writes live activity to the DB so the admin UI can display it.
*/
export async function runWorkerForAccount(
account: TelegramAccount
): Promise<void> {
const accountLog = childLogger("worker", { accountId: account.id, phone: account.phone });
// 1. Acquire advisory lock
const acquired = await tryAcquireLock(account.id);
if (!acquired) {
accountLog.info("Account already locked, skipping");
return;
}
let runId: string | undefined;
try {
// 2. Create ingestion run
const run = await createIngestionRun(account.id);
runId = run.id;
const activeRunId = runId;
accountLog.info({ runId }, "Ingestion run started");
const throttled = createThrottledActivityUpdater(activeRunId);
// 3. Initialize TDLib client
await updateRunActivity(activeRunId, {
currentActivity: "Connecting to Telegram",
currentStep: "connecting",
});
const client = await createTdlibClient({
id: account.id,
phone: account.phone,
});
const counters = {
messagesScanned: 0,
zipsFound: 0,
zipsDuplicate: 0,
zipsIngested: 0,
};
try {
// 4. Get assigned source channels and destination
const channelMappings = await getSourceChannelMappings(account.id);
const destChannel = await getDestinationChannel(account.id);
if (!destChannel) {
throw new Error("No active destination channel configured");
}
for (const mapping of channelMappings) {
const channel = mapping.channel;
// ── Check if channel is a forum ──
const forum = await isChatForum(client, channel.telegramId);
if (forum !== channel.isForum) {
await setChannelForum(channel.id, forum);
accountLog.info(
{ channelId: channel.id, title: channel.title, isForum: forum },
"Updated channel forum status"
);
}
const pipelineCtx: PipelineContext = {
client,
runId: activeRunId,
channelTitle: channel.title,
channel,
destChannelTelegramId: destChannel.telegramId,
destChannelId: destChannel.id,
throttled,
counters,
topicCreator: null,
sourceTopicId: null,
accountLog,
};
if (forum) {
// ── Forum channel: scan per-topic ──
await updateRunActivity(activeRunId, {
currentActivity: `Enumerating topics in "${channel.title}"`,
currentStep: "scanning",
currentChannel: channel.title,
currentFile: null,
currentFileNum: null,
totalFiles: null,
downloadedBytes: null,
totalBytes: null,
downloadPercent: null,
});
const topics = await getForumTopicList(client, channel.telegramId);
const topicProgressList = await getTopicProgress(mapping.id);
accountLog.info(
{ channelId: channel.id, title: channel.title, topicCount: topics.length },
"Scanning forum channel by topic"
);
for (const topic of topics) {
const progress = topicProgressList.find(
(tp) => tp.topicId === topic.topicId
);
await updateRunActivity(activeRunId, {
currentActivity: `Scanning topic "${topic.name}" in "${channel.title}"`,
currentStep: "scanning",
currentChannel: `${channel.title} ${topic.name}`,
currentFile: null,
currentFileNum: null,
totalFiles: null,
downloadedBytes: null,
totalBytes: null,
downloadPercent: null,
});
const scanResult = await getTopicMessages(
client,
channel.telegramId,
topic.topicId,
progress?.lastProcessedMessageId
);
if (scanResult.archives.length === 0) {
accountLog.debug(
{ channelId: channel.id, topic: topic.name },
"No new archives in topic"
);
continue;
}
accountLog.info(
{ topic: topic.name, archives: scanResult.archives.length, photos: scanResult.photos.length },
"Found messages in topic"
);
// Process archives with topic creator
pipelineCtx.topicCreator = topic.name;
pipelineCtx.sourceTopicId = topic.topicId;
pipelineCtx.channelTitle = `${channel.title} ${topic.name}`;
await processArchiveSets(pipelineCtx, scanResult, run.id);
// Update topic progress
const allMsgIds = [
...scanResult.archives.map((m) => m.id),
...scanResult.photos.map((p) => p.id),
];
if (allMsgIds.length > 0) {
const maxId = allMsgIds.reduce((a, b) => (a > b ? a : b));
await upsertTopicProgress(
mapping.id,
topic.topicId,
topic.name,
maxId
);
}
}
} else {
// ── Non-forum channel: flat scan (existing behavior) ──
await updateRunActivity(activeRunId, {
currentActivity: `Scanning "${channel.title}" for new archives`,
currentStep: "scanning",
currentChannel: channel.title,
currentFile: null,
currentFileNum: null,
totalFiles: null,
downloadedBytes: null,
totalBytes: null,
downloadPercent: null,
});
accountLog.info(
{ channelId: channel.id, title: channel.title },
"Processing source channel"
);
const scanResult = await getChannelMessages(
client,
channel.telegramId,
mapping.lastProcessedMessageId
);
if (scanResult.archives.length === 0) {
accountLog.debug({ channelId: channel.id }, "No new archives");
continue;
}
accountLog.info(
{ archives: scanResult.archives.length, photos: scanResult.photos.length },
"Found messages in channel"
);
// For non-forum, creator comes from filename (set to null, resolved per-archive)
pipelineCtx.topicCreator = null;
pipelineCtx.sourceTopicId = null;
pipelineCtx.channelTitle = channel.title;
await processArchiveSets(pipelineCtx, scanResult, run.id);
// Update last processed message
const allMsgIds = [
...scanResult.archives.map((m) => m.id),
...scanResult.photos.map((p) => p.id),
];
if (allMsgIds.length > 0) {
const maxId = allMsgIds.reduce((a, b) => (a > b ? a : b));
await updateLastProcessedMessage(mapping.id, maxId);
}
}
}
// ── Done ──
await completeIngestionRun(activeRunId, counters);
accountLog.info({ counters }, "Ingestion run completed");
} finally {
await closeTdlibClient(client);
}
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
accountLog.error({ err }, "Ingestion run failed");
if (runId) {
await failIngestionRun(runId, message).catch((e) =>
accountLog.error({ e }, "Failed to mark run as failed")
);
}
} finally {
await releaseLock(account.id);
}
}
/**
* Process a scan result through the archive pipeline:
* group → download → hash → dedup → metadata → split → upload → preview → index.
*/
async function processArchiveSets(
ctx: PipelineContext,
scanResult: ChannelScanResult,
ingestionRunId: string
): Promise<void> {
const { client, runId, channelTitle, channel, throttled, counters, accountLog } = ctx;
// Group into archive sets
const archiveSets = groupArchiveSets(scanResult.archives);
counters.zipsFound += archiveSets.length;
// Match preview photos to archive sets
const previewMatches = matchPreviewToArchive(
scanResult.photos,
archiveSets.map((s) => ({
baseName: s.baseName,
firstMessageId: s.parts[0].id,
firstMessageDate: s.parts[0].date,
}))
);
if (previewMatches.size > 0) {
accountLog.info(
{ matched: previewMatches.size, total: archiveSets.length },
"Matched preview photos to archives"
);
}
await updateRunActivity(runId, {
currentActivity: `Found ${archiveSets.length} archive(s) in "${channelTitle}"`,
currentStep: "scanning",
currentChannel: channelTitle,
totalFiles: archiveSets.length,
zipsFound: counters.zipsFound,
});
for (let setIdx = 0; setIdx < archiveSets.length; setIdx++) {
await processOneArchiveSet(
ctx,
archiveSets[setIdx],
setIdx,
archiveSets.length,
previewMatches,
ingestionRunId
);
}
}
/**
* Process a single archive set through the full pipeline.
*/
async function processOneArchiveSet(
ctx: PipelineContext,
archiveSet: ArchiveSet,
setIdx: number,
totalSets: number,
previewMatches: Map<string, { id: bigint; fileId: string }>,
ingestionRunId: string
): Promise<void> {
const {
client, runId, channelTitle, channel,
destChannelTelegramId, destChannelId,
throttled, counters, topicCreator, sourceTopicId, accountLog,
} = ctx;
counters.messagesScanned += archiveSet.parts.length;
const archiveName = archiveSet.parts[0].fileName;
const tempPaths: string[] = [];
let splitPaths: string[] = [];
try {
// ── Downloading ──
for (let partIdx = 0; partIdx < archiveSet.parts.length; partIdx++) {
const part = archiveSet.parts[partIdx];
const tempPath = path.join(
config.tempDir,
`${ingestionRunId}_${part.id}_${part.fileName}`
);
const partLabel = archiveSet.parts.length > 1
? ` (part ${partIdx + 1}/${archiveSet.parts.length})`
: "";
await updateRunActivity(runId, {
currentActivity: `Downloading ${part.fileName}${partLabel}`,
currentStep: "downloading",
currentChannel: channelTitle,
currentFile: part.fileName,
currentFileNum: setIdx + 1,
totalFiles: totalSets,
downloadedBytes: 0n,
totalBytes: part.fileSize,
downloadPercent: 0,
messagesScanned: counters.messagesScanned,
});
accountLog.info(
{
fileName: part.fileName,
fileSize: Number(part.fileSize),
part: partIdx + 1,
totalParts: archiveSet.parts.length,
},
"Downloading archive part"
);
await downloadFile(
client,
part.fileId,
tempPath,
part.fileSize,
part.fileName,
(progress: DownloadProgress) => {
throttled.update({
currentActivity: `Downloading ${part.fileName}${partLabel}${progress.percent}%`,
currentStep: "downloading",
currentChannel: channelTitle,
currentFile: part.fileName,
currentFileNum: setIdx + 1,
totalFiles: totalSets,
downloadedBytes: BigInt(progress.downloadedBytes),
totalBytes: BigInt(progress.totalBytes),
downloadPercent: progress.percent,
});
}
);
await throttled.flush();
tempPaths.push(tempPath);
}
// ── Hashing ──
await updateRunActivity(runId, {
currentActivity: `Computing hash for ${archiveName}`,
currentStep: "hashing",
currentChannel: channelTitle,
currentFile: archiveName,
currentFileNum: setIdx + 1,
totalFiles: totalSets,
downloadedBytes: null,
totalBytes: null,
downloadPercent: null,
});
const contentHash = await hashParts(tempPaths);
// ── Deduplicating ──
await updateRunActivity(runId, {
currentActivity: `Checking if ${archiveName} is a duplicate`,
currentStep: "deduplicating",
currentChannel: channelTitle,
currentFile: archiveName,
currentFileNum: setIdx + 1,
totalFiles: totalSets,
});
const exists = await packageExistsByHash(contentHash);
if (exists) {
counters.zipsDuplicate++;
accountLog.debug({ contentHash }, "Duplicate archive, skipping");
await updateRunActivity(runId, {
currentActivity: `Skipped ${archiveName} (duplicate)`,
currentStep: "deduplicating",
currentChannel: channelTitle,
currentFile: archiveName,
currentFileNum: setIdx + 1,
totalFiles: totalSets,
zipsDuplicate: counters.zipsDuplicate,
});
return;
}
// ── Reading metadata ──
await updateRunActivity(runId, {
currentActivity: `Reading file list from ${archiveName}`,
currentStep: "reading_metadata",
currentChannel: channelTitle,
currentFile: archiveName,
currentFileNum: setIdx + 1,
totalFiles: totalSets,
});
let entries: { path: string; fileName: string; extension: string | null; compressedSize: bigint; uncompressedSize: bigint; crc32: string | null }[] = [];
try {
if (archiveSet.type === "ZIP") {
entries = await readZipCentralDirectory(tempPaths);
} else {
entries = await readRarContents(tempPaths[0]);
}
} catch (err) {
accountLog.warn({ err, baseName: archiveSet.baseName }, "Failed to read archive metadata, ingesting without file list");
}
// ── Splitting (if needed) ──
let uploadPaths = tempPaths;
const totalSize = archiveSet.parts.reduce(
(sum, p) => sum + p.fileSize,
0n
);
if (!archiveSet.isMultipart && totalSize > 2n * 1024n * 1024n * 1024n) {
await updateRunActivity(runId, {
currentActivity: `Splitting ${archiveName} for upload (>2GB)`,
currentStep: "splitting",
currentChannel: channelTitle,
currentFile: archiveName,
currentFileNum: setIdx + 1,
totalFiles: totalSets,
});
splitPaths = await byteLevelSplit(tempPaths[0]);
uploadPaths = splitPaths;
}
// ── Uploading ──
const uploadLabel = uploadPaths.length > 1
? ` (${uploadPaths.length} parts)`
: "";
await updateRunActivity(runId, {
currentActivity: `Uploading ${archiveName} to archive channel${uploadLabel}`,
currentStep: "uploading",
currentChannel: channelTitle,
currentFile: archiveName,
currentFileNum: setIdx + 1,
totalFiles: totalSets,
});
const destResult = await uploadToChannel(
client,
destChannelTelegramId,
uploadPaths
);
// ── Preview thumbnail ──
let previewData: Buffer | null = null;
let previewMsgId: bigint | null = null;
const matchedPhoto = previewMatches.get(archiveSet.baseName);
if (matchedPhoto) {
await updateRunActivity(runId, {
currentActivity: `Downloading preview image for ${archiveName}`,
currentStep: "preview",
currentChannel: channelTitle,
currentFile: archiveName,
currentFileNum: setIdx + 1,
totalFiles: totalSets,
});
previewData = await downloadPhotoThumbnail(client, matchedPhoto.fileId);
previewMsgId = matchedPhoto.id;
}
// ── Resolve creator: topic name > filename extraction > null ──
const creator = topicCreator ?? extractCreatorFromFileName(archiveName) ?? null;
// ── Indexing ──
await updateRunActivity(runId, {
currentActivity: `Saving metadata for ${archiveName} (${entries.length} files)`,
currentStep: "indexing",
currentChannel: channelTitle,
currentFile: archiveName,
currentFileNum: setIdx + 1,
totalFiles: totalSets,
});
await createPackageWithFiles({
contentHash,
fileName: archiveName,
fileSize: totalSize,
archiveType: archiveSet.type,
sourceChannelId: channel.id,
sourceMessageId: archiveSet.parts[0].id,
sourceTopicId,
destChannelId,
destMessageId: destResult.messageId,
isMultipart:
archiveSet.parts.length > 1 || uploadPaths.length > 1,
partCount: uploadPaths.length,
ingestionRunId,
creator,
previewData,
previewMsgId,
files: entries,
});
counters.zipsIngested++;
await updateRunActivity(runId, {
currentActivity: `Ingested ${archiveName} (${entries.length} files indexed)`,
currentStep: "complete",
currentChannel: channelTitle,
currentFile: archiveName,
currentFileNum: setIdx + 1,
totalFiles: totalSets,
zipsIngested: counters.zipsIngested,
});
accountLog.info(
{ fileName: archiveName, contentHash, fileCount: entries.length, creator },
"Archive ingested"
);
} finally {
// ALWAYS delete temp files
await deleteFiles([...tempPaths, ...splitPaths]);
}
}
async function deleteFiles(paths: string[]): Promise<void> {
for (const p of paths) {
try {
await unlink(p);
} catch {
// File may already be deleted or never created
}
}
}
/**
* Clean up any leftover temp files from previous runs.
*/
export async function cleanupTempDir(): Promise<void> {
try {
const files = await readdir(config.tempDir);
for (const file of files) {
await unlink(path.join(config.tempDir, file)).catch(() => {});
}
if (files.length > 0) {
log.info({ count: files.length }, "Cleaned up stale temp files");
}
} catch {
// Directory might not exist yet
}
}

19
worker/tsconfig.json Normal file
View File

@@ -0,0 +1,19 @@
{
"compilerOptions": {
"target": "ES2022",
"module": "ESNext",
"moduleResolution": "bundler",
"lib": ["ES2022"],
"outDir": "dist",
"rootDir": "src",
"strict": true,
"esModuleInterop": true,
"skipLibCheck": true,
"resolveJsonModule": true,
"isolatedModules": true,
"declaration": true,
"sourceMap": true
},
"include": ["src/**/*.ts"],
"exclude": ["node_modules", "dist"]
}