mirror of
https://github.com/xCyanGrizzly/DragonsStash.git
synced 2026-05-10 22:01:16 +00:00
feat: add Telegram integration with forum topic support and creator tracking
Adds full Telegram ZIP ingestion pipeline: TDLib worker service scans source channels for archive files, deduplicates by content hash, extracts metadata, uploads to archive channel, and indexes in Postgres. Forum supergroups are scanned per-topic with topic names used as creator. Filename-based creator extraction (e.g. "Mammoth Factory - 2026-01.zip") serves as fallback. Includes admin UI for managing accounts/channels, simplified account setup (API credentials via env vars), auth code/password submission dialog, package browser with creator column, and live ingestion activity tracking. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
11
.env.example
11
.env.example
@@ -11,3 +11,14 @@ AUTH_GITHUB_SECRET=""
|
||||
|
||||
# App
|
||||
NEXT_PUBLIC_APP_URL="http://localhost:3000"
|
||||
|
||||
# Telegram integration (get from https://my.telegram.org/apps)
|
||||
TELEGRAM_API_ID=""
|
||||
TELEGRAM_API_HASH=""
|
||||
|
||||
# Worker (only needed when running worker container)
|
||||
WORKER_INTERVAL_MINUTES=60
|
||||
WORKER_TEMP_DIR="/tmp/zips"
|
||||
TDLIB_STATE_DIR="/data/tdlib"
|
||||
WORKER_MAX_ZIP_SIZE_MB=4096
|
||||
LOG_LEVEL="info"
|
||||
|
||||
5
.gitignore
vendored
5
.gitignore
vendored
@@ -1,5 +1,6 @@
|
||||
# dependencies
|
||||
/node_modules
|
||||
worker/node_modules
|
||||
/.pnp
|
||||
.pnp.*
|
||||
.yarn/*
|
||||
@@ -48,3 +49,7 @@ src/generated
|
||||
# ide
|
||||
.idea
|
||||
.vscode
|
||||
|
||||
# temp files
|
||||
nul
|
||||
tmpclaude-*
|
||||
|
||||
@@ -15,5 +15,27 @@ services:
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
worker:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: worker/Dockerfile
|
||||
environment:
|
||||
- DATABASE_URL=postgresql://dragons:stash@db:5432/dragonsstash
|
||||
- WORKER_INTERVAL_MINUTES=5
|
||||
- WORKER_TEMP_DIR=/tmp/zips
|
||||
- TDLIB_STATE_DIR=/data/tdlib
|
||||
- WORKER_MAX_ZIP_SIZE_MB=4096
|
||||
- LOG_LEVEL=debug
|
||||
- TELEGRAM_API_ID=${TELEGRAM_API_ID}
|
||||
- TELEGRAM_API_HASH=${TELEGRAM_API_HASH}
|
||||
volumes:
|
||||
- tdlib_dev_state:/data/tdlib
|
||||
- tmp_dev_zips:/tmp/zips
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
|
||||
volumes:
|
||||
postgres_dev_data:
|
||||
tdlib_dev_state:
|
||||
tmp_dev_zips:
|
||||
|
||||
@@ -10,11 +10,37 @@ services:
|
||||
- AUTH_SECRET=change-me-to-a-random-secret-in-production
|
||||
- AUTH_TRUST_HOST=true
|
||||
- NEXT_PUBLIC_APP_URL=http://localhost:3000
|
||||
- TELEGRAM_API_KEY=${TELEGRAM_API_KEY:-}
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
|
||||
worker:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: worker/Dockerfile
|
||||
environment:
|
||||
- DATABASE_URL=postgresql://dragons:stash@db:5432/dragonsstash
|
||||
- WORKER_INTERVAL_MINUTES=60
|
||||
- WORKER_TEMP_DIR=/tmp/zips
|
||||
- TDLIB_STATE_DIR=/data/tdlib
|
||||
- WORKER_MAX_ZIP_SIZE_MB=4096
|
||||
- LOG_LEVEL=info
|
||||
volumes:
|
||||
- tdlib_state:/data/tdlib
|
||||
- tmp_zips:/tmp/zips
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 1G
|
||||
reservations:
|
||||
memory: 256M
|
||||
|
||||
db:
|
||||
image: postgres:16-alpine
|
||||
ports:
|
||||
@@ -34,3 +60,5 @@ services:
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
tdlib_state:
|
||||
tmp_zips:
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
-- CreateEnum
|
||||
CREATE TYPE "AuthState" AS ENUM ('PENDING', 'AWAITING_CODE', 'AWAITING_PASSWORD', 'AUTHENTICATED', 'EXPIRED');
|
||||
|
||||
-- CreateEnum
|
||||
CREATE TYPE "ChannelType" AS ENUM ('SOURCE', 'DESTINATION');
|
||||
|
||||
-- CreateEnum
|
||||
CREATE TYPE "ChannelRole" AS ENUM ('READER', 'WRITER');
|
||||
|
||||
-- CreateEnum
|
||||
CREATE TYPE "ArchiveType" AS ENUM ('ZIP', 'RAR');
|
||||
|
||||
-- CreateEnum
|
||||
CREATE TYPE "IngestionStatus" AS ENUM ('RUNNING', 'COMPLETED', 'FAILED', 'CANCELLED');
|
||||
|
||||
-- CreateTable
|
||||
CREATE TABLE "telegram_accounts" (
|
||||
"id" TEXT NOT NULL,
|
||||
"phone" TEXT NOT NULL,
|
||||
"displayName" TEXT,
|
||||
"apiId" INTEGER NOT NULL,
|
||||
"apiHash" TEXT NOT NULL,
|
||||
"sessionPath" TEXT NOT NULL,
|
||||
"isActive" BOOLEAN NOT NULL DEFAULT true,
|
||||
"authState" "AuthState" NOT NULL DEFAULT 'PENDING',
|
||||
"authCode" TEXT,
|
||||
"lastSeenAt" TIMESTAMP(3),
|
||||
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"updatedAt" TIMESTAMP(3) NOT NULL,
|
||||
|
||||
CONSTRAINT "telegram_accounts_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
-- CreateTable
|
||||
CREATE TABLE "telegram_channels" (
|
||||
"id" TEXT NOT NULL,
|
||||
"telegramId" BIGINT NOT NULL,
|
||||
"title" TEXT NOT NULL,
|
||||
"type" "ChannelType" NOT NULL,
|
||||
"isActive" BOOLEAN NOT NULL DEFAULT true,
|
||||
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"updatedAt" TIMESTAMP(3) NOT NULL,
|
||||
|
||||
CONSTRAINT "telegram_channels_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
-- CreateTable
|
||||
CREATE TABLE "account_channel_map" (
|
||||
"id" TEXT NOT NULL,
|
||||
"accountId" TEXT NOT NULL,
|
||||
"channelId" TEXT NOT NULL,
|
||||
"role" "ChannelRole" NOT NULL DEFAULT 'READER',
|
||||
"lastProcessedMessageId" BIGINT,
|
||||
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
|
||||
CONSTRAINT "account_channel_map_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
-- CreateTable
|
||||
CREATE TABLE "packages" (
|
||||
"id" TEXT NOT NULL,
|
||||
"contentHash" TEXT NOT NULL,
|
||||
"fileName" TEXT NOT NULL,
|
||||
"fileSize" BIGINT NOT NULL,
|
||||
"archiveType" "ArchiveType" NOT NULL,
|
||||
"sourceChannelId" TEXT NOT NULL,
|
||||
"sourceMessageId" BIGINT NOT NULL,
|
||||
"destChannelId" TEXT,
|
||||
"destMessageId" BIGINT,
|
||||
"isMultipart" BOOLEAN NOT NULL DEFAULT false,
|
||||
"partCount" INTEGER NOT NULL DEFAULT 1,
|
||||
"fileCount" INTEGER NOT NULL DEFAULT 0,
|
||||
"indexedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"ingestionRunId" TEXT,
|
||||
|
||||
CONSTRAINT "packages_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
-- CreateTable
|
||||
CREATE TABLE "package_files" (
|
||||
"id" TEXT NOT NULL,
|
||||
"packageId" TEXT NOT NULL,
|
||||
"path" TEXT NOT NULL,
|
||||
"fileName" TEXT NOT NULL,
|
||||
"extension" TEXT,
|
||||
"compressedSize" BIGINT NOT NULL DEFAULT 0,
|
||||
"uncompressedSize" BIGINT NOT NULL DEFAULT 0,
|
||||
"crc32" TEXT,
|
||||
|
||||
CONSTRAINT "package_files_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
-- CreateTable
|
||||
CREATE TABLE "ingestion_runs" (
|
||||
"id" TEXT NOT NULL,
|
||||
"accountId" TEXT NOT NULL,
|
||||
"status" "IngestionStatus" NOT NULL DEFAULT 'RUNNING',
|
||||
"startedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"finishedAt" TIMESTAMP(3),
|
||||
"messagesScanned" INTEGER NOT NULL DEFAULT 0,
|
||||
"zipsFound" INTEGER NOT NULL DEFAULT 0,
|
||||
"zipsDuplicate" INTEGER NOT NULL DEFAULT 0,
|
||||
"zipsIngested" INTEGER NOT NULL DEFAULT 0,
|
||||
"errorMessage" TEXT,
|
||||
|
||||
CONSTRAINT "ingestion_runs_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
-- CreateIndex
|
||||
CREATE UNIQUE INDEX "telegram_accounts_phone_key" ON "telegram_accounts"("phone");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "telegram_accounts_isActive_idx" ON "telegram_accounts"("isActive");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE UNIQUE INDEX "telegram_channels_telegramId_key" ON "telegram_channels"("telegramId");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "telegram_channels_type_isActive_idx" ON "telegram_channels"("type", "isActive");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "account_channel_map_accountId_idx" ON "account_channel_map"("accountId");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "account_channel_map_channelId_idx" ON "account_channel_map"("channelId");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE UNIQUE INDEX "account_channel_map_accountId_channelId_key" ON "account_channel_map"("accountId", "channelId");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE UNIQUE INDEX "packages_contentHash_key" ON "packages"("contentHash");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "packages_sourceChannelId_idx" ON "packages"("sourceChannelId");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "packages_destChannelId_idx" ON "packages"("destChannelId");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "packages_fileName_idx" ON "packages"("fileName");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "packages_indexedAt_idx" ON "packages"("indexedAt");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "packages_archiveType_idx" ON "packages"("archiveType");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "package_files_packageId_idx" ON "package_files"("packageId");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "package_files_extension_idx" ON "package_files"("extension");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "package_files_fileName_idx" ON "package_files"("fileName");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "ingestion_runs_accountId_idx" ON "ingestion_runs"("accountId");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "ingestion_runs_status_idx" ON "ingestion_runs"("status");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "ingestion_runs_startedAt_idx" ON "ingestion_runs"("startedAt");
|
||||
|
||||
-- AddForeignKey
|
||||
ALTER TABLE "account_channel_map" ADD CONSTRAINT "account_channel_map_accountId_fkey" FOREIGN KEY ("accountId") REFERENCES "telegram_accounts"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||
|
||||
-- AddForeignKey
|
||||
ALTER TABLE "account_channel_map" ADD CONSTRAINT "account_channel_map_channelId_fkey" FOREIGN KEY ("channelId") REFERENCES "telegram_channels"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||
|
||||
-- AddForeignKey
|
||||
ALTER TABLE "packages" ADD CONSTRAINT "packages_sourceChannelId_fkey" FOREIGN KEY ("sourceChannelId") REFERENCES "telegram_channels"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
|
||||
|
||||
-- AddForeignKey
|
||||
ALTER TABLE "packages" ADD CONSTRAINT "packages_ingestionRunId_fkey" FOREIGN KEY ("ingestionRunId") REFERENCES "ingestion_runs"("id") ON DELETE SET NULL ON UPDATE CASCADE;
|
||||
|
||||
-- AddForeignKey
|
||||
ALTER TABLE "package_files" ADD CONSTRAINT "package_files_packageId_fkey" FOREIGN KEY ("packageId") REFERENCES "packages"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||
|
||||
-- AddForeignKey
|
||||
ALTER TABLE "ingestion_runs" ADD CONSTRAINT "ingestion_runs_accountId_fkey" FOREIGN KEY ("accountId") REFERENCES "telegram_accounts"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
|
||||
@@ -0,0 +1,11 @@
|
||||
-- AlterTable
|
||||
ALTER TABLE "ingestion_runs" ADD COLUMN "currentActivity" TEXT,
|
||||
ADD COLUMN "currentChannel" TEXT,
|
||||
ADD COLUMN "currentFile" TEXT,
|
||||
ADD COLUMN "currentFileNum" INTEGER,
|
||||
ADD COLUMN "currentStep" TEXT,
|
||||
ADD COLUMN "downloadPercent" INTEGER,
|
||||
ADD COLUMN "downloadedBytes" BIGINT,
|
||||
ADD COLUMN "lastActivityAt" TIMESTAMP(3),
|
||||
ADD COLUMN "totalBytes" BIGINT,
|
||||
ADD COLUMN "totalFiles" INTEGER;
|
||||
@@ -0,0 +1,3 @@
|
||||
-- AlterTable
|
||||
ALTER TABLE "packages" ADD COLUMN "previewData" BYTEA,
|
||||
ADD COLUMN "previewMsgId" BIGINT;
|
||||
@@ -0,0 +1,12 @@
|
||||
/*
|
||||
Warnings:
|
||||
|
||||
- You are about to drop the column `apiHash` on the `telegram_accounts` table. All the data in the column will be lost.
|
||||
- You are about to drop the column `apiId` on the `telegram_accounts` table. All the data in the column will be lost.
|
||||
- You are about to drop the column `sessionPath` on the `telegram_accounts` table. All the data in the column will be lost.
|
||||
|
||||
*/
|
||||
-- AlterTable
|
||||
ALTER TABLE "telegram_accounts" DROP COLUMN "apiHash",
|
||||
DROP COLUMN "apiId",
|
||||
DROP COLUMN "sessionPath";
|
||||
@@ -0,0 +1,29 @@
|
||||
-- AlterTable
|
||||
ALTER TABLE "packages" ADD COLUMN "creator" TEXT,
|
||||
ADD COLUMN "sourceTopicId" BIGINT;
|
||||
|
||||
-- AlterTable
|
||||
ALTER TABLE "telegram_channels" ADD COLUMN "isForum" BOOLEAN NOT NULL DEFAULT false;
|
||||
|
||||
-- CreateTable
|
||||
CREATE TABLE "topic_progress" (
|
||||
"id" TEXT NOT NULL,
|
||||
"accountChannelMapId" TEXT NOT NULL,
|
||||
"topicId" BIGINT NOT NULL,
|
||||
"topicName" TEXT,
|
||||
"lastProcessedMessageId" BIGINT,
|
||||
|
||||
CONSTRAINT "topic_progress_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "topic_progress_accountChannelMapId_idx" ON "topic_progress"("accountChannelMapId");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE UNIQUE INDEX "topic_progress_accountChannelMapId_topicId_key" ON "topic_progress"("accountChannelMapId", "topicId");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "packages_creator_idx" ON "packages"("creator");
|
||||
|
||||
-- AddForeignKey
|
||||
ALTER TABLE "topic_progress" ADD CONSTRAINT "topic_progress_accountChannelMapId_fkey" FOREIGN KEY ("accountChannelMapId") REFERENCES "account_channel_map"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||
@@ -349,3 +349,189 @@ model UserSettings {
|
||||
|
||||
user User @relation(fields: [userId], references: [id], onDelete: Cascade)
|
||||
}
|
||||
|
||||
// ───────────────────────────────────────
|
||||
// Telegram ingestion models
|
||||
// ───────────────────────────────────────
|
||||
|
||||
enum AuthState {
|
||||
PENDING
|
||||
AWAITING_CODE
|
||||
AWAITING_PASSWORD
|
||||
AUTHENTICATED
|
||||
EXPIRED
|
||||
}
|
||||
|
||||
enum ChannelType {
|
||||
SOURCE
|
||||
DESTINATION
|
||||
}
|
||||
|
||||
enum ChannelRole {
|
||||
READER
|
||||
WRITER
|
||||
}
|
||||
|
||||
enum ArchiveType {
|
||||
ZIP
|
||||
RAR
|
||||
}
|
||||
|
||||
enum IngestionStatus {
|
||||
RUNNING
|
||||
COMPLETED
|
||||
FAILED
|
||||
CANCELLED
|
||||
}
|
||||
|
||||
model TelegramAccount {
|
||||
id String @id @default(cuid())
|
||||
phone String @unique
|
||||
displayName String?
|
||||
isActive Boolean @default(true)
|
||||
authState AuthState @default(PENDING)
|
||||
authCode String?
|
||||
lastSeenAt DateTime?
|
||||
createdAt DateTime @default(now())
|
||||
updatedAt DateTime @updatedAt
|
||||
|
||||
channelMaps AccountChannelMap[]
|
||||
ingestionRuns IngestionRun[]
|
||||
|
||||
@@index([isActive])
|
||||
@@map("telegram_accounts")
|
||||
}
|
||||
|
||||
model TelegramChannel {
|
||||
id String @id @default(cuid())
|
||||
telegramId BigInt @unique
|
||||
title String
|
||||
type ChannelType
|
||||
isForum Boolean @default(false)
|
||||
isActive Boolean @default(true)
|
||||
createdAt DateTime @default(now())
|
||||
updatedAt DateTime @updatedAt
|
||||
|
||||
accountMaps AccountChannelMap[]
|
||||
packages Package[]
|
||||
|
||||
@@index([type, isActive])
|
||||
@@map("telegram_channels")
|
||||
}
|
||||
|
||||
model AccountChannelMap {
|
||||
id String @id @default(cuid())
|
||||
accountId String
|
||||
channelId String
|
||||
role ChannelRole @default(READER)
|
||||
lastProcessedMessageId BigInt?
|
||||
createdAt DateTime @default(now())
|
||||
|
||||
account TelegramAccount @relation(fields: [accountId], references: [id], onDelete: Cascade)
|
||||
channel TelegramChannel @relation(fields: [channelId], references: [id], onDelete: Cascade)
|
||||
topicProgress TopicProgress[]
|
||||
|
||||
@@unique([accountId, channelId])
|
||||
@@index([accountId])
|
||||
@@index([channelId])
|
||||
@@map("account_channel_map")
|
||||
}
|
||||
|
||||
model Package {
|
||||
id String @id @default(cuid())
|
||||
contentHash String @unique
|
||||
fileName String
|
||||
fileSize BigInt
|
||||
archiveType ArchiveType
|
||||
creator String?
|
||||
sourceChannelId String
|
||||
sourceMessageId BigInt
|
||||
sourceTopicId BigInt?
|
||||
destChannelId String?
|
||||
destMessageId BigInt?
|
||||
isMultipart Boolean @default(false)
|
||||
partCount Int @default(1)
|
||||
fileCount Int @default(0)
|
||||
previewData Bytes? // JPEG thumbnail from nearby Telegram photo (stored as raw bytes)
|
||||
previewMsgId BigInt? // Telegram message ID of the matched photo
|
||||
indexedAt DateTime @default(now())
|
||||
createdAt DateTime @default(now())
|
||||
|
||||
sourceChannel TelegramChannel @relation(fields: [sourceChannelId], references: [id])
|
||||
files PackageFile[]
|
||||
ingestionRun IngestionRun? @relation(fields: [ingestionRunId], references: [id])
|
||||
ingestionRunId String?
|
||||
|
||||
@@index([sourceChannelId])
|
||||
@@index([destChannelId])
|
||||
@@index([fileName])
|
||||
@@index([indexedAt])
|
||||
@@index([archiveType])
|
||||
@@index([creator])
|
||||
@@map("packages")
|
||||
}
|
||||
|
||||
model PackageFile {
|
||||
id String @id @default(cuid())
|
||||
packageId String
|
||||
path String
|
||||
fileName String
|
||||
extension String?
|
||||
compressedSize BigInt @default(0)
|
||||
uncompressedSize BigInt @default(0)
|
||||
crc32 String?
|
||||
|
||||
package Package @relation(fields: [packageId], references: [id], onDelete: Cascade)
|
||||
|
||||
@@index([packageId])
|
||||
@@index([extension])
|
||||
@@index([fileName])
|
||||
@@map("package_files")
|
||||
}
|
||||
|
||||
model IngestionRun {
|
||||
id String @id @default(cuid())
|
||||
accountId String
|
||||
status IngestionStatus @default(RUNNING)
|
||||
startedAt DateTime @default(now())
|
||||
finishedAt DateTime?
|
||||
messagesScanned Int @default(0)
|
||||
zipsFound Int @default(0)
|
||||
zipsDuplicate Int @default(0)
|
||||
zipsIngested Int @default(0)
|
||||
errorMessage String?
|
||||
|
||||
// Live activity tracking — written by worker in real-time
|
||||
currentActivity String? // Human-readable: "Downloading pack.zip (part 2/5)"
|
||||
currentStep String? // Machine-readable step key
|
||||
currentChannel String? // Channel title being processed
|
||||
currentFile String? // File currently being processed
|
||||
currentFileNum Int? // Which archive set (1-indexed)
|
||||
totalFiles Int? // Total archive sets found
|
||||
downloadedBytes BigInt? // Current download progress in bytes
|
||||
totalBytes BigInt? // Total size of current download
|
||||
downloadPercent Int? // 0-100
|
||||
lastActivityAt DateTime? // When activity was last updated
|
||||
|
||||
account TelegramAccount @relation(fields: [accountId], references: [id])
|
||||
packages Package[]
|
||||
|
||||
@@index([accountId])
|
||||
@@index([status])
|
||||
@@index([startedAt])
|
||||
@@map("ingestion_runs")
|
||||
}
|
||||
|
||||
model TopicProgress {
|
||||
id String @id @default(cuid())
|
||||
accountChannelMapId String
|
||||
topicId BigInt
|
||||
topicName String?
|
||||
lastProcessedMessageId BigInt?
|
||||
|
||||
accountChannelMap AccountChannelMap @relation(fields: [accountChannelMapId], references: [id], onDelete: Cascade)
|
||||
|
||||
@@unique([accountChannelMapId, topicId])
|
||||
@@index([accountChannelMapId])
|
||||
@@map("topic_progress")
|
||||
}
|
||||
|
||||
149
src/app/(app)/stls/_components/ingestion-status.tsx
Normal file
149
src/app/(app)/stls/_components/ingestion-status.tsx
Normal file
@@ -0,0 +1,149 @@
|
||||
"use client";
|
||||
|
||||
import { useEffect, useState } from "react";
|
||||
import { Loader2, CheckCircle2, XCircle, CloudOff } from "lucide-react";
|
||||
import { cn } from "@/lib/utils";
|
||||
import type { IngestionAccountStatus } from "@/lib/telegram/types";
|
||||
|
||||
interface IngestionStatusProps {
|
||||
initialStatus: IngestionAccountStatus[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Polls /api/ingestion/status every 3 seconds while a run is active,
|
||||
* or every 30 seconds when idle. Shows a compact status banner with
|
||||
* a spinning throbber when ingestion is running.
|
||||
*/
|
||||
export function IngestionStatus({ initialStatus }: IngestionStatusProps) {
|
||||
const [accounts, setAccounts] = useState(initialStatus);
|
||||
const [error, setError] = useState(false);
|
||||
|
||||
// Determine if any account is currently running
|
||||
const activeRun = accounts.find((a) => a.currentRun);
|
||||
const isRunning = !!activeRun;
|
||||
|
||||
useEffect(() => {
|
||||
let timer: ReturnType<typeof setTimeout>;
|
||||
let mounted = true;
|
||||
|
||||
const poll = async () => {
|
||||
try {
|
||||
const res = await fetch("/api/ingestion/status");
|
||||
if (!res.ok) throw new Error("fetch failed");
|
||||
const data = await res.json();
|
||||
if (mounted) {
|
||||
setAccounts(data.accounts ?? []);
|
||||
setError(false);
|
||||
}
|
||||
} catch {
|
||||
if (mounted) setError(true);
|
||||
}
|
||||
if (mounted) {
|
||||
// Poll fast while running, slow when idle
|
||||
const interval = accounts.some((a) => a.currentRun) ? 3_000 : 30_000;
|
||||
timer = setTimeout(poll, interval);
|
||||
}
|
||||
};
|
||||
|
||||
// Start polling after a short delay to avoid double-fetching on mount
|
||||
timer = setTimeout(poll, 3_000);
|
||||
|
||||
return () => {
|
||||
mounted = false;
|
||||
clearTimeout(timer);
|
||||
};
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [isRunning]);
|
||||
|
||||
// Nothing to show if no accounts configured
|
||||
if (accounts.length === 0 && !error) return null;
|
||||
|
||||
// If we can't reach the API, show a muted offline badge
|
||||
if (error) {
|
||||
return (
|
||||
<div className="flex items-center gap-2 rounded-lg border border-border bg-card px-3 py-2 text-xs text-muted-foreground">
|
||||
<CloudOff className="h-3.5 w-3.5" />
|
||||
<span>Sync status unavailable</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// Active run — show throbber with live activity
|
||||
if (activeRun?.currentRun) {
|
||||
const run = activeRun.currentRun;
|
||||
return (
|
||||
<div className="flex items-center gap-3 rounded-lg border border-primary/20 bg-primary/5 px-3 py-2">
|
||||
<Loader2 className="h-4 w-4 shrink-0 animate-spin text-primary" />
|
||||
<div className="min-w-0 flex-1">
|
||||
<p className="truncate text-sm font-medium text-primary">
|
||||
{run.currentActivity ?? "Syncing..."}
|
||||
</p>
|
||||
{run.downloadPercent != null && run.downloadPercent > 0 && (
|
||||
<div className="mt-1 flex items-center gap-2">
|
||||
<div className="h-1.5 w-24 rounded-full bg-primary/20">
|
||||
<div
|
||||
className="h-full rounded-full bg-primary transition-all duration-300"
|
||||
style={{ width: `${Math.min(100, run.downloadPercent)}%` }}
|
||||
/>
|
||||
</div>
|
||||
<span className="text-[10px] text-primary/70">{run.downloadPercent}%</span>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
{run.totalFiles != null && run.currentFileNum != null && (
|
||||
<span className="shrink-0 text-[10px] text-primary/60">
|
||||
{run.currentFileNum}/{run.totalFiles}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// All idle — show last run summary
|
||||
const lastCompleted = accounts
|
||||
.filter((a) => a.lastRun)
|
||||
.sort(
|
||||
(a, b) =>
|
||||
new Date(b.lastRun!.finishedAt ?? b.lastRun!.startedAt).getTime() -
|
||||
new Date(a.lastRun!.finishedAt ?? a.lastRun!.startedAt).getTime()
|
||||
)[0];
|
||||
|
||||
if (!lastCompleted?.lastRun) return null;
|
||||
|
||||
const last = lastCompleted.lastRun;
|
||||
const isFailed = last.status === "FAILED";
|
||||
const timeAgo = getTimeAgo(last.finishedAt ?? last.startedAt);
|
||||
|
||||
return (
|
||||
<div
|
||||
className={cn(
|
||||
"flex items-center gap-2 rounded-lg border px-3 py-2 text-xs",
|
||||
isFailed
|
||||
? "border-red-500/20 bg-red-500/5 text-red-400"
|
||||
: "border-border bg-card text-muted-foreground"
|
||||
)}
|
||||
>
|
||||
{isFailed ? (
|
||||
<XCircle className="h-3.5 w-3.5 shrink-0" />
|
||||
) : (
|
||||
<CheckCircle2 className="h-3.5 w-3.5 shrink-0 text-emerald-400" />
|
||||
)}
|
||||
<span className="truncate">
|
||||
{isFailed
|
||||
? `Last sync failed ${timeAgo}`
|
||||
: `Last sync ${timeAgo} — ${last.zipsIngested} new, ${last.zipsDuplicate} skipped`}
|
||||
</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function getTimeAgo(dateStr: string): string {
|
||||
const diff = Date.now() - new Date(dateStr).getTime();
|
||||
const mins = Math.floor(diff / 60_000);
|
||||
if (mins < 1) return "just now";
|
||||
if (mins < 60) return `${mins}m ago`;
|
||||
const hours = Math.floor(mins / 60);
|
||||
if (hours < 24) return `${hours}h ago`;
|
||||
const days = Math.floor(hours / 24);
|
||||
return `${days}d ago`;
|
||||
}
|
||||
154
src/app/(app)/stls/_components/package-columns.tsx
Normal file
154
src/app/(app)/stls/_components/package-columns.tsx
Normal file
@@ -0,0 +1,154 @@
|
||||
"use client";
|
||||
|
||||
import { type ColumnDef } from "@tanstack/react-table";
|
||||
import { FileArchive, Eye, ImageIcon } from "lucide-react";
|
||||
import { DataTableColumnHeader } from "@/components/shared/data-table-column-header";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { Button } from "@/components/ui/button";
|
||||
|
||||
export interface PackageRow {
|
||||
id: string;
|
||||
fileName: string;
|
||||
fileSize: string;
|
||||
contentHash: string;
|
||||
archiveType: "ZIP" | "RAR";
|
||||
fileCount: number;
|
||||
isMultipart: boolean;
|
||||
hasPreview: boolean;
|
||||
creator: string | null;
|
||||
indexedAt: string;
|
||||
sourceChannel: {
|
||||
id: string;
|
||||
title: string;
|
||||
};
|
||||
}
|
||||
|
||||
interface PackageColumnsProps {
|
||||
onViewFiles: (pkg: PackageRow) => void;
|
||||
}
|
||||
|
||||
function formatBytes(bytesStr: string): string {
|
||||
const bytes = Number(bytesStr);
|
||||
if (bytes === 0) return "0 B";
|
||||
const k = 1024;
|
||||
const sizes = ["B", "KB", "MB", "GB", "TB"];
|
||||
const i = Math.floor(Math.log(bytes) / Math.log(k));
|
||||
return `${parseFloat((bytes / Math.pow(k, i)).toFixed(1))} ${sizes[i]}`;
|
||||
}
|
||||
|
||||
function PreviewCell({ pkg }: { pkg: PackageRow }) {
|
||||
if (pkg.hasPreview) {
|
||||
return (
|
||||
<img
|
||||
src={`/api/zips/${pkg.id}/preview`}
|
||||
alt=""
|
||||
className="h-9 w-9 rounded-md object-cover bg-muted"
|
||||
loading="lazy"
|
||||
/>
|
||||
);
|
||||
}
|
||||
return (
|
||||
<div className="flex h-9 w-9 items-center justify-center rounded-md bg-muted">
|
||||
<FileArchive className="h-4 w-4 text-muted-foreground" />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export function getPackageColumns({
|
||||
onViewFiles,
|
||||
}: PackageColumnsProps): ColumnDef<PackageRow, unknown>[] {
|
||||
return [
|
||||
{
|
||||
id: "preview",
|
||||
header: "",
|
||||
cell: ({ row }) => <PreviewCell pkg={row.original} />,
|
||||
enableHiding: false,
|
||||
enableSorting: false,
|
||||
size: 52,
|
||||
},
|
||||
{
|
||||
accessorKey: "fileName",
|
||||
header: ({ column }) => <DataTableColumnHeader column={column} title="File Name" />,
|
||||
cell: ({ row }) => (
|
||||
<div className="flex items-center gap-2 min-w-0">
|
||||
<span className="font-medium truncate max-w-[300px]">{row.original.fileName}</span>
|
||||
{row.original.isMultipart && (
|
||||
<Badge variant="outline" className="text-[10px] shrink-0">
|
||||
Multi
|
||||
</Badge>
|
||||
)}
|
||||
</div>
|
||||
),
|
||||
enableHiding: false,
|
||||
},
|
||||
{
|
||||
accessorKey: "archiveType",
|
||||
header: ({ column }) => <DataTableColumnHeader column={column} title="Type" />,
|
||||
cell: ({ row }) => (
|
||||
<Badge variant="secondary" className="text-[10px]">
|
||||
{row.original.archiveType}
|
||||
</Badge>
|
||||
),
|
||||
},
|
||||
{
|
||||
accessorKey: "fileSize",
|
||||
header: ({ column }) => <DataTableColumnHeader column={column} title="Size" />,
|
||||
cell: ({ row }) => (
|
||||
<span className="text-sm text-muted-foreground">
|
||||
{formatBytes(row.original.fileSize)}
|
||||
</span>
|
||||
),
|
||||
},
|
||||
{
|
||||
accessorKey: "fileCount",
|
||||
header: ({ column }) => <DataTableColumnHeader column={column} title="Files" />,
|
||||
cell: ({ row }) => (
|
||||
<span className="text-sm">
|
||||
{row.original.fileCount.toLocaleString()}
|
||||
</span>
|
||||
),
|
||||
},
|
||||
{
|
||||
accessorKey: "creator",
|
||||
header: ({ column }) => <DataTableColumnHeader column={column} title="Creator" />,
|
||||
cell: ({ row }) => (
|
||||
<span className="text-sm text-muted-foreground truncate max-w-[160px] block">
|
||||
{row.original.creator ?? "\u2014"}
|
||||
</span>
|
||||
),
|
||||
},
|
||||
{
|
||||
id: "channel",
|
||||
header: ({ column }) => <DataTableColumnHeader column={column} title="Source" />,
|
||||
cell: ({ row }) => (
|
||||
<span className="text-sm text-muted-foreground truncate max-w-[160px] block">
|
||||
{row.original.sourceChannel.title}
|
||||
</span>
|
||||
),
|
||||
accessorFn: (row) => row.sourceChannel.title,
|
||||
},
|
||||
{
|
||||
accessorKey: "indexedAt",
|
||||
header: ({ column }) => <DataTableColumnHeader column={column} title="Indexed" />,
|
||||
cell: ({ row }) => (
|
||||
<span className="text-sm text-muted-foreground">
|
||||
{new Date(row.original.indexedAt).toLocaleDateString()}
|
||||
</span>
|
||||
),
|
||||
},
|
||||
{
|
||||
id: "actions",
|
||||
cell: ({ row }) => (
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
className="h-8 w-8"
|
||||
onClick={() => onViewFiles(row.original)}
|
||||
>
|
||||
<Eye className="h-4 w-4" />
|
||||
</Button>
|
||||
),
|
||||
enableHiding: false,
|
||||
},
|
||||
];
|
||||
}
|
||||
412
src/app/(app)/stls/_components/package-files-drawer.tsx
Normal file
412
src/app/(app)/stls/_components/package-files-drawer.tsx
Normal file
@@ -0,0 +1,412 @@
|
||||
"use client";
|
||||
|
||||
import { useEffect, useState, useCallback, useMemo } from "react";
|
||||
import {
|
||||
FileText,
|
||||
Folder,
|
||||
FolderOpen,
|
||||
Loader2,
|
||||
Search,
|
||||
ChevronDown,
|
||||
ChevronRight,
|
||||
} from "lucide-react";
|
||||
import {
|
||||
Dialog,
|
||||
DialogContent,
|
||||
DialogHeader,
|
||||
DialogTitle,
|
||||
DialogDescription,
|
||||
} from "@/components/ui/dialog";
|
||||
import { ScrollArea } from "@/components/ui/scroll-area";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { cn } from "@/lib/utils";
|
||||
import type { PackageRow } from "./package-columns";
|
||||
|
||||
interface FileItem {
|
||||
id: string;
|
||||
path: string;
|
||||
fileName: string;
|
||||
extension: string | null;
|
||||
compressedSize: string;
|
||||
uncompressedSize: string;
|
||||
crc32: string | null;
|
||||
}
|
||||
|
||||
interface TreeNode {
|
||||
name: string;
|
||||
isFolder: boolean;
|
||||
children: Map<string, TreeNode>;
|
||||
file?: FileItem;
|
||||
}
|
||||
|
||||
interface PackageFilesDrawerProps {
|
||||
pkg: PackageRow | null;
|
||||
open: boolean;
|
||||
onOpenChange: (open: boolean) => void;
|
||||
}
|
||||
|
||||
function formatBytes(bytesStr: string): string {
|
||||
const bytes = Number(bytesStr);
|
||||
if (bytes === 0) return "0 B";
|
||||
const k = 1024;
|
||||
const sizes = ["B", "KB", "MB", "GB", "TB"];
|
||||
const i = Math.floor(Math.log(bytes) / Math.log(k));
|
||||
return `${parseFloat((bytes / Math.pow(k, i)).toFixed(1))} ${sizes[i]}`;
|
||||
}
|
||||
|
||||
const EXTENSION_COLORS: Record<string, string> = {
|
||||
stl: "bg-blue-500/15 text-blue-400 border-blue-500/30",
|
||||
obj: "bg-violet-500/15 text-violet-400 border-violet-500/30",
|
||||
"3mf": "bg-cyan-500/15 text-cyan-400 border-cyan-500/30",
|
||||
gcode: "bg-amber-500/15 text-amber-400 border-amber-500/30",
|
||||
png: "bg-emerald-500/15 text-emerald-400 border-emerald-500/30",
|
||||
jpg: "bg-emerald-500/15 text-emerald-400 border-emerald-500/30",
|
||||
jpeg: "bg-emerald-500/15 text-emerald-400 border-emerald-500/30",
|
||||
pdf: "bg-red-500/15 text-red-400 border-red-500/30",
|
||||
txt: "bg-zinc-500/15 text-zinc-400 border-zinc-500/30",
|
||||
lys: "bg-pink-500/15 text-pink-400 border-pink-500/30",
|
||||
};
|
||||
|
||||
function getExtBadgeClass(ext: string | null): string {
|
||||
if (!ext) return "bg-zinc-500/15 text-zinc-400 border-zinc-500/30";
|
||||
return EXTENSION_COLORS[ext.toLowerCase()] ?? "bg-zinc-500/15 text-zinc-400 border-zinc-500/30";
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a tree structure from flat file paths.
|
||||
*/
|
||||
function buildFileTree(files: FileItem[]): TreeNode {
|
||||
const root: TreeNode = { name: "", isFolder: true, children: new Map() };
|
||||
|
||||
for (const file of files) {
|
||||
// Normalize path separators (Windows RAR archives may use backslashes)
|
||||
const parts = file.path.replace(/\\/g, "/").split("/").filter(Boolean);
|
||||
let current = root;
|
||||
|
||||
for (let i = 0; i < parts.length; i++) {
|
||||
const part = parts[i];
|
||||
const isLast = i === parts.length - 1;
|
||||
|
||||
if (!current.children.has(part)) {
|
||||
current.children.set(part, {
|
||||
name: part,
|
||||
isFolder: !isLast,
|
||||
children: new Map(),
|
||||
file: isLast ? file : undefined,
|
||||
});
|
||||
}
|
||||
|
||||
current = current.children.get(part)!;
|
||||
}
|
||||
}
|
||||
|
||||
return root;
|
||||
}
|
||||
|
||||
/**
|
||||
* Recursively renders a file tree node with indentation.
|
||||
*/
|
||||
function TreeNodeView({
|
||||
node,
|
||||
depth,
|
||||
search,
|
||||
defaultOpen,
|
||||
}: {
|
||||
node: TreeNode;
|
||||
depth: number;
|
||||
search: string;
|
||||
defaultOpen: boolean;
|
||||
}) {
|
||||
const [open, setOpen] = useState(defaultOpen);
|
||||
|
||||
// Sort children: folders first, then files, alphabetical within each group
|
||||
const sortedChildren = useMemo(() => {
|
||||
const arr = Array.from(node.children.values());
|
||||
return arr.sort((a, b) => {
|
||||
if (a.isFolder !== b.isFolder) return a.isFolder ? -1 : 1;
|
||||
return a.name.localeCompare(b.name);
|
||||
});
|
||||
}, [node.children]);
|
||||
|
||||
// If searching, force all open
|
||||
useEffect(() => {
|
||||
if (search) setOpen(true);
|
||||
}, [search]);
|
||||
|
||||
if (node.isFolder && node.children.size > 0) {
|
||||
return (
|
||||
<div>
|
||||
{/* Don't render a row for the root node */}
|
||||
{depth >= 0 && (
|
||||
<button
|
||||
type="button"
|
||||
className="flex w-full items-center gap-1.5 rounded-md px-1 py-1 text-sm hover:bg-muted/50 transition-colors"
|
||||
style={{ paddingLeft: `${Math.max(0, depth) * 16 + 4}px` }}
|
||||
onClick={() => setOpen(!open)}
|
||||
>
|
||||
{open ? (
|
||||
<ChevronDown className="h-3 w-3 shrink-0 text-muted-foreground" />
|
||||
) : (
|
||||
<ChevronRight className="h-3 w-3 shrink-0 text-muted-foreground" />
|
||||
)}
|
||||
{open ? (
|
||||
<FolderOpen className="h-3.5 w-3.5 shrink-0 text-primary/70" />
|
||||
) : (
|
||||
<Folder className="h-3.5 w-3.5 shrink-0 text-primary/70" />
|
||||
)}
|
||||
<span className="text-sm font-medium truncate">{node.name}</span>
|
||||
<span className="text-[10px] text-muted-foreground ml-auto shrink-0">
|
||||
{countFiles(node)}
|
||||
</span>
|
||||
</button>
|
||||
)}
|
||||
{open &&
|
||||
sortedChildren.map((child) => (
|
||||
<TreeNodeView
|
||||
key={child.name}
|
||||
node={child}
|
||||
depth={depth + 1}
|
||||
search={search}
|
||||
defaultOpen={depth < 1} // Auto-expand first 2 levels
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// File node
|
||||
if (node.file) {
|
||||
return (
|
||||
<div
|
||||
className="flex items-center gap-2 rounded-md px-1 py-1 hover:bg-muted/50 transition-colors"
|
||||
style={{ paddingLeft: `${Math.max(0, depth) * 16 + 4}px` }}
|
||||
>
|
||||
<FileText className="h-3.5 w-3.5 shrink-0 text-muted-foreground" />
|
||||
<span className="text-sm truncate flex-1 min-w-0" title={node.file.path}>
|
||||
{node.name}
|
||||
</span>
|
||||
{node.file.extension && (
|
||||
<Badge
|
||||
variant="outline"
|
||||
className={`text-[10px] shrink-0 ${getExtBadgeClass(node.file.extension)}`}
|
||||
>
|
||||
.{node.file.extension}
|
||||
</Badge>
|
||||
)}
|
||||
<span className="text-[11px] text-muted-foreground shrink-0 tabular-nums">
|
||||
{formatBytes(node.file.uncompressedSize)}
|
||||
</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function countFiles(node: TreeNode): number {
|
||||
if (!node.isFolder) return 1;
|
||||
let count = 0;
|
||||
for (const child of node.children.values()) {
|
||||
count += countFiles(child);
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
const PAGE_SIZE = 100;
|
||||
|
||||
export function PackageFilesDrawer({ pkg, open, onOpenChange }: PackageFilesDrawerProps) {
|
||||
const [files, setFiles] = useState<FileItem[]>([]);
|
||||
const [total, setTotal] = useState(0);
|
||||
const [loading, setLoading] = useState(false);
|
||||
const [loadingMore, setLoadingMore] = useState(false);
|
||||
const [search, setSearch] = useState("");
|
||||
const [page, setPage] = useState(1);
|
||||
|
||||
const fetchFiles = useCallback(
|
||||
async (pageNum: number, append: boolean) => {
|
||||
if (!pkg) return;
|
||||
if (pageNum === 1) setLoading(true);
|
||||
else setLoadingMore(true);
|
||||
|
||||
try {
|
||||
const params = new URLSearchParams({
|
||||
page: String(pageNum),
|
||||
limit: String(PAGE_SIZE),
|
||||
});
|
||||
const res = await fetch(`/api/zips/${pkg.id}/files?${params}`);
|
||||
if (!res.ok) throw new Error("fetch failed");
|
||||
const data = await res.json();
|
||||
setFiles((prev) => (append ? [...prev, ...data.items] : data.items));
|
||||
setTotal(data.pagination.total);
|
||||
} catch {
|
||||
// Silently handle
|
||||
} finally {
|
||||
setLoading(false);
|
||||
setLoadingMore(false);
|
||||
}
|
||||
},
|
||||
[pkg]
|
||||
);
|
||||
|
||||
// Reset and fetch when package changes
|
||||
useEffect(() => {
|
||||
if (open && pkg) {
|
||||
setFiles([]);
|
||||
setTotal(0);
|
||||
setSearch("");
|
||||
setPage(1);
|
||||
fetchFiles(1, false);
|
||||
}
|
||||
}, [open, pkg, fetchFiles]);
|
||||
|
||||
const loadMore = () => {
|
||||
const nextPage = page + 1;
|
||||
setPage(nextPage);
|
||||
fetchFiles(nextPage, true);
|
||||
};
|
||||
|
||||
const hasMore = files.length < total;
|
||||
|
||||
// Client-side search filter (over loaded files)
|
||||
const filtered = search
|
||||
? files.filter(
|
||||
(f) =>
|
||||
f.fileName.toLowerCase().includes(search.toLowerCase()) ||
|
||||
f.path.toLowerCase().includes(search.toLowerCase())
|
||||
)
|
||||
: files;
|
||||
|
||||
// Build tree from filtered files
|
||||
const tree = useMemo(() => buildFileTree(filtered), [filtered]);
|
||||
|
||||
// If all files are in root (no folders), skip the tree and show flat list
|
||||
const hasNesting = useMemo(() => {
|
||||
return filtered.some((f) => f.path.replace(/\\/g, "/").includes("/"));
|
||||
}, [filtered]);
|
||||
|
||||
return (
|
||||
<Dialog open={open} onOpenChange={onOpenChange}>
|
||||
<DialogContent className="sm:max-w-2xl max-h-[80vh] flex flex-col gap-0 p-0">
|
||||
<DialogHeader className="px-6 pt-6 pb-4 border-b border-border space-y-3">
|
||||
{/* Preview image + title row */}
|
||||
<div className="flex gap-4">
|
||||
{pkg?.hasPreview && (
|
||||
<img
|
||||
src={`/api/zips/${pkg.id}/preview`}
|
||||
alt=""
|
||||
className="h-20 w-20 rounded-lg object-cover bg-muted shrink-0"
|
||||
/>
|
||||
)}
|
||||
<div className="min-w-0 flex-1">
|
||||
<DialogTitle className="truncate pr-8">
|
||||
{pkg?.fileName ?? "Package Files"}
|
||||
</DialogTitle>
|
||||
<DialogDescription className="mt-1">
|
||||
{total.toLocaleString()} file{total !== 1 ? "s" : ""} in archive
|
||||
</DialogDescription>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Search within file list */}
|
||||
{files.length > 0 && (
|
||||
<div className="relative">
|
||||
<Search className="absolute left-2.5 top-2.5 h-4 w-4 text-muted-foreground" />
|
||||
<Input
|
||||
placeholder="Filter files..."
|
||||
value={search}
|
||||
onChange={(e) => setSearch(e.target.value)}
|
||||
className="pl-9 h-9"
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
</DialogHeader>
|
||||
|
||||
<ScrollArea className="flex-1 min-h-0">
|
||||
<div className="px-4 py-3">
|
||||
{loading ? (
|
||||
<div className="flex flex-col items-center justify-center gap-2 py-12">
|
||||
<Loader2 className="h-5 w-5 animate-spin text-muted-foreground" />
|
||||
<span className="text-sm text-muted-foreground">Loading files...</span>
|
||||
</div>
|
||||
) : filtered.length === 0 ? (
|
||||
<div className="flex flex-col items-center justify-center gap-2 py-12">
|
||||
<FileText className="h-6 w-6 text-muted-foreground/50" />
|
||||
<span className="text-sm text-muted-foreground">
|
||||
{search ? "No matching files" : "No files indexed"}
|
||||
</span>
|
||||
</div>
|
||||
) : hasNesting ? (
|
||||
<>
|
||||
{/* Render as folder tree */}
|
||||
{Array.from(tree.children.values())
|
||||
.sort((a, b) => {
|
||||
if (a.isFolder !== b.isFolder) return a.isFolder ? -1 : 1;
|
||||
return a.name.localeCompare(b.name);
|
||||
})
|
||||
.map((child) => (
|
||||
<TreeNodeView
|
||||
key={child.name}
|
||||
node={child}
|
||||
depth={0}
|
||||
search={search}
|
||||
defaultOpen={true}
|
||||
/>
|
||||
))}
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
{/* Flat list for archives without folders */}
|
||||
{filtered.map((file) => (
|
||||
<div
|
||||
key={file.id}
|
||||
className="flex items-center gap-3 rounded-md px-2 py-1.5 hover:bg-muted/50 transition-colors"
|
||||
>
|
||||
<FileText className="h-3.5 w-3.5 shrink-0 text-muted-foreground" />
|
||||
<div className="min-w-0 flex-1">
|
||||
<p className="text-sm truncate" title={file.path}>
|
||||
{file.fileName}
|
||||
</p>
|
||||
</div>
|
||||
{file.extension && (
|
||||
<Badge
|
||||
variant="outline"
|
||||
className={`text-[10px] shrink-0 ${getExtBadgeClass(file.extension)}`}
|
||||
>
|
||||
.{file.extension}
|
||||
</Badge>
|
||||
)}
|
||||
<span className="text-[11px] text-muted-foreground shrink-0 tabular-nums">
|
||||
{formatBytes(file.uncompressedSize)}
|
||||
</span>
|
||||
</div>
|
||||
))}
|
||||
</>
|
||||
)}
|
||||
|
||||
{/* Load more button */}
|
||||
{hasMore && !search && (
|
||||
<div className="flex justify-center pt-3 pb-1">
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={loadMore}
|
||||
disabled={loadingMore}
|
||||
className="gap-1"
|
||||
>
|
||||
{loadingMore ? (
|
||||
<Loader2 className="h-3.5 w-3.5 animate-spin" />
|
||||
) : (
|
||||
<ChevronDown className="h-3.5 w-3.5" />
|
||||
)}
|
||||
Load more ({files.length} of {total.toLocaleString()})
|
||||
</Button>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</ScrollArea>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
);
|
||||
}
|
||||
95
src/app/(app)/stls/_components/stl-table.tsx
Normal file
95
src/app/(app)/stls/_components/stl-table.tsx
Normal file
@@ -0,0 +1,95 @@
|
||||
"use client";
|
||||
|
||||
import { useState, useCallback } from "react";
|
||||
import { useRouter, usePathname, useSearchParams } from "next/navigation";
|
||||
import { Search, FileBox } from "lucide-react";
|
||||
import { useDataTable } from "@/hooks/use-data-table";
|
||||
import { getPackageColumns, type PackageRow } from "./package-columns";
|
||||
import { PackageFilesDrawer } from "./package-files-drawer";
|
||||
import { IngestionStatus } from "./ingestion-status";
|
||||
import { DataTable } from "@/components/shared/data-table";
|
||||
import { DataTablePagination } from "@/components/shared/data-table-pagination";
|
||||
import { DataTableViewOptions } from "@/components/shared/data-table-view-options";
|
||||
import { PageHeader } from "@/components/shared/page-header";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import type { IngestionAccountStatus } from "@/lib/telegram/types";
|
||||
|
||||
interface StlTableProps {
|
||||
data: PackageRow[];
|
||||
pageCount: number;
|
||||
totalCount: number;
|
||||
ingestionStatus: IngestionAccountStatus[];
|
||||
}
|
||||
|
||||
export function StlTable({
|
||||
data,
|
||||
pageCount,
|
||||
totalCount,
|
||||
ingestionStatus,
|
||||
}: StlTableProps) {
|
||||
const router = useRouter();
|
||||
const pathname = usePathname();
|
||||
const searchParams = useSearchParams();
|
||||
|
||||
const [searchValue, setSearchValue] = useState(searchParams.get("search") ?? "");
|
||||
const [viewPkg, setViewPkg] = useState<PackageRow | null>(null);
|
||||
|
||||
const updateSearch = useCallback(
|
||||
(value: string) => {
|
||||
setSearchValue(value);
|
||||
const params = new URLSearchParams(searchParams.toString());
|
||||
if (value) {
|
||||
params.set("search", value);
|
||||
params.set("page", "1");
|
||||
} else {
|
||||
params.delete("search");
|
||||
}
|
||||
router.push(`${pathname}?${params.toString()}`, { scroll: false });
|
||||
},
|
||||
[router, pathname, searchParams]
|
||||
);
|
||||
|
||||
const columns = getPackageColumns({
|
||||
onViewFiles: (pkg) => setViewPkg(pkg),
|
||||
});
|
||||
|
||||
const { table } = useDataTable({ data, columns, pageCount });
|
||||
|
||||
return (
|
||||
<div className="space-y-4">
|
||||
<PageHeader
|
||||
title="STL Files"
|
||||
description="Browse indexed archive packages from Telegram channels"
|
||||
>
|
||||
<IngestionStatus initialStatus={ingestionStatus} />
|
||||
</PageHeader>
|
||||
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<div className="relative flex-1 min-w-[200px] max-w-sm">
|
||||
<Search className="absolute left-2.5 top-2.5 h-4 w-4 text-muted-foreground" />
|
||||
<Input
|
||||
placeholder="Search packages or files..."
|
||||
value={searchValue}
|
||||
onChange={(e) => updateSearch(e.target.value)}
|
||||
className="pl-9 h-9"
|
||||
/>
|
||||
</div>
|
||||
<DataTableViewOptions table={table} />
|
||||
</div>
|
||||
|
||||
<DataTable
|
||||
table={table}
|
||||
emptyMessage="No packages found. Archives will appear here after ingestion."
|
||||
/>
|
||||
<DataTablePagination table={table} totalCount={totalCount} />
|
||||
|
||||
<PackageFilesDrawer
|
||||
pkg={viewPkg}
|
||||
open={!!viewPkg}
|
||||
onOpenChange={(open) => {
|
||||
if (!open) setViewPkg(null);
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
50
src/app/(app)/stls/page.tsx
Normal file
50
src/app/(app)/stls/page.tsx
Normal file
@@ -0,0 +1,50 @@
|
||||
import { auth } from "@/lib/auth";
|
||||
import { redirect } from "next/navigation";
|
||||
import { listPackages, searchPackages, getIngestionStatus } from "@/lib/telegram/queries";
|
||||
import { StlTable } from "./_components/stl-table";
|
||||
|
||||
interface Props {
|
||||
searchParams: Promise<Record<string, string | string[] | undefined>>;
|
||||
}
|
||||
|
||||
export default async function StlFilesPage({ searchParams }: Props) {
|
||||
const session = await auth();
|
||||
if (!session?.user?.id) redirect("/login");
|
||||
|
||||
const params = await searchParams;
|
||||
|
||||
const page = Number(params.page) || 1;
|
||||
const perPage = Number(params.perPage) || 20;
|
||||
const sort = (params.sort as string) ?? "indexedAt";
|
||||
const order = (params.order as "asc" | "desc") ?? "desc";
|
||||
const search = (params.search as string) ?? "";
|
||||
const creator = (params.creator as string) || undefined;
|
||||
|
||||
// Fetch packages and ingestion status in parallel
|
||||
const [result, ingestionStatus] = await Promise.all([
|
||||
search
|
||||
? searchPackages({
|
||||
query: search,
|
||||
page,
|
||||
limit: perPage,
|
||||
searchIn: "both",
|
||||
})
|
||||
: listPackages({
|
||||
page,
|
||||
limit: perPage,
|
||||
creator,
|
||||
sortBy: sort as "indexedAt" | "fileName" | "fileSize",
|
||||
order,
|
||||
}),
|
||||
getIngestionStatus(),
|
||||
]);
|
||||
|
||||
return (
|
||||
<StlTable
|
||||
data={result.items}
|
||||
pageCount={result.pagination.totalPages}
|
||||
totalCount={result.pagination.total}
|
||||
ingestionStatus={ingestionStatus}
|
||||
/>
|
||||
);
|
||||
}
|
||||
184
src/app/(app)/telegram/_components/account-columns.tsx
Normal file
184
src/app/(app)/telegram/_components/account-columns.tsx
Normal file
@@ -0,0 +1,184 @@
|
||||
"use client";
|
||||
|
||||
import { type ColumnDef } from "@tanstack/react-table";
|
||||
import {
|
||||
MoreHorizontal,
|
||||
Pencil,
|
||||
Trash2,
|
||||
Power,
|
||||
Link2,
|
||||
Play,
|
||||
KeyRound,
|
||||
} from "lucide-react";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import {
|
||||
DropdownMenu,
|
||||
DropdownMenuContent,
|
||||
DropdownMenuItem,
|
||||
DropdownMenuSeparator,
|
||||
DropdownMenuTrigger,
|
||||
} from "@/components/ui/dropdown-menu";
|
||||
import type { AccountRow } from "@/lib/telegram/admin-queries";
|
||||
|
||||
const authStateColors: Record<string, string> = {
|
||||
PENDING: "bg-yellow-500/10 text-yellow-600 border-yellow-500/20",
|
||||
AWAITING_CODE: "bg-orange-500/10 text-orange-600 border-orange-500/20",
|
||||
AWAITING_PASSWORD: "bg-orange-500/10 text-orange-600 border-orange-500/20",
|
||||
AUTHENTICATED: "bg-green-500/10 text-green-600 border-green-500/20",
|
||||
EXPIRED: "bg-red-500/10 text-red-600 border-red-500/20",
|
||||
};
|
||||
|
||||
interface AccountColumnsProps {
|
||||
onEdit: (account: AccountRow) => void;
|
||||
onToggleActive: (id: string) => void;
|
||||
onDelete: (id: string) => void;
|
||||
onViewLinks: (id: string) => void;
|
||||
onTriggerSync: (id: string) => void;
|
||||
onEnterCode: (account: AccountRow) => void;
|
||||
}
|
||||
|
||||
export function getAccountColumns({
|
||||
onEdit,
|
||||
onToggleActive,
|
||||
onDelete,
|
||||
onViewLinks,
|
||||
onTriggerSync,
|
||||
onEnterCode,
|
||||
}: AccountColumnsProps): ColumnDef<AccountRow, unknown>[] {
|
||||
return [
|
||||
{
|
||||
accessorKey: "displayName",
|
||||
header: "Account",
|
||||
cell: ({ row }) => (
|
||||
<div className="flex flex-col">
|
||||
<span className="font-medium">
|
||||
{row.original.displayName || row.original.phone}
|
||||
</span>
|
||||
{row.original.displayName && (
|
||||
<span className="text-xs text-muted-foreground">
|
||||
{row.original.phone}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
),
|
||||
enableHiding: false,
|
||||
},
|
||||
{
|
||||
accessorKey: "authState",
|
||||
header: "Auth State",
|
||||
cell: ({ row }) => {
|
||||
const needsCode =
|
||||
row.original.authState === "AWAITING_CODE" ||
|
||||
row.original.authState === "AWAITING_PASSWORD";
|
||||
return (
|
||||
<div className="flex items-center gap-2">
|
||||
<Badge
|
||||
variant="outline"
|
||||
className={authStateColors[row.original.authState] ?? ""}
|
||||
>
|
||||
{row.original.authState.replace(/_/g, " ")}
|
||||
</Badge>
|
||||
{needsCode && (
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
className="h-6 gap-1 px-2 text-xs"
|
||||
onClick={() => onEnterCode(row.original)}
|
||||
>
|
||||
<KeyRound className="h-3 w-3" />
|
||||
Enter Code
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
},
|
||||
},
|
||||
{
|
||||
accessorKey: "isActive",
|
||||
header: "Status",
|
||||
cell: ({ row }) => (
|
||||
<Badge variant={row.original.isActive ? "default" : "secondary"}>
|
||||
{row.original.isActive ? "Active" : "Disabled"}
|
||||
</Badge>
|
||||
),
|
||||
},
|
||||
{
|
||||
id: "channels",
|
||||
header: "Channels",
|
||||
cell: ({ row }) => (
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
className="h-7 gap-1 px-2 text-xs"
|
||||
onClick={() => onViewLinks(row.original.id)}
|
||||
>
|
||||
<Link2 className="h-3 w-3" />
|
||||
{row.original.channelCount}
|
||||
</Button>
|
||||
),
|
||||
},
|
||||
{
|
||||
id: "runs",
|
||||
header: "Runs",
|
||||
cell: ({ row }) => (
|
||||
<span className="text-sm text-muted-foreground">
|
||||
{row.original.runCount}
|
||||
</span>
|
||||
),
|
||||
},
|
||||
{
|
||||
accessorKey: "lastSeenAt",
|
||||
header: "Last Seen",
|
||||
cell: ({ row }) =>
|
||||
row.original.lastSeenAt ? (
|
||||
<span className="text-sm text-muted-foreground">
|
||||
{new Date(row.original.lastSeenAt).toLocaleDateString()}
|
||||
</span>
|
||||
) : (
|
||||
<span className="text-sm text-muted-foreground">Never</span>
|
||||
),
|
||||
},
|
||||
{
|
||||
id: "actions",
|
||||
cell: ({ row }) => (
|
||||
<DropdownMenu>
|
||||
<DropdownMenuTrigger asChild>
|
||||
<Button variant="ghost" size="icon" className="h-8 w-8">
|
||||
<MoreHorizontal className="h-4 w-4" />
|
||||
</Button>
|
||||
</DropdownMenuTrigger>
|
||||
<DropdownMenuContent align="end">
|
||||
<DropdownMenuItem onClick={() => onEdit(row.original)}>
|
||||
<Pencil className="mr-2 h-3.5 w-3.5" />
|
||||
Edit
|
||||
</DropdownMenuItem>
|
||||
<DropdownMenuItem onClick={() => onViewLinks(row.original.id)}>
|
||||
<Link2 className="mr-2 h-3.5 w-3.5" />
|
||||
Manage Channels
|
||||
</DropdownMenuItem>
|
||||
<DropdownMenuItem onClick={() => onTriggerSync(row.original.id)}>
|
||||
<Play className="mr-2 h-3.5 w-3.5" />
|
||||
Sync Now
|
||||
</DropdownMenuItem>
|
||||
<DropdownMenuItem
|
||||
onClick={() => onToggleActive(row.original.id)}
|
||||
>
|
||||
<Power className="mr-2 h-3.5 w-3.5" />
|
||||
{row.original.isActive ? "Disable" : "Enable"}
|
||||
</DropdownMenuItem>
|
||||
<DropdownMenuSeparator />
|
||||
<DropdownMenuItem
|
||||
onClick={() => onDelete(row.original.id)}
|
||||
className="text-destructive focus:text-destructive"
|
||||
>
|
||||
<Trash2 className="mr-2 h-3.5 w-3.5" />
|
||||
Delete
|
||||
</DropdownMenuItem>
|
||||
</DropdownMenuContent>
|
||||
</DropdownMenu>
|
||||
),
|
||||
enableHiding: false,
|
||||
},
|
||||
];
|
||||
}
|
||||
102
src/app/(app)/telegram/_components/account-form.tsx
Normal file
102
src/app/(app)/telegram/_components/account-form.tsx
Normal file
@@ -0,0 +1,102 @@
|
||||
"use client";
|
||||
|
||||
import { useTransition } from "react";
|
||||
import { useForm } from "react-hook-form";
|
||||
import { zodResolver } from "@hookform/resolvers/zod";
|
||||
import { toast } from "sonner";
|
||||
import {
|
||||
telegramAccountSchema,
|
||||
type TelegramAccountInput,
|
||||
} from "@/schemas/telegram";
|
||||
import { createAccount, updateAccount } from "../actions";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import {
|
||||
Form,
|
||||
FormControl,
|
||||
FormDescription,
|
||||
FormField,
|
||||
FormItem,
|
||||
FormLabel,
|
||||
FormMessage,
|
||||
} from "@/components/ui/form";
|
||||
import type { AccountRow } from "@/lib/telegram/admin-queries";
|
||||
|
||||
interface AccountFormProps {
|
||||
account?: AccountRow;
|
||||
onSuccess: () => void;
|
||||
}
|
||||
|
||||
export function AccountForm({ account, onSuccess }: AccountFormProps) {
|
||||
const [isPending, startTransition] = useTransition();
|
||||
const isEditing = !!account;
|
||||
|
||||
const form = useForm<TelegramAccountInput>({
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
resolver: zodResolver(telegramAccountSchema) as any,
|
||||
defaultValues: {
|
||||
phone: account?.phone ?? "",
|
||||
displayName: account?.displayName ?? "",
|
||||
},
|
||||
});
|
||||
|
||||
function onSubmit(values: TelegramAccountInput) {
|
||||
startTransition(async () => {
|
||||
const result = isEditing
|
||||
? await updateAccount(account!.id, values)
|
||||
: await createAccount(values);
|
||||
|
||||
if (!result.success) {
|
||||
toast.error(result.error);
|
||||
return;
|
||||
}
|
||||
|
||||
toast.success(isEditing ? "Account updated" : "Account created");
|
||||
form.reset();
|
||||
onSuccess();
|
||||
});
|
||||
}
|
||||
|
||||
return (
|
||||
<Form {...form}>
|
||||
<form onSubmit={form.handleSubmit(onSubmit)} className="space-y-4">
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="phone"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>Phone Number</FormLabel>
|
||||
<FormControl>
|
||||
<Input placeholder="+31612345678" {...field} />
|
||||
</FormControl>
|
||||
<FormDescription>
|
||||
International format with country code
|
||||
</FormDescription>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="displayName"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>Display Name</FormLabel>
|
||||
<FormControl>
|
||||
<Input placeholder="My Bot Account" {...field} />
|
||||
</FormControl>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
|
||||
<div className="flex justify-end gap-2">
|
||||
<Button type="submit" disabled={isPending}>
|
||||
{isPending ? "Saving..." : isEditing ? "Update" : "Create"}
|
||||
</Button>
|
||||
</div>
|
||||
</form>
|
||||
</Form>
|
||||
);
|
||||
}
|
||||
233
src/app/(app)/telegram/_components/account-links-drawer.tsx
Normal file
233
src/app/(app)/telegram/_components/account-links-drawer.tsx
Normal file
@@ -0,0 +1,233 @@
|
||||
"use client";
|
||||
|
||||
import { useState, useEffect, useTransition, useCallback } from "react";
|
||||
import { Link2Off, Plus } from "lucide-react";
|
||||
import { toast } from "sonner";
|
||||
import { linkChannel, unlinkChannel } from "../actions";
|
||||
import {
|
||||
Dialog,
|
||||
DialogContent,
|
||||
DialogDescription,
|
||||
DialogHeader,
|
||||
DialogTitle,
|
||||
} from "@/components/ui/dialog";
|
||||
import {
|
||||
Select,
|
||||
SelectContent,
|
||||
SelectItem,
|
||||
SelectTrigger,
|
||||
SelectValue,
|
||||
} from "@/components/ui/select";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Separator } from "@/components/ui/separator";
|
||||
|
||||
interface ChannelLink {
|
||||
id: string;
|
||||
channelId: string;
|
||||
role: string;
|
||||
lastProcessedMessageId: string | null;
|
||||
channel: {
|
||||
id: string;
|
||||
title: string;
|
||||
type: string;
|
||||
telegramId: string;
|
||||
};
|
||||
}
|
||||
|
||||
interface UnlinkedChannel {
|
||||
id: string;
|
||||
title: string;
|
||||
type: string;
|
||||
telegramId: string;
|
||||
}
|
||||
|
||||
interface AccountLinksDrawerProps {
|
||||
accountId: string | null;
|
||||
open: boolean;
|
||||
onOpenChange: (open: boolean) => void;
|
||||
}
|
||||
|
||||
export function AccountLinksDrawer({
|
||||
accountId,
|
||||
open,
|
||||
onOpenChange,
|
||||
}: AccountLinksDrawerProps) {
|
||||
const [isPending, startTransition] = useTransition();
|
||||
const [links, setLinks] = useState<ChannelLink[]>([]);
|
||||
const [unlinked, setUnlinked] = useState<UnlinkedChannel[]>([]);
|
||||
const [selectedChannelId, setSelectedChannelId] = useState("");
|
||||
const [selectedRole, setSelectedRole] = useState<"READER" | "WRITER">("READER");
|
||||
const [loading, setLoading] = useState(false);
|
||||
|
||||
const fetchLinks = useCallback(async () => {
|
||||
if (!accountId) return;
|
||||
setLoading(true);
|
||||
try {
|
||||
const [linksRes, unlinkedRes] = await Promise.all([
|
||||
fetch(`/api/telegram/accounts/${accountId}/links`),
|
||||
fetch(`/api/telegram/accounts/${accountId}/unlinked-channels`),
|
||||
]);
|
||||
if (linksRes.ok) setLinks(await linksRes.json());
|
||||
if (unlinkedRes.ok) setUnlinked(await unlinkedRes.json());
|
||||
} catch {
|
||||
toast.error("Failed to load channel links");
|
||||
}
|
||||
setLoading(false);
|
||||
}, [accountId]);
|
||||
|
||||
useEffect(() => {
|
||||
if (open && accountId) {
|
||||
fetchLinks();
|
||||
}
|
||||
}, [open, accountId, fetchLinks]);
|
||||
|
||||
const handleLink = () => {
|
||||
if (!accountId || !selectedChannelId) return;
|
||||
startTransition(async () => {
|
||||
const result = await linkChannel({
|
||||
accountId,
|
||||
channelId: selectedChannelId,
|
||||
role: selectedRole,
|
||||
});
|
||||
if (result.success) {
|
||||
toast.success("Channel linked");
|
||||
setSelectedChannelId("");
|
||||
await fetchLinks();
|
||||
} else {
|
||||
toast.error(result.error);
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
const handleUnlink = (linkId: string) => {
|
||||
startTransition(async () => {
|
||||
const result = await unlinkChannel(linkId);
|
||||
if (result.success) {
|
||||
toast.success("Channel unlinked");
|
||||
await fetchLinks();
|
||||
} else {
|
||||
toast.error(result.error);
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
return (
|
||||
<Dialog open={open} onOpenChange={onOpenChange}>
|
||||
<DialogContent className="sm:max-w-lg">
|
||||
<DialogHeader>
|
||||
<DialogTitle>Manage Channel Links</DialogTitle>
|
||||
<DialogDescription>
|
||||
Link channels to this account. The account will read from Source
|
||||
channels and write to Destination channels.
|
||||
</DialogDescription>
|
||||
</DialogHeader>
|
||||
|
||||
{/* Add new link */}
|
||||
{unlinked.length > 0 && (
|
||||
<div className="space-y-3">
|
||||
<h4 className="text-sm font-medium">Link a Channel</h4>
|
||||
<div className="flex items-end gap-2">
|
||||
<div className="flex-1">
|
||||
<Select
|
||||
value={selectedChannelId}
|
||||
onValueChange={setSelectedChannelId}
|
||||
>
|
||||
<SelectTrigger>
|
||||
<SelectValue placeholder="Select channel" />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{unlinked.map((ch) => (
|
||||
<SelectItem key={ch.id} value={ch.id}>
|
||||
{ch.title} ({ch.type})
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</div>
|
||||
<Select
|
||||
value={selectedRole}
|
||||
onValueChange={(v) => setSelectedRole(v as "READER" | "WRITER")}
|
||||
>
|
||||
<SelectTrigger className="w-28">
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="READER">Reader</SelectItem>
|
||||
<SelectItem value="WRITER">Writer</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<Button
|
||||
size="sm"
|
||||
disabled={!selectedChannelId || isPending}
|
||||
onClick={handleLink}
|
||||
>
|
||||
<Plus className="mr-1 h-3.5 w-3.5" />
|
||||
Link
|
||||
</Button>
|
||||
</div>
|
||||
<Separator />
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Existing links */}
|
||||
<div className="space-y-2">
|
||||
<h4 className="text-sm font-medium">
|
||||
Linked Channels ({links.length})
|
||||
</h4>
|
||||
{loading ? (
|
||||
<p className="text-sm text-muted-foreground">Loading...</p>
|
||||
) : links.length === 0 ? (
|
||||
<p className="text-sm text-muted-foreground">
|
||||
No channels linked to this account.
|
||||
</p>
|
||||
) : (
|
||||
<div className="space-y-2">
|
||||
{links.map((link) => (
|
||||
<div
|
||||
key={link.id}
|
||||
className="flex items-center justify-between rounded-md border p-3"
|
||||
>
|
||||
<div className="flex flex-col gap-1">
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="text-sm font-medium">
|
||||
{link.channel.title}
|
||||
</span>
|
||||
<Badge
|
||||
variant="outline"
|
||||
className={
|
||||
link.channel.type === "SOURCE"
|
||||
? "bg-blue-500/10 text-blue-600 border-blue-500/20"
|
||||
: "bg-purple-500/10 text-purple-600 border-purple-500/20"
|
||||
}
|
||||
>
|
||||
{link.channel.type}
|
||||
</Badge>
|
||||
<Badge variant="secondary" className="text-[10px]">
|
||||
{link.role}
|
||||
</Badge>
|
||||
</div>
|
||||
<span className="text-xs text-muted-foreground">
|
||||
ID: {link.channel.telegramId}
|
||||
{link.lastProcessedMessageId &&
|
||||
` | Last msg: ${link.lastProcessedMessageId}`}
|
||||
</span>
|
||||
</div>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
className="h-8 w-8 text-muted-foreground hover:text-destructive"
|
||||
disabled={isPending}
|
||||
onClick={() => handleUnlink(link.id)}
|
||||
>
|
||||
<Link2Off className="h-4 w-4" />
|
||||
</Button>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
);
|
||||
}
|
||||
44
src/app/(app)/telegram/_components/account-modal.tsx
Normal file
44
src/app/(app)/telegram/_components/account-modal.tsx
Normal file
@@ -0,0 +1,44 @@
|
||||
"use client";
|
||||
|
||||
import {
|
||||
Dialog,
|
||||
DialogContent,
|
||||
DialogDescription,
|
||||
DialogHeader,
|
||||
DialogTitle,
|
||||
} from "@/components/ui/dialog";
|
||||
import { AccountForm } from "./account-form";
|
||||
import type { AccountRow } from "@/lib/telegram/admin-queries";
|
||||
|
||||
interface AccountModalProps {
|
||||
open: boolean;
|
||||
onOpenChange: (open: boolean) => void;
|
||||
account?: AccountRow;
|
||||
}
|
||||
|
||||
export function AccountModal({
|
||||
open,
|
||||
onOpenChange,
|
||||
account,
|
||||
}: AccountModalProps) {
|
||||
return (
|
||||
<Dialog open={open} onOpenChange={onOpenChange}>
|
||||
<DialogContent className="sm:max-w-lg">
|
||||
<DialogHeader>
|
||||
<DialogTitle>
|
||||
{account ? "Edit Account" : "Add Telegram Account"}
|
||||
</DialogTitle>
|
||||
<DialogDescription>
|
||||
{account
|
||||
? "Update the account details below."
|
||||
: "Configure a new Telegram account for ingestion. You'll need an API ID and hash from my.telegram.org."}
|
||||
</DialogDescription>
|
||||
</DialogHeader>
|
||||
<AccountForm
|
||||
account={account}
|
||||
onSuccess={() => onOpenChange(false)}
|
||||
/>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
);
|
||||
}
|
||||
140
src/app/(app)/telegram/_components/accounts-tab.tsx
Normal file
140
src/app/(app)/telegram/_components/accounts-tab.tsx
Normal file
@@ -0,0 +1,140 @@
|
||||
"use client";
|
||||
|
||||
import { useState, useTransition } from "react";
|
||||
import { Plus, Play } from "lucide-react";
|
||||
import { toast } from "sonner";
|
||||
import { getAccountColumns } from "./account-columns";
|
||||
import { AccountModal } from "./account-modal";
|
||||
import { AccountLinksDrawer } from "./account-links-drawer";
|
||||
import { AuthCodeDialog } from "./auth-code-dialog";
|
||||
import { deleteAccount, toggleAccountActive, triggerIngestion } from "../actions";
|
||||
import { DataTable } from "@/components/shared/data-table";
|
||||
import { DeleteDialog } from "@/components/shared/delete-dialog";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import type { AccountRow } from "@/lib/telegram/admin-queries";
|
||||
import { useDataTable } from "@/hooks/use-data-table";
|
||||
|
||||
interface AccountsTabProps {
|
||||
accounts: AccountRow[];
|
||||
}
|
||||
|
||||
export function AccountsTab({ accounts }: AccountsTabProps) {
|
||||
const [isPending, startTransition] = useTransition();
|
||||
const [modalOpen, setModalOpen] = useState(false);
|
||||
const [editAccount, setEditAccount] = useState<AccountRow | undefined>();
|
||||
const [deleteId, setDeleteId] = useState<string | null>(null);
|
||||
const [linksAccountId, setLinksAccountId] = useState<string | null>(null);
|
||||
const [authCodeAccount, setAuthCodeAccount] = useState<AccountRow | null>(null);
|
||||
|
||||
const columns = getAccountColumns({
|
||||
onEdit: (account) => {
|
||||
setEditAccount(account);
|
||||
setModalOpen(true);
|
||||
},
|
||||
onToggleActive: (id) => {
|
||||
startTransition(async () => {
|
||||
const result = await toggleAccountActive(id);
|
||||
if (result.success) toast.success("Account toggled");
|
||||
else toast.error(result.error);
|
||||
});
|
||||
},
|
||||
onDelete: (id) => setDeleteId(id),
|
||||
onViewLinks: (id) => setLinksAccountId(id),
|
||||
onEnterCode: (account) => setAuthCodeAccount(account),
|
||||
onTriggerSync: (id) => {
|
||||
startTransition(async () => {
|
||||
const result = await triggerIngestion(id);
|
||||
if (result.success) toast.success("Ingestion triggered");
|
||||
else toast.error(result.error);
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
const { table } = useDataTable({
|
||||
data: accounts,
|
||||
columns,
|
||||
pageCount: 1,
|
||||
});
|
||||
|
||||
const handleDelete = () => {
|
||||
if (!deleteId) return;
|
||||
startTransition(async () => {
|
||||
const result = await deleteAccount(deleteId);
|
||||
if (result.success) {
|
||||
toast.success("Account deleted");
|
||||
setDeleteId(null);
|
||||
} else {
|
||||
toast.error(result.error);
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="space-y-4">
|
||||
<div className="flex items-center gap-2">
|
||||
<Button
|
||||
onClick={() => {
|
||||
setEditAccount(undefined);
|
||||
setModalOpen(true);
|
||||
}}
|
||||
>
|
||||
<Plus className="mr-2 h-4 w-4" />
|
||||
Add Account
|
||||
</Button>
|
||||
<Button
|
||||
variant="outline"
|
||||
disabled={isPending}
|
||||
onClick={() => {
|
||||
startTransition(async () => {
|
||||
const result = await triggerIngestion();
|
||||
if (result.success) toast.success("Ingestion triggered for all accounts");
|
||||
else toast.error(result.error);
|
||||
});
|
||||
}}
|
||||
>
|
||||
<Play className="mr-2 h-4 w-4" />
|
||||
Sync All
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
<DataTable
|
||||
table={table}
|
||||
emptyMessage="No accounts configured. Add your first Telegram account."
|
||||
/>
|
||||
|
||||
<AccountModal
|
||||
open={modalOpen}
|
||||
onOpenChange={(open) => {
|
||||
setModalOpen(open);
|
||||
if (!open) setEditAccount(undefined);
|
||||
}}
|
||||
account={editAccount}
|
||||
/>
|
||||
|
||||
<DeleteDialog
|
||||
open={!!deleteId}
|
||||
onOpenChange={(open) => !open && setDeleteId(null)}
|
||||
title="Delete Account"
|
||||
description="This will permanently delete this Telegram account and all its channel links. Existing packages will NOT be deleted."
|
||||
onConfirm={handleDelete}
|
||||
isLoading={isPending}
|
||||
/>
|
||||
|
||||
<AccountLinksDrawer
|
||||
accountId={linksAccountId}
|
||||
open={!!linksAccountId}
|
||||
onOpenChange={(open) => {
|
||||
if (!open) setLinksAccountId(null);
|
||||
}}
|
||||
/>
|
||||
|
||||
<AuthCodeDialog
|
||||
account={authCodeAccount}
|
||||
open={!!authCodeAccount}
|
||||
onOpenChange={(open) => {
|
||||
if (!open) setAuthCodeAccount(null);
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
102
src/app/(app)/telegram/_components/auth-code-dialog.tsx
Normal file
102
src/app/(app)/telegram/_components/auth-code-dialog.tsx
Normal file
@@ -0,0 +1,102 @@
|
||||
"use client";
|
||||
|
||||
import { useState, useTransition } from "react";
|
||||
import { toast } from "sonner";
|
||||
import { submitAuthCode } from "../actions";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import {
|
||||
Dialog,
|
||||
DialogContent,
|
||||
DialogDescription,
|
||||
DialogFooter,
|
||||
DialogHeader,
|
||||
DialogTitle,
|
||||
} from "@/components/ui/dialog";
|
||||
import { Label } from "@/components/ui/label";
|
||||
import type { AccountRow } from "@/lib/telegram/admin-queries";
|
||||
|
||||
interface AuthCodeDialogProps {
|
||||
account: AccountRow | null;
|
||||
open: boolean;
|
||||
onOpenChange: (open: boolean) => void;
|
||||
}
|
||||
|
||||
export function AuthCodeDialog({
|
||||
account,
|
||||
open,
|
||||
onOpenChange,
|
||||
}: AuthCodeDialogProps) {
|
||||
const [code, setCode] = useState("");
|
||||
const [isPending, startTransition] = useTransition();
|
||||
|
||||
const isPassword = account?.authState === "AWAITING_PASSWORD";
|
||||
const title = isPassword ? "Enter 2FA Password" : "Enter Auth Code";
|
||||
const description = isPassword
|
||||
? "Your Telegram account requires a two-factor authentication password."
|
||||
: "Enter the code sent to your Telegram app or SMS.";
|
||||
const placeholder = isPassword ? "Password" : "12345";
|
||||
|
||||
function handleSubmit() {
|
||||
if (!account || !code.trim()) return;
|
||||
|
||||
startTransition(async () => {
|
||||
const result = await submitAuthCode(account.id, { code: code.trim() });
|
||||
if (result.success) {
|
||||
toast.success(isPassword ? "Password submitted" : "Code submitted");
|
||||
setCode("");
|
||||
onOpenChange(false);
|
||||
} else {
|
||||
toast.error(result.error);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
return (
|
||||
<Dialog
|
||||
open={open}
|
||||
onOpenChange={(v) => {
|
||||
if (!v) setCode("");
|
||||
onOpenChange(v);
|
||||
}}
|
||||
>
|
||||
<DialogContent className="sm:max-w-md">
|
||||
<DialogHeader>
|
||||
<DialogTitle>{title}</DialogTitle>
|
||||
<DialogDescription>{description}</DialogDescription>
|
||||
</DialogHeader>
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor="auth-code">
|
||||
{isPassword ? "Password" : "Code"}
|
||||
</Label>
|
||||
<Input
|
||||
id="auth-code"
|
||||
type={isPassword ? "password" : "text"}
|
||||
placeholder={placeholder}
|
||||
value={code}
|
||||
onChange={(e) => setCode(e.target.value)}
|
||||
onKeyDown={(e) => {
|
||||
if (e.key === "Enter") handleSubmit();
|
||||
}}
|
||||
autoFocus
|
||||
/>
|
||||
</div>
|
||||
<DialogFooter>
|
||||
<Button
|
||||
variant="outline"
|
||||
onClick={() => onOpenChange(false)}
|
||||
disabled={isPending}
|
||||
>
|
||||
Cancel
|
||||
</Button>
|
||||
<Button
|
||||
onClick={handleSubmit}
|
||||
disabled={isPending || !code.trim()}
|
||||
>
|
||||
{isPending ? "Submitting..." : "Submit"}
|
||||
</Button>
|
||||
</DialogFooter>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
);
|
||||
}
|
||||
132
src/app/(app)/telegram/_components/channel-columns.tsx
Normal file
132
src/app/(app)/telegram/_components/channel-columns.tsx
Normal file
@@ -0,0 +1,132 @@
|
||||
"use client";
|
||||
|
||||
import { type ColumnDef } from "@tanstack/react-table";
|
||||
import {
|
||||
MoreHorizontal,
|
||||
Pencil,
|
||||
Trash2,
|
||||
Power,
|
||||
} from "lucide-react";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import {
|
||||
DropdownMenu,
|
||||
DropdownMenuContent,
|
||||
DropdownMenuItem,
|
||||
DropdownMenuSeparator,
|
||||
DropdownMenuTrigger,
|
||||
} from "@/components/ui/dropdown-menu";
|
||||
import type { ChannelRow } from "@/lib/telegram/admin-queries";
|
||||
|
||||
interface ChannelColumnsProps {
|
||||
onEdit: (channel: ChannelRow) => void;
|
||||
onToggleActive: (id: string) => void;
|
||||
onDelete: (id: string) => void;
|
||||
}
|
||||
|
||||
export function getChannelColumns({
|
||||
onEdit,
|
||||
onToggleActive,
|
||||
onDelete,
|
||||
}: ChannelColumnsProps): ColumnDef<ChannelRow, unknown>[] {
|
||||
return [
|
||||
{
|
||||
accessorKey: "title",
|
||||
header: "Channel",
|
||||
cell: ({ row }) => (
|
||||
<div className="flex flex-col">
|
||||
<span className="font-medium">{row.original.title}</span>
|
||||
<span className="text-xs text-muted-foreground">
|
||||
ID: {row.original.telegramId}
|
||||
</span>
|
||||
</div>
|
||||
),
|
||||
enableHiding: false,
|
||||
},
|
||||
{
|
||||
accessorKey: "type",
|
||||
header: "Type",
|
||||
cell: ({ row }) => (
|
||||
<Badge
|
||||
variant="outline"
|
||||
className={
|
||||
row.original.type === "SOURCE"
|
||||
? "bg-blue-500/10 text-blue-600 border-blue-500/20"
|
||||
: "bg-purple-500/10 text-purple-600 border-purple-500/20"
|
||||
}
|
||||
>
|
||||
{row.original.type}
|
||||
</Badge>
|
||||
),
|
||||
},
|
||||
{
|
||||
accessorKey: "isActive",
|
||||
header: "Status",
|
||||
cell: ({ row }) => (
|
||||
<Badge variant={row.original.isActive ? "default" : "secondary"}>
|
||||
{row.original.isActive ? "Active" : "Disabled"}
|
||||
</Badge>
|
||||
),
|
||||
},
|
||||
{
|
||||
id: "accounts",
|
||||
header: "Accounts",
|
||||
cell: ({ row }) => (
|
||||
<span className="text-sm text-muted-foreground">
|
||||
{row.original.accountCount}
|
||||
</span>
|
||||
),
|
||||
},
|
||||
{
|
||||
id: "packages",
|
||||
header: "Packages",
|
||||
cell: ({ row }) => (
|
||||
<span className="text-sm text-muted-foreground">
|
||||
{row.original.packageCount}
|
||||
</span>
|
||||
),
|
||||
},
|
||||
{
|
||||
accessorKey: "createdAt",
|
||||
header: "Created",
|
||||
cell: ({ row }) => (
|
||||
<span className="text-sm text-muted-foreground">
|
||||
{new Date(row.original.createdAt).toLocaleDateString()}
|
||||
</span>
|
||||
),
|
||||
},
|
||||
{
|
||||
id: "actions",
|
||||
cell: ({ row }) => (
|
||||
<DropdownMenu>
|
||||
<DropdownMenuTrigger asChild>
|
||||
<Button variant="ghost" size="icon" className="h-8 w-8">
|
||||
<MoreHorizontal className="h-4 w-4" />
|
||||
</Button>
|
||||
</DropdownMenuTrigger>
|
||||
<DropdownMenuContent align="end">
|
||||
<DropdownMenuItem onClick={() => onEdit(row.original)}>
|
||||
<Pencil className="mr-2 h-3.5 w-3.5" />
|
||||
Edit
|
||||
</DropdownMenuItem>
|
||||
<DropdownMenuItem
|
||||
onClick={() => onToggleActive(row.original.id)}
|
||||
>
|
||||
<Power className="mr-2 h-3.5 w-3.5" />
|
||||
{row.original.isActive ? "Disable" : "Enable"}
|
||||
</DropdownMenuItem>
|
||||
<DropdownMenuSeparator />
|
||||
<DropdownMenuItem
|
||||
onClick={() => onDelete(row.original.id)}
|
||||
className="text-destructive focus:text-destructive"
|
||||
>
|
||||
<Trash2 className="mr-2 h-3.5 w-3.5" />
|
||||
Delete
|
||||
</DropdownMenuItem>
|
||||
</DropdownMenuContent>
|
||||
</DropdownMenu>
|
||||
),
|
||||
enableHiding: false,
|
||||
},
|
||||
];
|
||||
}
|
||||
142
src/app/(app)/telegram/_components/channel-form.tsx
Normal file
142
src/app/(app)/telegram/_components/channel-form.tsx
Normal file
@@ -0,0 +1,142 @@
|
||||
"use client";
|
||||
|
||||
import { useTransition } from "react";
|
||||
import { useForm } from "react-hook-form";
|
||||
import { zodResolver } from "@hookform/resolvers/zod";
|
||||
import { toast } from "sonner";
|
||||
import {
|
||||
telegramChannelSchema,
|
||||
type TelegramChannelInput,
|
||||
} from "@/schemas/telegram";
|
||||
import { createChannel, updateChannel } from "../actions";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import {
|
||||
Select,
|
||||
SelectContent,
|
||||
SelectItem,
|
||||
SelectTrigger,
|
||||
SelectValue,
|
||||
} from "@/components/ui/select";
|
||||
import {
|
||||
Form,
|
||||
FormControl,
|
||||
FormDescription,
|
||||
FormField,
|
||||
FormItem,
|
||||
FormLabel,
|
||||
FormMessage,
|
||||
} from "@/components/ui/form";
|
||||
import type { ChannelRow } from "@/lib/telegram/admin-queries";
|
||||
|
||||
interface ChannelFormProps {
|
||||
channel?: ChannelRow;
|
||||
onSuccess: () => void;
|
||||
}
|
||||
|
||||
export function ChannelForm({ channel, onSuccess }: ChannelFormProps) {
|
||||
const [isPending, startTransition] = useTransition();
|
||||
const isEditing = !!channel;
|
||||
|
||||
const form = useForm<TelegramChannelInput>({
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
resolver: zodResolver(telegramChannelSchema) as any,
|
||||
defaultValues: {
|
||||
telegramId: channel ? Number(channel.telegramId) : (0 as unknown as number),
|
||||
title: channel?.title ?? "",
|
||||
type: channel?.type ?? "SOURCE",
|
||||
},
|
||||
});
|
||||
|
||||
function onSubmit(values: TelegramChannelInput) {
|
||||
startTransition(async () => {
|
||||
const result = isEditing
|
||||
? await updateChannel(channel!.id, values)
|
||||
: await createChannel(values);
|
||||
|
||||
if (!result.success) {
|
||||
toast.error(result.error);
|
||||
return;
|
||||
}
|
||||
|
||||
toast.success(isEditing ? "Channel updated" : "Channel created");
|
||||
form.reset();
|
||||
onSuccess();
|
||||
});
|
||||
}
|
||||
|
||||
return (
|
||||
<Form {...form}>
|
||||
<form onSubmit={form.handleSubmit(onSubmit)} className="space-y-4">
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="title"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>Title</FormLabel>
|
||||
<FormControl>
|
||||
<Input placeholder="Channel name" {...field} />
|
||||
</FormControl>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="telegramId"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>Telegram ID</FormLabel>
|
||||
<FormControl>
|
||||
<Input
|
||||
type="number"
|
||||
placeholder="1234567890"
|
||||
{...field}
|
||||
value={field.value || ""}
|
||||
/>
|
||||
</FormControl>
|
||||
<FormDescription>
|
||||
Numeric ID of the Telegram channel or group
|
||||
</FormDescription>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="type"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>Type</FormLabel>
|
||||
<Select
|
||||
onValueChange={field.onChange}
|
||||
defaultValue={field.value}
|
||||
>
|
||||
<FormControl>
|
||||
<SelectTrigger>
|
||||
<SelectValue placeholder="Select type" />
|
||||
</SelectTrigger>
|
||||
</FormControl>
|
||||
<SelectContent>
|
||||
<SelectItem value="SOURCE">Source (read archives)</SelectItem>
|
||||
<SelectItem value="DESTINATION">
|
||||
Destination (forward indexed)
|
||||
</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
|
||||
<div className="flex justify-end gap-2">
|
||||
<Button type="submit" disabled={isPending}>
|
||||
{isPending ? "Saving..." : isEditing ? "Update" : "Create"}
|
||||
</Button>
|
||||
</div>
|
||||
</form>
|
||||
</Form>
|
||||
);
|
||||
}
|
||||
44
src/app/(app)/telegram/_components/channel-modal.tsx
Normal file
44
src/app/(app)/telegram/_components/channel-modal.tsx
Normal file
@@ -0,0 +1,44 @@
|
||||
"use client";
|
||||
|
||||
import {
|
||||
Dialog,
|
||||
DialogContent,
|
||||
DialogDescription,
|
||||
DialogHeader,
|
||||
DialogTitle,
|
||||
} from "@/components/ui/dialog";
|
||||
import { ChannelForm } from "./channel-form";
|
||||
import type { ChannelRow } from "@/lib/telegram/admin-queries";
|
||||
|
||||
interface ChannelModalProps {
|
||||
open: boolean;
|
||||
onOpenChange: (open: boolean) => void;
|
||||
channel?: ChannelRow;
|
||||
}
|
||||
|
||||
export function ChannelModal({
|
||||
open,
|
||||
onOpenChange,
|
||||
channel,
|
||||
}: ChannelModalProps) {
|
||||
return (
|
||||
<Dialog open={open} onOpenChange={onOpenChange}>
|
||||
<DialogContent className="sm:max-w-md">
|
||||
<DialogHeader>
|
||||
<DialogTitle>
|
||||
{channel ? "Edit Channel" : "Add Channel"}
|
||||
</DialogTitle>
|
||||
<DialogDescription>
|
||||
{channel
|
||||
? "Update the channel details below."
|
||||
: "Add a Telegram channel. Source channels are scanned for archives, destination channels receive indexed files."}
|
||||
</DialogDescription>
|
||||
</DialogHeader>
|
||||
<ChannelForm
|
||||
channel={channel}
|
||||
onSuccess={() => onOpenChange(false)}
|
||||
/>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
);
|
||||
}
|
||||
97
src/app/(app)/telegram/_components/channels-tab.tsx
Normal file
97
src/app/(app)/telegram/_components/channels-tab.tsx
Normal file
@@ -0,0 +1,97 @@
|
||||
"use client";
|
||||
|
||||
import { useState, useTransition } from "react";
|
||||
import { Plus } from "lucide-react";
|
||||
import { toast } from "sonner";
|
||||
import { getChannelColumns } from "./channel-columns";
|
||||
import { ChannelModal } from "./channel-modal";
|
||||
import { deleteChannel, toggleChannelActive } from "../actions";
|
||||
import { DataTable } from "@/components/shared/data-table";
|
||||
import { DeleteDialog } from "@/components/shared/delete-dialog";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import type { ChannelRow } from "@/lib/telegram/admin-queries";
|
||||
import { useDataTable } from "@/hooks/use-data-table";
|
||||
|
||||
interface ChannelsTabProps {
|
||||
channels: ChannelRow[];
|
||||
}
|
||||
|
||||
export function ChannelsTab({ channels }: ChannelsTabProps) {
|
||||
const [isPending, startTransition] = useTransition();
|
||||
const [modalOpen, setModalOpen] = useState(false);
|
||||
const [editChannel, setEditChannel] = useState<ChannelRow | undefined>();
|
||||
const [deleteId, setDeleteId] = useState<string | null>(null);
|
||||
|
||||
const columns = getChannelColumns({
|
||||
onEdit: (channel) => {
|
||||
setEditChannel(channel);
|
||||
setModalOpen(true);
|
||||
},
|
||||
onToggleActive: (id) => {
|
||||
startTransition(async () => {
|
||||
const result = await toggleChannelActive(id);
|
||||
if (result.success) toast.success("Channel toggled");
|
||||
else toast.error(result.error);
|
||||
});
|
||||
},
|
||||
onDelete: (id) => setDeleteId(id),
|
||||
});
|
||||
|
||||
const { table } = useDataTable({
|
||||
data: channels,
|
||||
columns,
|
||||
pageCount: 1,
|
||||
});
|
||||
|
||||
const handleDelete = () => {
|
||||
if (!deleteId) return;
|
||||
startTransition(async () => {
|
||||
const result = await deleteChannel(deleteId);
|
||||
if (result.success) {
|
||||
toast.success("Channel deleted");
|
||||
setDeleteId(null);
|
||||
} else {
|
||||
toast.error(result.error);
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="space-y-4">
|
||||
<div className="flex items-center gap-2">
|
||||
<Button
|
||||
onClick={() => {
|
||||
setEditChannel(undefined);
|
||||
setModalOpen(true);
|
||||
}}
|
||||
>
|
||||
<Plus className="mr-2 h-4 w-4" />
|
||||
Add Channel
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
<DataTable
|
||||
table={table}
|
||||
emptyMessage="No channels configured. Add a Telegram channel to start ingesting."
|
||||
/>
|
||||
|
||||
<ChannelModal
|
||||
open={modalOpen}
|
||||
onOpenChange={(open) => {
|
||||
setModalOpen(open);
|
||||
if (!open) setEditChannel(undefined);
|
||||
}}
|
||||
channel={editChannel}
|
||||
/>
|
||||
|
||||
<DeleteDialog
|
||||
open={!!deleteId}
|
||||
onOpenChange={(open) => !open && setDeleteId(null)}
|
||||
title="Delete Channel"
|
||||
description="This will permanently delete this channel and unlink it from all accounts. Existing packages will NOT be deleted."
|
||||
onConfirm={handleDelete}
|
||||
isLoading={isPending}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
41
src/app/(app)/telegram/_components/telegram-admin.tsx
Normal file
41
src/app/(app)/telegram/_components/telegram-admin.tsx
Normal file
@@ -0,0 +1,41 @@
|
||||
"use client";
|
||||
|
||||
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
|
||||
import { PageHeader } from "@/components/shared/page-header";
|
||||
import { AccountsTab } from "./accounts-tab";
|
||||
import { ChannelsTab } from "./channels-tab";
|
||||
import type { AccountRow, ChannelRow } from "@/lib/telegram/admin-queries";
|
||||
|
||||
interface TelegramAdminProps {
|
||||
accounts: AccountRow[];
|
||||
channels: ChannelRow[];
|
||||
}
|
||||
|
||||
export function TelegramAdmin({ accounts, channels }: TelegramAdminProps) {
|
||||
return (
|
||||
<div className="space-y-4">
|
||||
<PageHeader
|
||||
title="Telegram"
|
||||
description="Manage Telegram accounts, channels, and ingestion"
|
||||
/>
|
||||
|
||||
<Tabs defaultValue="accounts" className="space-y-4">
|
||||
<TabsList>
|
||||
<TabsTrigger value="accounts">
|
||||
Accounts ({accounts.length})
|
||||
</TabsTrigger>
|
||||
<TabsTrigger value="channels">
|
||||
Channels ({channels.length})
|
||||
</TabsTrigger>
|
||||
</TabsList>
|
||||
|
||||
<TabsContent value="accounts">
|
||||
<AccountsTab accounts={accounts} />
|
||||
</TabsContent>
|
||||
<TabsContent value="channels">
|
||||
<ChannelsTab channels={channels} />
|
||||
</TabsContent>
|
||||
</Tabs>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
345
src/app/(app)/telegram/actions.ts
Normal file
345
src/app/(app)/telegram/actions.ts
Normal file
@@ -0,0 +1,345 @@
|
||||
"use server";
|
||||
|
||||
import { auth } from "@/lib/auth";
|
||||
import { prisma } from "@/lib/prisma";
|
||||
import { revalidatePath } from "next/cache";
|
||||
import type { ActionResult } from "@/types/api.types";
|
||||
import {
|
||||
telegramAccountSchema,
|
||||
telegramChannelSchema,
|
||||
linkChannelSchema,
|
||||
submitAuthCodeSchema,
|
||||
} from "@/schemas/telegram";
|
||||
|
||||
const REVALIDATE_PATH = "/telegram";
|
||||
|
||||
async function requireAdmin(): Promise<
|
||||
{ success: true; userId: string } | { success: false; error: string }
|
||||
> {
|
||||
const session = await auth();
|
||||
if (!session?.user?.id) return { success: false, error: "Unauthorized" };
|
||||
if (session.user.role !== "ADMIN")
|
||||
return { success: false, error: "Admin access required" };
|
||||
return { success: true, userId: session.user.id };
|
||||
}
|
||||
|
||||
// ── Account actions ──
|
||||
|
||||
export async function createAccount(
|
||||
input: unknown
|
||||
): Promise<ActionResult<{ id: string }>> {
|
||||
const admin = await requireAdmin();
|
||||
if (!admin.success) return admin;
|
||||
|
||||
const parsed = telegramAccountSchema.safeParse(input);
|
||||
if (!parsed.success) return { success: false, error: "Validation failed" };
|
||||
|
||||
try {
|
||||
const account = await prisma.telegramAccount.create({
|
||||
data: {
|
||||
phone: parsed.data.phone.replace(/[\s\-]/g, ""),
|
||||
displayName: parsed.data.displayName || null,
|
||||
},
|
||||
});
|
||||
revalidatePath(REVALIDATE_PATH);
|
||||
return { success: true, data: { id: account.id } };
|
||||
} catch (err: unknown) {
|
||||
if (
|
||||
err instanceof Error &&
|
||||
err.message.includes("Unique constraint failed")
|
||||
) {
|
||||
return { success: false, error: "Phone number already registered" };
|
||||
}
|
||||
return { success: false, error: "Failed to create account" };
|
||||
}
|
||||
}
|
||||
|
||||
export async function updateAccount(
|
||||
id: string,
|
||||
input: unknown
|
||||
): Promise<ActionResult> {
|
||||
const admin = await requireAdmin();
|
||||
if (!admin.success) return admin;
|
||||
|
||||
const parsed = telegramAccountSchema.safeParse(input);
|
||||
if (!parsed.success) return { success: false, error: "Validation failed" };
|
||||
|
||||
const existing = await prisma.telegramAccount.findUnique({ where: { id } });
|
||||
if (!existing) return { success: false, error: "Account not found" };
|
||||
|
||||
try {
|
||||
await prisma.telegramAccount.update({
|
||||
where: { id },
|
||||
data: {
|
||||
phone: parsed.data.phone.replace(/[\s\-]/g, ""),
|
||||
displayName: parsed.data.displayName || null,
|
||||
},
|
||||
});
|
||||
revalidatePath(REVALIDATE_PATH);
|
||||
return { success: true, data: undefined };
|
||||
} catch (err: unknown) {
|
||||
if (
|
||||
err instanceof Error &&
|
||||
err.message.includes("Unique constraint failed")
|
||||
) {
|
||||
return { success: false, error: "Phone number already registered" };
|
||||
}
|
||||
return { success: false, error: "Failed to update account" };
|
||||
}
|
||||
}
|
||||
|
||||
export async function toggleAccountActive(id: string): Promise<ActionResult> {
|
||||
const admin = await requireAdmin();
|
||||
if (!admin.success) return admin;
|
||||
|
||||
const existing = await prisma.telegramAccount.findUnique({ where: { id } });
|
||||
if (!existing) return { success: false, error: "Account not found" };
|
||||
|
||||
try {
|
||||
await prisma.telegramAccount.update({
|
||||
where: { id },
|
||||
data: { isActive: !existing.isActive },
|
||||
});
|
||||
revalidatePath(REVALIDATE_PATH);
|
||||
return { success: true, data: undefined };
|
||||
} catch {
|
||||
return { success: false, error: "Failed to toggle account" };
|
||||
}
|
||||
}
|
||||
|
||||
export async function deleteAccount(id: string): Promise<ActionResult> {
|
||||
const admin = await requireAdmin();
|
||||
if (!admin.success) return admin;
|
||||
|
||||
const existing = await prisma.telegramAccount.findUnique({ where: { id } });
|
||||
if (!existing) return { success: false, error: "Account not found" };
|
||||
|
||||
try {
|
||||
await prisma.telegramAccount.delete({ where: { id } });
|
||||
revalidatePath(REVALIDATE_PATH);
|
||||
return { success: true, data: undefined };
|
||||
} catch {
|
||||
return { success: false, error: "Failed to delete account" };
|
||||
}
|
||||
}
|
||||
|
||||
export async function submitAuthCode(
|
||||
accountId: string,
|
||||
input: unknown
|
||||
): Promise<ActionResult> {
|
||||
const admin = await requireAdmin();
|
||||
if (!admin.success) return admin;
|
||||
|
||||
const parsed = submitAuthCodeSchema.safeParse(input);
|
||||
if (!parsed.success) return { success: false, error: "Validation failed" };
|
||||
|
||||
const existing = await prisma.telegramAccount.findUnique({
|
||||
where: { id: accountId },
|
||||
});
|
||||
if (!existing) return { success: false, error: "Account not found" };
|
||||
if (
|
||||
existing.authState !== "AWAITING_CODE" &&
|
||||
existing.authState !== "AWAITING_PASSWORD"
|
||||
) {
|
||||
return { success: false, error: "Account is not waiting for a code" };
|
||||
}
|
||||
|
||||
try {
|
||||
await prisma.telegramAccount.update({
|
||||
where: { id: accountId },
|
||||
data: { authCode: parsed.data.code },
|
||||
});
|
||||
revalidatePath(REVALIDATE_PATH);
|
||||
return { success: true, data: undefined };
|
||||
} catch {
|
||||
return { success: false, error: "Failed to submit code" };
|
||||
}
|
||||
}
|
||||
|
||||
// ── Channel actions ──
|
||||
|
||||
export async function createChannel(
|
||||
input: unknown
|
||||
): Promise<ActionResult<{ id: string }>> {
|
||||
const admin = await requireAdmin();
|
||||
if (!admin.success) return admin;
|
||||
|
||||
const parsed = telegramChannelSchema.safeParse(input);
|
||||
if (!parsed.success) return { success: false, error: "Validation failed" };
|
||||
|
||||
try {
|
||||
const channel = await prisma.telegramChannel.create({
|
||||
data: {
|
||||
telegramId: BigInt(parsed.data.telegramId),
|
||||
title: parsed.data.title,
|
||||
type: parsed.data.type,
|
||||
},
|
||||
});
|
||||
revalidatePath(REVALIDATE_PATH);
|
||||
return { success: true, data: { id: channel.id } };
|
||||
} catch (err: unknown) {
|
||||
if (
|
||||
err instanceof Error &&
|
||||
err.message.includes("Unique constraint failed")
|
||||
) {
|
||||
return { success: false, error: "Channel with this Telegram ID already exists" };
|
||||
}
|
||||
return { success: false, error: "Failed to create channel" };
|
||||
}
|
||||
}
|
||||
|
||||
export async function updateChannel(
|
||||
id: string,
|
||||
input: unknown
|
||||
): Promise<ActionResult> {
|
||||
const admin = await requireAdmin();
|
||||
if (!admin.success) return admin;
|
||||
|
||||
const parsed = telegramChannelSchema.safeParse(input);
|
||||
if (!parsed.success) return { success: false, error: "Validation failed" };
|
||||
|
||||
const existing = await prisma.telegramChannel.findUnique({ where: { id } });
|
||||
if (!existing) return { success: false, error: "Channel not found" };
|
||||
|
||||
try {
|
||||
await prisma.telegramChannel.update({
|
||||
where: { id },
|
||||
data: {
|
||||
telegramId: BigInt(parsed.data.telegramId),
|
||||
title: parsed.data.title,
|
||||
type: parsed.data.type,
|
||||
},
|
||||
});
|
||||
revalidatePath(REVALIDATE_PATH);
|
||||
return { success: true, data: undefined };
|
||||
} catch (err: unknown) {
|
||||
if (
|
||||
err instanceof Error &&
|
||||
err.message.includes("Unique constraint failed")
|
||||
) {
|
||||
return { success: false, error: "Channel with this Telegram ID already exists" };
|
||||
}
|
||||
return { success: false, error: "Failed to update channel" };
|
||||
}
|
||||
}
|
||||
|
||||
export async function toggleChannelActive(id: string): Promise<ActionResult> {
|
||||
const admin = await requireAdmin();
|
||||
if (!admin.success) return admin;
|
||||
|
||||
const existing = await prisma.telegramChannel.findUnique({ where: { id } });
|
||||
if (!existing) return { success: false, error: "Channel not found" };
|
||||
|
||||
try {
|
||||
await prisma.telegramChannel.update({
|
||||
where: { id },
|
||||
data: { isActive: !existing.isActive },
|
||||
});
|
||||
revalidatePath(REVALIDATE_PATH);
|
||||
return { success: true, data: undefined };
|
||||
} catch {
|
||||
return { success: false, error: "Failed to toggle channel" };
|
||||
}
|
||||
}
|
||||
|
||||
export async function deleteChannel(id: string): Promise<ActionResult> {
|
||||
const admin = await requireAdmin();
|
||||
if (!admin.success) return admin;
|
||||
|
||||
const existing = await prisma.telegramChannel.findUnique({ where: { id } });
|
||||
if (!existing) return { success: false, error: "Channel not found" };
|
||||
|
||||
try {
|
||||
await prisma.telegramChannel.delete({ where: { id } });
|
||||
revalidatePath(REVALIDATE_PATH);
|
||||
return { success: true, data: undefined };
|
||||
} catch {
|
||||
return { success: false, error: "Failed to delete channel" };
|
||||
}
|
||||
}
|
||||
|
||||
// ── Account-Channel link actions ──
|
||||
|
||||
export async function linkChannel(
|
||||
input: unknown
|
||||
): Promise<ActionResult<{ id: string }>> {
|
||||
const admin = await requireAdmin();
|
||||
if (!admin.success) return admin;
|
||||
|
||||
const parsed = linkChannelSchema.safeParse(input);
|
||||
if (!parsed.success) return { success: false, error: "Validation failed" };
|
||||
|
||||
try {
|
||||
const link = await prisma.accountChannelMap.create({
|
||||
data: {
|
||||
accountId: parsed.data.accountId,
|
||||
channelId: parsed.data.channelId,
|
||||
role: parsed.data.role,
|
||||
},
|
||||
});
|
||||
revalidatePath(REVALIDATE_PATH);
|
||||
return { success: true, data: { id: link.id } };
|
||||
} catch (err: unknown) {
|
||||
if (
|
||||
err instanceof Error &&
|
||||
err.message.includes("Unique constraint failed")
|
||||
) {
|
||||
return { success: false, error: "This channel is already linked to this account" };
|
||||
}
|
||||
return { success: false, error: "Failed to link channel" };
|
||||
}
|
||||
}
|
||||
|
||||
export async function unlinkChannel(id: string): Promise<ActionResult> {
|
||||
const admin = await requireAdmin();
|
||||
if (!admin.success) return admin;
|
||||
|
||||
const existing = await prisma.accountChannelMap.findUnique({
|
||||
where: { id },
|
||||
});
|
||||
if (!existing) return { success: false, error: "Link not found" };
|
||||
|
||||
try {
|
||||
await prisma.accountChannelMap.delete({ where: { id } });
|
||||
revalidatePath(REVALIDATE_PATH);
|
||||
return { success: true, data: undefined };
|
||||
} catch {
|
||||
return { success: false, error: "Failed to unlink channel" };
|
||||
}
|
||||
}
|
||||
|
||||
// ── Ingestion trigger ──
|
||||
|
||||
export async function triggerIngestion(
|
||||
accountId?: string
|
||||
): Promise<ActionResult> {
|
||||
const admin = await requireAdmin();
|
||||
if (!admin.success) return admin;
|
||||
|
||||
try {
|
||||
const res = await fetch(
|
||||
`${process.env.NEXT_PUBLIC_APP_URL || "http://localhost:3000"}/api/ingestion/trigger`,
|
||||
{
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
"X-API-Key": process.env.INGESTION_API_KEY || "",
|
||||
},
|
||||
body: JSON.stringify({ accountId }),
|
||||
}
|
||||
);
|
||||
|
||||
if (!res.ok) {
|
||||
const data = await res.json().catch(() => ({}));
|
||||
return {
|
||||
success: false,
|
||||
error: (data as { error?: string }).error || "Failed to trigger ingestion",
|
||||
};
|
||||
}
|
||||
|
||||
revalidatePath(REVALIDATE_PATH);
|
||||
return { success: true, data: undefined };
|
||||
} catch {
|
||||
return { success: false, error: "Failed to trigger ingestion" };
|
||||
}
|
||||
}
|
||||
17
src/app/(app)/telegram/page.tsx
Normal file
17
src/app/(app)/telegram/page.tsx
Normal file
@@ -0,0 +1,17 @@
|
||||
import { auth } from "@/lib/auth";
|
||||
import { redirect } from "next/navigation";
|
||||
import { listAccounts, listChannels } from "@/lib/telegram/admin-queries";
|
||||
import { TelegramAdmin } from "./_components/telegram-admin";
|
||||
|
||||
export default async function TelegramPage() {
|
||||
const session = await auth();
|
||||
if (!session?.user?.id) redirect("/login");
|
||||
if (session.user.role !== "ADMIN") redirect("/dashboard");
|
||||
|
||||
const [accounts, channels] = await Promise.all([
|
||||
listAccounts(),
|
||||
listChannels(),
|
||||
]);
|
||||
|
||||
return <TelegramAdmin accounts={accounts} channels={channels} />;
|
||||
}
|
||||
13
src/app/api/ingestion/status/route.ts
Normal file
13
src/app/api/ingestion/status/route.ts
Normal file
@@ -0,0 +1,13 @@
|
||||
import { NextResponse } from "next/server";
|
||||
import { authenticateApiRequest } from "@/lib/telegram/api-auth";
|
||||
import { getIngestionStatus } from "@/lib/telegram/queries";
|
||||
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
export async function GET(request: Request) {
|
||||
const authResult = await authenticateApiRequest(request);
|
||||
if ("error" in authResult) return authResult.error;
|
||||
|
||||
const accounts = await getIngestionStatus();
|
||||
return NextResponse.json({ accounts });
|
||||
}
|
||||
77
src/app/api/ingestion/trigger/route.ts
Normal file
77
src/app/api/ingestion/trigger/route.ts
Normal file
@@ -0,0 +1,77 @@
|
||||
import { NextResponse } from "next/server";
|
||||
import { authenticateApiRequest } from "@/lib/telegram/api-auth";
|
||||
import { triggerIngestionSchema } from "@/schemas/telegram";
|
||||
import { prisma } from "@/lib/prisma";
|
||||
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
export async function POST(request: Request) {
|
||||
const authResult = await authenticateApiRequest(request, true);
|
||||
if ("error" in authResult) return authResult.error;
|
||||
|
||||
let body: unknown = {};
|
||||
try {
|
||||
body = await request.json();
|
||||
} catch {
|
||||
// Empty body is fine — triggers all accounts
|
||||
}
|
||||
|
||||
const parsed = triggerIngestionSchema.safeParse(body);
|
||||
if (!parsed.success) {
|
||||
return NextResponse.json(
|
||||
{ error: "Invalid parameters", details: parsed.error.flatten() },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
// Find accounts to trigger
|
||||
const where: { isActive: boolean; authState: "AUTHENTICATED"; id?: string } = {
|
||||
isActive: true,
|
||||
authState: "AUTHENTICATED",
|
||||
};
|
||||
if (parsed.data.accountId) {
|
||||
where.id = parsed.data.accountId;
|
||||
}
|
||||
|
||||
const accounts = await prisma.telegramAccount.findMany({
|
||||
where,
|
||||
select: { id: true },
|
||||
});
|
||||
|
||||
if (accounts.length === 0) {
|
||||
return NextResponse.json(
|
||||
{ triggered: false, message: "No eligible accounts found" },
|
||||
{ status: 404 }
|
||||
);
|
||||
}
|
||||
|
||||
// Create ingestion runs marked as RUNNING — the worker will pick these up
|
||||
// when it next polls, or we use pg_notify for immediate pickup
|
||||
for (const account of accounts) {
|
||||
// Only create if no run is already RUNNING for this account
|
||||
const existing = await prisma.ingestionRun.findFirst({
|
||||
where: { accountId: account.id, status: "RUNNING" },
|
||||
});
|
||||
if (!existing) {
|
||||
await prisma.ingestionRun.create({
|
||||
data: { accountId: account.id, status: "RUNNING" },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Send pg_notify for immediate worker pickup
|
||||
try {
|
||||
await prisma.$queryRawUnsafe(
|
||||
`SELECT pg_notify('ingestion_trigger', $1)`,
|
||||
accounts.map((a) => a.id).join(",")
|
||||
);
|
||||
} catch {
|
||||
// pg_notify is best-effort — worker will pick up on next cycle anyway
|
||||
}
|
||||
|
||||
return NextResponse.json({
|
||||
triggered: true,
|
||||
accountIds: accounts.map((a) => a.id),
|
||||
message: `Ingestion queued for ${accounts.length} account(s)`,
|
||||
});
|
||||
}
|
||||
17
src/app/api/telegram/accounts/[accountId]/links/route.ts
Normal file
17
src/app/api/telegram/accounts/[accountId]/links/route.ts
Normal file
@@ -0,0 +1,17 @@
|
||||
import { NextResponse } from "next/server";
|
||||
import { authenticateApiRequest } from "@/lib/telegram/api-auth";
|
||||
import { listAccountChannelLinks } from "@/lib/telegram/admin-queries";
|
||||
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
export async function GET(
|
||||
request: Request,
|
||||
{ params }: { params: Promise<{ accountId: string }> }
|
||||
) {
|
||||
const authResult = await authenticateApiRequest(request, true);
|
||||
if ("error" in authResult) return authResult.error;
|
||||
|
||||
const { accountId } = await params;
|
||||
const links = await listAccountChannelLinks(accountId);
|
||||
return NextResponse.json(links);
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
import { NextResponse } from "next/server";
|
||||
import { authenticateApiRequest } from "@/lib/telegram/api-auth";
|
||||
import { getUnlinkedChannels } from "@/lib/telegram/admin-queries";
|
||||
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
export async function GET(
|
||||
request: Request,
|
||||
{ params }: { params: Promise<{ accountId: string }> }
|
||||
) {
|
||||
const authResult = await authenticateApiRequest(request, true);
|
||||
if ("error" in authResult) return authResult.error;
|
||||
|
||||
const { accountId } = await params;
|
||||
const channels = await getUnlinkedChannels(accountId);
|
||||
return NextResponse.json(channels);
|
||||
}
|
||||
32
src/app/api/zips/[id]/files/route.ts
Normal file
32
src/app/api/zips/[id]/files/route.ts
Normal file
@@ -0,0 +1,32 @@
|
||||
import { NextResponse } from "next/server";
|
||||
import { authenticateApiRequest } from "@/lib/telegram/api-auth";
|
||||
import { listPackageFiles } from "@/lib/telegram/queries";
|
||||
import { listFilesSchema } from "@/schemas/telegram";
|
||||
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
export async function GET(
|
||||
request: Request,
|
||||
{ params }: { params: Promise<{ id: string }> }
|
||||
) {
|
||||
const authResult = await authenticateApiRequest(request);
|
||||
if ("error" in authResult) return authResult.error;
|
||||
|
||||
const { id } = await params;
|
||||
const { searchParams } = new URL(request.url);
|
||||
const parsed = listFilesSchema.safeParse(Object.fromEntries(searchParams));
|
||||
|
||||
if (!parsed.success) {
|
||||
return NextResponse.json(
|
||||
{ error: "Invalid parameters", details: parsed.error.flatten() },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
const result = await listPackageFiles({
|
||||
packageId: id,
|
||||
...parsed.data,
|
||||
});
|
||||
|
||||
return NextResponse.json(result);
|
||||
}
|
||||
42
src/app/api/zips/[id]/preview/route.ts
Normal file
42
src/app/api/zips/[id]/preview/route.ts
Normal file
@@ -0,0 +1,42 @@
|
||||
import { NextResponse } from "next/server";
|
||||
import { prisma } from "@/lib/prisma";
|
||||
import { authenticateApiRequest } from "@/lib/telegram/api-auth";
|
||||
|
||||
/**
|
||||
* GET /api/zips/:id/preview
|
||||
* Returns the preview thumbnail image as JPEG binary.
|
||||
* Cached for 1 hour (immutable once set).
|
||||
*/
|
||||
export async function GET(
|
||||
request: Request,
|
||||
{ params }: { params: Promise<{ id: string }> }
|
||||
) {
|
||||
const authResult = await authenticateApiRequest(request);
|
||||
if ("error" in authResult) return authResult.error;
|
||||
|
||||
const { id } = await params;
|
||||
|
||||
const pkg = await prisma.package.findUnique({
|
||||
where: { id },
|
||||
select: { previewData: true },
|
||||
});
|
||||
|
||||
if (!pkg || !pkg.previewData) {
|
||||
return new NextResponse(null, { status: 404 });
|
||||
}
|
||||
|
||||
// previewData is stored as Bytes (Buffer) from Prisma
|
||||
const buffer =
|
||||
pkg.previewData instanceof Buffer
|
||||
? pkg.previewData
|
||||
: Buffer.from(pkg.previewData);
|
||||
|
||||
return new NextResponse(buffer, {
|
||||
status: 200,
|
||||
headers: {
|
||||
"Content-Type": "image/jpeg",
|
||||
"Content-Length": String(buffer.length),
|
||||
"Cache-Control": "public, max-age=3600, immutable",
|
||||
},
|
||||
});
|
||||
}
|
||||
22
src/app/api/zips/[id]/route.ts
Normal file
22
src/app/api/zips/[id]/route.ts
Normal file
@@ -0,0 +1,22 @@
|
||||
import { NextResponse } from "next/server";
|
||||
import { authenticateApiRequest } from "@/lib/telegram/api-auth";
|
||||
import { getPackageById } from "@/lib/telegram/queries";
|
||||
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
export async function GET(
|
||||
request: Request,
|
||||
{ params }: { params: Promise<{ id: string }> }
|
||||
) {
|
||||
const authResult = await authenticateApiRequest(request);
|
||||
if ("error" in authResult) return authResult.error;
|
||||
|
||||
const { id } = await params;
|
||||
const pkg = await getPackageById(id);
|
||||
|
||||
if (!pkg) {
|
||||
return NextResponse.json({ error: "Package not found" }, { status: 404 });
|
||||
}
|
||||
|
||||
return NextResponse.json(pkg);
|
||||
}
|
||||
24
src/app/api/zips/route.ts
Normal file
24
src/app/api/zips/route.ts
Normal file
@@ -0,0 +1,24 @@
|
||||
import { NextResponse } from "next/server";
|
||||
import { authenticateApiRequest } from "@/lib/telegram/api-auth";
|
||||
import { listPackages } from "@/lib/telegram/queries";
|
||||
import { listPackagesSchema } from "@/schemas/telegram";
|
||||
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
export async function GET(request: Request) {
|
||||
const authResult = await authenticateApiRequest(request);
|
||||
if ("error" in authResult) return authResult.error;
|
||||
|
||||
const { searchParams } = new URL(request.url);
|
||||
const parsed = listPackagesSchema.safeParse(Object.fromEntries(searchParams));
|
||||
|
||||
if (!parsed.success) {
|
||||
return NextResponse.json(
|
||||
{ error: "Invalid parameters", details: parsed.error.flatten() },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
const result = await listPackages(parsed.data);
|
||||
return NextResponse.json(result);
|
||||
}
|
||||
25
src/app/api/zips/search/route.ts
Normal file
25
src/app/api/zips/search/route.ts
Normal file
@@ -0,0 +1,25 @@
|
||||
import { NextResponse } from "next/server";
|
||||
import { authenticateApiRequest } from "@/lib/telegram/api-auth";
|
||||
import { searchPackages } from "@/lib/telegram/queries";
|
||||
import { searchSchema } from "@/schemas/telegram";
|
||||
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
export async function GET(request: Request) {
|
||||
const authResult = await authenticateApiRequest(request);
|
||||
if ("error" in authResult) return authResult.error;
|
||||
|
||||
const { searchParams } = new URL(request.url);
|
||||
const parsed = searchSchema.safeParse(Object.fromEntries(searchParams));
|
||||
|
||||
if (!parsed.success) {
|
||||
return NextResponse.json(
|
||||
{ error: "Invalid parameters", details: parsed.error.flatten() },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
const { q, ...rest } = parsed.data;
|
||||
const result = await searchPackages({ query: q, ...rest });
|
||||
return NextResponse.json(result);
|
||||
}
|
||||
@@ -8,6 +8,8 @@ import {
|
||||
Droplets,
|
||||
Paintbrush,
|
||||
Gem,
|
||||
FileBox,
|
||||
Send,
|
||||
ClipboardList,
|
||||
Building2,
|
||||
MapPin,
|
||||
@@ -18,7 +20,7 @@ import { cn } from "@/lib/utils";
|
||||
import { APP_NAME } from "@/lib/constants";
|
||||
import { SheetHeader, SheetTitle } from "@/components/ui/sheet";
|
||||
|
||||
const icons = { LayoutDashboard, Cylinder, Droplets, Paintbrush, Gem, ClipboardList, Building2, MapPin, Settings };
|
||||
const icons = { LayoutDashboard, Cylinder, Droplets, Paintbrush, Gem, FileBox, Send, ClipboardList, Building2, MapPin, Settings };
|
||||
|
||||
const navItems = [
|
||||
{ label: "Dashboard", href: "/dashboard", icon: "LayoutDashboard" as const },
|
||||
@@ -26,6 +28,8 @@ const navItems = [
|
||||
{ label: "Resins", href: "/resins", icon: "Droplets" as const },
|
||||
{ label: "Paints", href: "/paints", icon: "Paintbrush" as const },
|
||||
{ label: "Supplies", href: "/supplies", icon: "Gem" as const },
|
||||
{ label: "STL Files", href: "/stls", icon: "FileBox" as const },
|
||||
{ label: "Telegram", href: "/telegram", icon: "Send" as const },
|
||||
{ label: "Usage", href: "/usage", icon: "ClipboardList" as const },
|
||||
{ label: "Vendors", href: "/vendors", icon: "Building2" as const },
|
||||
{ label: "Locations", href: "/locations", icon: "MapPin" as const },
|
||||
|
||||
@@ -9,6 +9,8 @@ import {
|
||||
Droplets,
|
||||
Paintbrush,
|
||||
Gem,
|
||||
FileBox,
|
||||
Send,
|
||||
ClipboardList,
|
||||
Building2,
|
||||
MapPin,
|
||||
@@ -28,6 +30,8 @@ const icons = {
|
||||
Droplets,
|
||||
Paintbrush,
|
||||
Gem,
|
||||
FileBox,
|
||||
Send,
|
||||
ClipboardList,
|
||||
Building2,
|
||||
MapPin,
|
||||
@@ -40,6 +44,8 @@ const navItems = [
|
||||
{ label: "Resins", href: "/resins", icon: "Droplets" as const },
|
||||
{ label: "Paints", href: "/paints", icon: "Paintbrush" as const },
|
||||
{ label: "Supplies", href: "/supplies", icon: "Gem" as const },
|
||||
{ label: "STL Files", href: "/stls", icon: "FileBox" as const },
|
||||
{ label: "Telegram", href: "/telegram", icon: "Send" as const },
|
||||
{ label: "Usage", href: "/usage", icon: "ClipboardList" as const },
|
||||
{ label: "Vendors", href: "/vendors", icon: "Building2" as const },
|
||||
{ label: "Locations", href: "/locations", icon: "MapPin" as const },
|
||||
|
||||
@@ -6,6 +6,8 @@ export const NAV_ITEMS = [
|
||||
{ label: "Resins", href: "/resins", icon: "Droplets" },
|
||||
{ label: "Paints", href: "/paints", icon: "Paintbrush" },
|
||||
{ label: "Supplies", href: "/supplies", icon: "Gem" },
|
||||
{ label: "STL Files", href: "/stls", icon: "FileBox" },
|
||||
{ label: "Telegram", href: "/telegram", icon: "Send" },
|
||||
{ label: "Usage", href: "/usage", icon: "ClipboardList" },
|
||||
{ label: "Vendors", href: "/vendors", icon: "Building2" },
|
||||
{ label: "Locations", href: "/locations", icon: "MapPin" },
|
||||
|
||||
105
src/lib/telegram/admin-queries.ts
Normal file
105
src/lib/telegram/admin-queries.ts
Normal file
@@ -0,0 +1,105 @@
|
||||
import { prisma } from "@/lib/prisma";
|
||||
|
||||
// ── Account queries ──
|
||||
|
||||
export async function listAccounts() {
|
||||
const accounts = await prisma.telegramAccount.findMany({
|
||||
orderBy: { createdAt: "desc" },
|
||||
include: {
|
||||
_count: { select: { channelMaps: true, ingestionRuns: true } },
|
||||
},
|
||||
});
|
||||
|
||||
return accounts.map((a) => ({
|
||||
id: a.id,
|
||||
phone: a.phone,
|
||||
displayName: a.displayName,
|
||||
isActive: a.isActive,
|
||||
authState: a.authState,
|
||||
authCode: a.authCode,
|
||||
lastSeenAt: a.lastSeenAt?.toISOString() ?? null,
|
||||
createdAt: a.createdAt.toISOString(),
|
||||
channelCount: a._count.channelMaps,
|
||||
runCount: a._count.ingestionRuns,
|
||||
}));
|
||||
}
|
||||
|
||||
export type AccountRow = Awaited<ReturnType<typeof listAccounts>>[number];
|
||||
|
||||
// ── Channel queries ──
|
||||
|
||||
export async function listChannels() {
|
||||
const channels = await prisma.telegramChannel.findMany({
|
||||
orderBy: { createdAt: "desc" },
|
||||
include: {
|
||||
_count: { select: { accountMaps: true, packages: true } },
|
||||
},
|
||||
});
|
||||
|
||||
return channels.map((c) => ({
|
||||
id: c.id,
|
||||
telegramId: c.telegramId.toString(),
|
||||
title: c.title,
|
||||
type: c.type,
|
||||
isActive: c.isActive,
|
||||
createdAt: c.createdAt.toISOString(),
|
||||
accountCount: c._count.accountMaps,
|
||||
packageCount: c._count.packages,
|
||||
}));
|
||||
}
|
||||
|
||||
export type ChannelRow = Awaited<ReturnType<typeof listChannels>>[number];
|
||||
|
||||
// ── Account-Channel link queries ──
|
||||
|
||||
export async function listAccountChannelLinks(accountId: string) {
|
||||
const links = await prisma.accountChannelMap.findMany({
|
||||
where: { accountId },
|
||||
include: {
|
||||
channel: { select: { id: true, title: true, type: true, telegramId: true } },
|
||||
},
|
||||
orderBy: { createdAt: "desc" },
|
||||
});
|
||||
|
||||
return links.map((l) => ({
|
||||
id: l.id,
|
||||
accountId: l.accountId,
|
||||
channelId: l.channelId,
|
||||
role: l.role,
|
||||
lastProcessedMessageId: l.lastProcessedMessageId?.toString() ?? null,
|
||||
channel: {
|
||||
id: l.channel.id,
|
||||
title: l.channel.title,
|
||||
type: l.channel.type,
|
||||
telegramId: l.channel.telegramId.toString(),
|
||||
},
|
||||
}));
|
||||
}
|
||||
|
||||
export type AccountChannelLinkRow = Awaited<
|
||||
ReturnType<typeof listAccountChannelLinks>
|
||||
>[number];
|
||||
|
||||
export async function getUnlinkedChannels(accountId: string) {
|
||||
const linked = await prisma.accountChannelMap.findMany({
|
||||
where: { accountId },
|
||||
select: { channelId: true },
|
||||
});
|
||||
const linkedIds = linked.map((l) => l.channelId);
|
||||
|
||||
const unlinked = await prisma.telegramChannel.findMany({
|
||||
where: {
|
||||
id: { notIn: linkedIds },
|
||||
isActive: true,
|
||||
},
|
||||
orderBy: { title: "asc" },
|
||||
select: { id: true, title: true, type: true, telegramId: true },
|
||||
});
|
||||
|
||||
return unlinked.map((c) => ({
|
||||
id: c.id,
|
||||
title: c.title,
|
||||
type: c.type,
|
||||
telegramId: c.telegramId.toString(),
|
||||
}));
|
||||
}
|
||||
45
src/lib/telegram/api-auth.ts
Normal file
45
src/lib/telegram/api-auth.ts
Normal file
@@ -0,0 +1,45 @@
|
||||
import { auth } from "@/lib/auth";
|
||||
import { NextResponse } from "next/server";
|
||||
|
||||
/**
|
||||
* Authenticate an API request. Checks:
|
||||
* 1. X-API-Key header against TELEGRAM_API_KEY env var
|
||||
* 2. NextAuth session
|
||||
*
|
||||
* Returns null if authenticated, or a NextResponse error if not.
|
||||
*/
|
||||
export async function authenticateApiRequest(
|
||||
request: Request,
|
||||
requireAdmin = false
|
||||
): Promise<{ error: NextResponse } | { userId: string; role: string }> {
|
||||
// Check API key first
|
||||
const apiKey = request.headers.get("X-API-Key");
|
||||
const envKey = process.env.TELEGRAM_API_KEY;
|
||||
|
||||
if (apiKey && envKey && apiKey === envKey) {
|
||||
// API key auth — treated as admin
|
||||
return { userId: "api-key", role: "ADMIN" };
|
||||
}
|
||||
|
||||
// Fall back to session auth
|
||||
const session = await auth();
|
||||
if (!session?.user?.id) {
|
||||
return {
|
||||
error: NextResponse.json(
|
||||
{ error: "Unauthorized" },
|
||||
{ status: 401 }
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
if (requireAdmin && session.user.role !== "ADMIN") {
|
||||
return {
|
||||
error: NextResponse.json(
|
||||
{ error: "Forbidden: admin role required" },
|
||||
{ status: 403 }
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
return { userId: session.user.id, role: session.user.role };
|
||||
}
|
||||
314
src/lib/telegram/queries.ts
Normal file
314
src/lib/telegram/queries.ts
Normal file
@@ -0,0 +1,314 @@
|
||||
import { prisma } from "@/lib/prisma";
|
||||
import type {
|
||||
PackageListItem,
|
||||
PackageDetail,
|
||||
PackageFileItem,
|
||||
IngestionAccountStatus,
|
||||
} from "./types";
|
||||
|
||||
export async function listPackages(options: {
|
||||
page: number;
|
||||
limit: number;
|
||||
channelId?: string;
|
||||
creator?: string;
|
||||
sortBy: "indexedAt" | "fileName" | "fileSize";
|
||||
order: "asc" | "desc";
|
||||
}) {
|
||||
const where: Record<string, unknown> = {};
|
||||
if (options.channelId) where.sourceChannelId = options.channelId;
|
||||
if (options.creator) where.creator = options.creator;
|
||||
|
||||
const [items, total] = await Promise.all([
|
||||
prisma.package.findMany({
|
||||
where,
|
||||
orderBy: { [options.sortBy]: options.order },
|
||||
skip: (options.page - 1) * options.limit,
|
||||
take: options.limit,
|
||||
select: {
|
||||
id: true,
|
||||
fileName: true,
|
||||
fileSize: true,
|
||||
contentHash: true,
|
||||
archiveType: true,
|
||||
fileCount: true,
|
||||
isMultipart: true,
|
||||
indexedAt: true,
|
||||
creator: true,
|
||||
previewMsgId: true, // cheap null check — avoids loading blob
|
||||
sourceChannel: { select: { id: true, title: true } },
|
||||
},
|
||||
}),
|
||||
prisma.package.count({ where }),
|
||||
]);
|
||||
|
||||
const mapped: PackageListItem[] = items.map((pkg) => ({
|
||||
id: pkg.id,
|
||||
fileName: pkg.fileName,
|
||||
fileSize: pkg.fileSize.toString(),
|
||||
contentHash: pkg.contentHash,
|
||||
archiveType: pkg.archiveType,
|
||||
fileCount: pkg.fileCount,
|
||||
isMultipart: pkg.isMultipart,
|
||||
hasPreview: pkg.previewMsgId !== null,
|
||||
creator: pkg.creator,
|
||||
indexedAt: pkg.indexedAt.toISOString(),
|
||||
sourceChannel: pkg.sourceChannel,
|
||||
}));
|
||||
|
||||
return {
|
||||
items: mapped,
|
||||
pagination: {
|
||||
page: options.page,
|
||||
limit: options.limit,
|
||||
total,
|
||||
totalPages: Math.ceil(total / options.limit),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export async function getPackageById(
|
||||
id: string
|
||||
): Promise<PackageDetail | null> {
|
||||
const pkg = await prisma.package.findUnique({
|
||||
where: { id },
|
||||
include: {
|
||||
sourceChannel: { select: { id: true, title: true } },
|
||||
ingestionRun: { select: { id: true, startedAt: true } },
|
||||
},
|
||||
});
|
||||
|
||||
if (!pkg) return null;
|
||||
|
||||
let destChannel: { id: string; title: string } | null = null;
|
||||
if (pkg.destChannelId) {
|
||||
const ch = await prisma.telegramChannel.findUnique({
|
||||
where: { id: pkg.destChannelId },
|
||||
select: { id: true, title: true },
|
||||
});
|
||||
destChannel = ch;
|
||||
}
|
||||
|
||||
return {
|
||||
id: pkg.id,
|
||||
fileName: pkg.fileName,
|
||||
fileSize: pkg.fileSize.toString(),
|
||||
contentHash: pkg.contentHash,
|
||||
archiveType: pkg.archiveType,
|
||||
fileCount: pkg.fileCount,
|
||||
isMultipart: pkg.isMultipart,
|
||||
hasPreview: pkg.previewMsgId !== null,
|
||||
creator: pkg.creator,
|
||||
partCount: pkg.partCount,
|
||||
indexedAt: pkg.indexedAt.toISOString(),
|
||||
sourceChannel: pkg.sourceChannel,
|
||||
destChannel,
|
||||
destMessageId: pkg.destMessageId?.toString() ?? null,
|
||||
sourceMessageId: pkg.sourceMessageId.toString(),
|
||||
ingestionRun: pkg.ingestionRun
|
||||
? {
|
||||
id: pkg.ingestionRun.id,
|
||||
startedAt: pkg.ingestionRun.startedAt.toISOString(),
|
||||
}
|
||||
: null,
|
||||
};
|
||||
}
|
||||
|
||||
export async function listPackageFiles(options: {
|
||||
packageId: string;
|
||||
page: number;
|
||||
limit: number;
|
||||
extension?: string;
|
||||
}) {
|
||||
const where: { packageId: string; extension?: string } = {
|
||||
packageId: options.packageId,
|
||||
};
|
||||
if (options.extension) {
|
||||
where.extension = options.extension;
|
||||
}
|
||||
|
||||
const [items, total] = await Promise.all([
|
||||
prisma.packageFile.findMany({
|
||||
where,
|
||||
orderBy: { path: "asc" },
|
||||
skip: (options.page - 1) * options.limit,
|
||||
take: options.limit,
|
||||
}),
|
||||
prisma.packageFile.count({ where }),
|
||||
]);
|
||||
|
||||
const mapped: PackageFileItem[] = items.map((f) => ({
|
||||
id: f.id,
|
||||
path: f.path,
|
||||
fileName: f.fileName,
|
||||
extension: f.extension,
|
||||
compressedSize: f.compressedSize.toString(),
|
||||
uncompressedSize: f.uncompressedSize.toString(),
|
||||
crc32: f.crc32,
|
||||
}));
|
||||
|
||||
return {
|
||||
items: mapped,
|
||||
pagination: {
|
||||
page: options.page,
|
||||
limit: options.limit,
|
||||
total,
|
||||
totalPages: Math.ceil(total / options.limit),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export async function searchPackages(options: {
|
||||
query: string;
|
||||
page: number;
|
||||
limit: number;
|
||||
searchIn: "packages" | "files" | "both";
|
||||
}) {
|
||||
const q = options.query;
|
||||
|
||||
if (options.searchIn === "files" || options.searchIn === "both") {
|
||||
// Search in package files, return parent packages
|
||||
const fileMatches = await prisma.packageFile.findMany({
|
||||
where: {
|
||||
OR: [
|
||||
{ fileName: { contains: q, mode: "insensitive" } },
|
||||
{ path: { contains: q, mode: "insensitive" } },
|
||||
],
|
||||
},
|
||||
select: { packageId: true },
|
||||
distinct: ["packageId"],
|
||||
});
|
||||
|
||||
const packageIds = fileMatches.map((f) => f.packageId);
|
||||
|
||||
const packageNameIds =
|
||||
options.searchIn === "both"
|
||||
? (
|
||||
await prisma.package.findMany({
|
||||
where: { fileName: { contains: q, mode: "insensitive" } },
|
||||
select: { id: true },
|
||||
})
|
||||
).map((p) => p.id)
|
||||
: [];
|
||||
|
||||
const allIds = [...new Set([...packageIds, ...packageNameIds])];
|
||||
|
||||
const [items, total] = await Promise.all([
|
||||
prisma.package.findMany({
|
||||
where: { id: { in: allIds } },
|
||||
orderBy: { indexedAt: "desc" },
|
||||
skip: (options.page - 1) * options.limit,
|
||||
take: options.limit,
|
||||
select: {
|
||||
id: true,
|
||||
fileName: true,
|
||||
fileSize: true,
|
||||
contentHash: true,
|
||||
archiveType: true,
|
||||
fileCount: true,
|
||||
isMultipart: true,
|
||||
indexedAt: true,
|
||||
creator: true,
|
||||
previewMsgId: true,
|
||||
sourceChannel: { select: { id: true, title: true } },
|
||||
},
|
||||
}),
|
||||
Promise.resolve(allIds.length),
|
||||
]);
|
||||
|
||||
const mapped: PackageListItem[] = items.map((pkg) => ({
|
||||
id: pkg.id,
|
||||
fileName: pkg.fileName,
|
||||
fileSize: pkg.fileSize.toString(),
|
||||
contentHash: pkg.contentHash,
|
||||
archiveType: pkg.archiveType,
|
||||
fileCount: pkg.fileCount,
|
||||
isMultipart: pkg.isMultipart,
|
||||
hasPreview: pkg.previewMsgId !== null,
|
||||
creator: pkg.creator,
|
||||
indexedAt: pkg.indexedAt.toISOString(),
|
||||
sourceChannel: pkg.sourceChannel,
|
||||
}));
|
||||
|
||||
return {
|
||||
items: mapped,
|
||||
pagination: {
|
||||
page: options.page,
|
||||
limit: options.limit,
|
||||
total,
|
||||
totalPages: Math.ceil(total / options.limit),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// Search packages only
|
||||
return listPackages({
|
||||
page: options.page,
|
||||
limit: options.limit,
|
||||
sortBy: "indexedAt",
|
||||
order: "desc",
|
||||
});
|
||||
}
|
||||
|
||||
export async function getIngestionStatus(): Promise<IngestionAccountStatus[]> {
|
||||
const accounts = await prisma.telegramAccount.findMany({
|
||||
orderBy: { createdAt: "asc" },
|
||||
});
|
||||
|
||||
const statuses: IngestionAccountStatus[] = [];
|
||||
|
||||
for (const account of accounts) {
|
||||
const lastRun = await prisma.ingestionRun.findFirst({
|
||||
where: { accountId: account.id, status: { not: "RUNNING" } },
|
||||
orderBy: { startedAt: "desc" },
|
||||
});
|
||||
|
||||
const currentRun = await prisma.ingestionRun.findFirst({
|
||||
where: { accountId: account.id, status: "RUNNING" },
|
||||
orderBy: { startedAt: "desc" },
|
||||
});
|
||||
|
||||
statuses.push({
|
||||
id: account.id,
|
||||
displayName: account.displayName,
|
||||
phone: account.phone,
|
||||
isActive: account.isActive,
|
||||
authState: account.authState,
|
||||
lastSeenAt: account.lastSeenAt?.toISOString() ?? null,
|
||||
lastRun: lastRun
|
||||
? {
|
||||
id: lastRun.id,
|
||||
status: lastRun.status,
|
||||
startedAt: lastRun.startedAt.toISOString(),
|
||||
finishedAt: lastRun.finishedAt?.toISOString() ?? null,
|
||||
messagesScanned: lastRun.messagesScanned,
|
||||
zipsFound: lastRun.zipsFound,
|
||||
zipsDuplicate: lastRun.zipsDuplicate,
|
||||
zipsIngested: lastRun.zipsIngested,
|
||||
}
|
||||
: null,
|
||||
currentRun: currentRun
|
||||
? {
|
||||
id: currentRun.id,
|
||||
startedAt: currentRun.startedAt.toISOString(),
|
||||
messagesScanned: currentRun.messagesScanned,
|
||||
zipsFound: currentRun.zipsFound,
|
||||
zipsDuplicate: currentRun.zipsDuplicate,
|
||||
zipsIngested: currentRun.zipsIngested,
|
||||
// Live activity tracking
|
||||
currentActivity: currentRun.currentActivity,
|
||||
currentStep: currentRun.currentStep,
|
||||
currentChannel: currentRun.currentChannel,
|
||||
currentFile: currentRun.currentFile,
|
||||
currentFileNum: currentRun.currentFileNum,
|
||||
totalFiles: currentRun.totalFiles,
|
||||
downloadedBytes: currentRun.downloadedBytes?.toString() ?? null,
|
||||
totalBytes: currentRun.totalBytes?.toString() ?? null,
|
||||
downloadPercent: currentRun.downloadPercent,
|
||||
lastActivityAt: currentRun.lastActivityAt?.toISOString() ?? null,
|
||||
}
|
||||
: null,
|
||||
});
|
||||
}
|
||||
|
||||
return statuses;
|
||||
}
|
||||
88
src/lib/telegram/types.ts
Normal file
88
src/lib/telegram/types.ts
Normal file
@@ -0,0 +1,88 @@
|
||||
export interface PackageListItem {
|
||||
id: string;
|
||||
fileName: string;
|
||||
fileSize: string; // BigInt serialized as string
|
||||
contentHash: string;
|
||||
archiveType: "ZIP" | "RAR";
|
||||
fileCount: number;
|
||||
isMultipart: boolean;
|
||||
hasPreview: boolean;
|
||||
creator: string | null;
|
||||
indexedAt: string;
|
||||
sourceChannel: {
|
||||
id: string;
|
||||
title: string;
|
||||
};
|
||||
}
|
||||
|
||||
export interface PackageDetail extends PackageListItem {
|
||||
partCount: number;
|
||||
destChannel: {
|
||||
id: string;
|
||||
title: string;
|
||||
} | null;
|
||||
destMessageId: string | null;
|
||||
sourceMessageId: string;
|
||||
ingestionRun: {
|
||||
id: string;
|
||||
startedAt: string;
|
||||
} | null;
|
||||
}
|
||||
|
||||
export interface PackageFileItem {
|
||||
id: string;
|
||||
path: string;
|
||||
fileName: string;
|
||||
extension: string | null;
|
||||
compressedSize: string;
|
||||
uncompressedSize: string;
|
||||
crc32: string | null;
|
||||
}
|
||||
|
||||
export interface PaginatedResponse<T> {
|
||||
items: T[];
|
||||
pagination: {
|
||||
page: number;
|
||||
limit: number;
|
||||
total: number;
|
||||
totalPages: number;
|
||||
};
|
||||
}
|
||||
|
||||
export interface IngestionAccountStatus {
|
||||
id: string;
|
||||
displayName: string | null;
|
||||
phone: string;
|
||||
isActive: boolean;
|
||||
authState: string;
|
||||
lastSeenAt: string | null;
|
||||
lastRun: {
|
||||
id: string;
|
||||
status: string;
|
||||
startedAt: string;
|
||||
finishedAt: string | null;
|
||||
messagesScanned: number;
|
||||
zipsFound: number;
|
||||
zipsDuplicate: number;
|
||||
zipsIngested: number;
|
||||
} | null;
|
||||
currentRun: {
|
||||
id: string;
|
||||
startedAt: string;
|
||||
messagesScanned: number;
|
||||
zipsFound: number;
|
||||
zipsDuplicate: number;
|
||||
zipsIngested: number;
|
||||
// Live activity tracking
|
||||
currentActivity: string | null;
|
||||
currentStep: string | null;
|
||||
currentChannel: string | null;
|
||||
currentFile: string | null;
|
||||
currentFileNum: number | null;
|
||||
totalFiles: number | null;
|
||||
downloadedBytes: string | null; // BigInt serialized as string
|
||||
totalBytes: string | null; // BigInt serialized as string
|
||||
downloadPercent: number | null;
|
||||
lastActivityAt: string | null;
|
||||
} | null;
|
||||
}
|
||||
71
src/schemas/telegram.ts
Normal file
71
src/schemas/telegram.ts
Normal file
@@ -0,0 +1,71 @@
|
||||
import { z } from "zod/v4";
|
||||
|
||||
export const paginationSchema = z.object({
|
||||
page: z.coerce.number().int().min(1).default(1),
|
||||
limit: z.coerce.number().int().min(1).max(100).default(25),
|
||||
});
|
||||
|
||||
export const listPackagesSchema = paginationSchema.extend({
|
||||
channelId: z.string().optional(),
|
||||
creator: z.string().optional(),
|
||||
sortBy: z.enum(["indexedAt", "fileName", "fileSize"]).default("indexedAt"),
|
||||
order: z.enum(["asc", "desc"]).default("desc"),
|
||||
});
|
||||
|
||||
export const listFilesSchema = paginationSchema.extend({
|
||||
limit: z.coerce.number().int().min(1).max(500).default(50),
|
||||
extension: z.string().optional(),
|
||||
});
|
||||
|
||||
export const searchSchema = paginationSchema.extend({
|
||||
q: z.string().min(1),
|
||||
searchIn: z.enum(["packages", "files", "both"]).default("both"),
|
||||
});
|
||||
|
||||
export const triggerIngestionSchema = z.object({
|
||||
accountId: z.string().optional(),
|
||||
});
|
||||
|
||||
// ── Account CRUD ──
|
||||
|
||||
export const telegramAccountSchema = z.object({
|
||||
phone: z
|
||||
.string()
|
||||
.min(1, "Phone number is required")
|
||||
.regex(/^\+?\d[\d\s\-]{6,20}$/, "Invalid phone format (e.g. +31612345678)"),
|
||||
displayName: z.string().max(64).optional().or(z.literal("")),
|
||||
});
|
||||
|
||||
export type TelegramAccountInput = z.infer<typeof telegramAccountSchema>;
|
||||
|
||||
export const submitAuthCodeSchema = z.object({
|
||||
code: z.string().min(3, "Auth code is required").max(10),
|
||||
});
|
||||
|
||||
export type SubmitAuthCodeInput = z.infer<typeof submitAuthCodeSchema>;
|
||||
|
||||
export const submitPasswordSchema = z.object({
|
||||
password: z.string().min(1, "Password is required"),
|
||||
});
|
||||
|
||||
export type SubmitPasswordInput = z.infer<typeof submitPasswordSchema>;
|
||||
|
||||
// ── Channel CRUD ──
|
||||
|
||||
export const telegramChannelSchema = z.object({
|
||||
telegramId: z.coerce.number().int().min(1, "Telegram ID is required"),
|
||||
title: z.string().min(1, "Title is required").max(256),
|
||||
type: z.enum(["SOURCE", "DESTINATION"]),
|
||||
});
|
||||
|
||||
export type TelegramChannelInput = z.infer<typeof telegramChannelSchema>;
|
||||
|
||||
// ── Account-Channel linking ──
|
||||
|
||||
export const linkChannelSchema = z.object({
|
||||
accountId: z.string().min(1),
|
||||
channelId: z.string().min(1),
|
||||
role: z.enum(["READER", "WRITER"]).default("READER"),
|
||||
});
|
||||
|
||||
export type LinkChannelInput = z.infer<typeof linkChannelSchema>;
|
||||
@@ -31,5 +31,5 @@
|
||||
".next/dev/types/**/*.ts",
|
||||
"**/*.mts"
|
||||
],
|
||||
"exclude": ["node_modules", "prisma/seed.ts", "scripts/**"]
|
||||
"exclude": ["node_modules", "prisma/seed.ts", "scripts/**", "worker/**"]
|
||||
}
|
||||
|
||||
48
worker/Dockerfile
Normal file
48
worker/Dockerfile
Normal file
@@ -0,0 +1,48 @@
|
||||
# ── Stage 1: Install production deps ─────────────────────────
|
||||
FROM node:20-bookworm-slim AS deps
|
||||
|
||||
RUN sed -i 's/^Components: main$/Components: main non-free/' /etc/apt/sources.list.d/debian.sources && \
|
||||
apt-get update && apt-get install -y \
|
||||
libssl-dev zlib1g-dev unrar \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY worker/package.json worker/package-lock.json* ./
|
||||
COPY prisma/ ./prisma/
|
||||
|
||||
# Install ALL deps (including devDependencies for tsc) and generate Prisma
|
||||
RUN npm ci && npx prisma generate
|
||||
|
||||
# ── Stage 2: Build TypeScript ─────────────────────────────────
|
||||
FROM deps AS builder
|
||||
|
||||
COPY worker/tsconfig.json ./
|
||||
COPY worker/src/ ./src/
|
||||
RUN npx tsc
|
||||
|
||||
# ── Stage 3: Production runner ────────────────────────────────
|
||||
FROM node:20-bookworm-slim AS runner
|
||||
|
||||
RUN sed -i 's/^Components: main$/Components: main non-free/' /etc/apt/sources.list.d/debian.sources && \
|
||||
apt-get update && apt-get install -y \
|
||||
libssl3 zlib1g unrar \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Copy only production node_modules (prune devDeps)
|
||||
COPY --from=builder /app/node_modules ./node_modules
|
||||
COPY --from=builder /app/prisma ./prisma
|
||||
COPY --from=builder /app/dist ./dist
|
||||
|
||||
# Re-generate Prisma client for production (after pruning isn't needed since we copy all)
|
||||
RUN npx prisma generate
|
||||
|
||||
RUN addgroup --system worker && adduser --system --ingroup worker worker
|
||||
RUN mkdir -p /data/tdlib /tmp/zips && chown -R worker:worker /data/tdlib /tmp/zips
|
||||
USER worker
|
||||
|
||||
VOLUME ["/data/tdlib", "/tmp/zips"]
|
||||
|
||||
CMD ["node", "dist/index.js"]
|
||||
2140
worker/package-lock.json
generated
Normal file
2140
worker/package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load Diff
28
worker/package.json
Normal file
28
worker/package.json
Normal file
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"name": "dragonsstash-worker",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
"start": "node dist/index.js",
|
||||
"dev": "tsx watch src/index.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"@prisma/adapter-pg": "^7.4.0",
|
||||
"@prisma/client": "^7.4.0",
|
||||
"pg": "^8.18.0",
|
||||
"pino": "^9.6.0",
|
||||
"prebuilt-tdlib": "^0.1008050.0",
|
||||
"tdl": "^8.0.0",
|
||||
"yauzl": "^3.2.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^20",
|
||||
"@types/pg": "^8.16.0",
|
||||
"@types/yauzl": "^2.10.3",
|
||||
"prisma": "^7.4.0",
|
||||
"tsx": "^4.21.0",
|
||||
"typescript": "^5"
|
||||
}
|
||||
}
|
||||
21
worker/src/archive/creator.ts
Normal file
21
worker/src/archive/creator.ts
Normal file
@@ -0,0 +1,21 @@
|
||||
/**
|
||||
* Extract a creator name from common archive file naming patterns.
|
||||
*
|
||||
* Priority in the worker: topic name > filename extraction.
|
||||
* This is the fallback when no forum topic name is available.
|
||||
*
|
||||
* Patterns handled (split on ` - `):
|
||||
* "Mammoth Factory - 2026-01.zip" → "Mammoth Factory"
|
||||
* "Artist Name - Pack Title.part01.rar" → "Artist Name"
|
||||
* "some_random_file.zip" → null
|
||||
*/
|
||||
export function extractCreatorFromFileName(fileName: string): string | null {
|
||||
// Strip archive extensions (.zip, .rar, .part01.rar, .z01, etc.)
|
||||
const bare = fileName.replace(/(\.(part\d+\.rar|z\d{2}|zip|rar))+$/i, "");
|
||||
|
||||
const idx = bare.indexOf(" - ");
|
||||
if (idx <= 0) return null;
|
||||
|
||||
const creator = bare.slice(0, idx).trim();
|
||||
return creator.length > 0 ? creator : null;
|
||||
}
|
||||
96
worker/src/archive/detect.ts
Normal file
96
worker/src/archive/detect.ts
Normal file
@@ -0,0 +1,96 @@
|
||||
export type ArchiveFormat = "ZIP" | "RAR";
|
||||
|
||||
export interface MultipartInfo {
|
||||
baseName: string;
|
||||
partNumber: number;
|
||||
format: ArchiveFormat;
|
||||
pattern: "ZIP_NUMBERED" | "ZIP_LEGACY" | "RAR_PART" | "RAR_LEGACY" | "SINGLE";
|
||||
}
|
||||
|
||||
const patterns: {
|
||||
regex: RegExp;
|
||||
format: ArchiveFormat;
|
||||
pattern: MultipartInfo["pattern"];
|
||||
getBaseName: (match: RegExpMatchArray) => string;
|
||||
getPartNumber: (match: RegExpMatchArray) => number;
|
||||
}[] = [
|
||||
// pack.zip.001, pack.zip.002
|
||||
{
|
||||
regex: /^(.+\.zip)\.(\d{3,})$/i,
|
||||
format: "ZIP",
|
||||
pattern: "ZIP_NUMBERED",
|
||||
getBaseName: (m) => m[1],
|
||||
getPartNumber: (m) => parseInt(m[2], 10),
|
||||
},
|
||||
// pack.z01, pack.z02 (legacy split — final part is pack.zip)
|
||||
{
|
||||
regex: /^(.+)\.z(\d{2,})$/i,
|
||||
format: "ZIP",
|
||||
pattern: "ZIP_LEGACY",
|
||||
getBaseName: (m) => m[1],
|
||||
getPartNumber: (m) => parseInt(m[2], 10),
|
||||
},
|
||||
// pack.part1.rar, pack.part2.rar
|
||||
{
|
||||
regex: /^(.+)\.part(\d+)\.rar$/i,
|
||||
format: "RAR",
|
||||
pattern: "RAR_PART",
|
||||
getBaseName: (m) => m[1],
|
||||
getPartNumber: (m) => parseInt(m[2], 10),
|
||||
},
|
||||
// pack.r00, pack.r01 (legacy split — final part is pack.rar)
|
||||
{
|
||||
regex: /^(.+)\.r(\d{2,})$/i,
|
||||
format: "RAR",
|
||||
pattern: "RAR_LEGACY",
|
||||
getBaseName: (m) => m[1],
|
||||
getPartNumber: (m) => parseInt(m[2], 10),
|
||||
},
|
||||
];
|
||||
|
||||
/**
|
||||
* Detect if a filename is an archive and extract multipart info.
|
||||
*/
|
||||
export function detectArchive(fileName: string): MultipartInfo | null {
|
||||
// Check multipart patterns first
|
||||
for (const p of patterns) {
|
||||
const match = fileName.match(p.regex);
|
||||
if (match) {
|
||||
return {
|
||||
baseName: p.getBaseName(match),
|
||||
partNumber: p.getPartNumber(match),
|
||||
format: p.format,
|
||||
pattern: p.pattern,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Single .zip file — could be a standalone or the final part of a ZIP_LEGACY set
|
||||
if (/\.zip$/i.test(fileName)) {
|
||||
return {
|
||||
baseName: fileName.replace(/\.zip$/i, ""),
|
||||
partNumber: -1, // -1 signals "could be single or final legacy part"
|
||||
format: "ZIP",
|
||||
pattern: "SINGLE",
|
||||
};
|
||||
}
|
||||
|
||||
// Single .rar file — could be standalone or final part of RAR_LEGACY set
|
||||
if (/\.rar$/i.test(fileName)) {
|
||||
return {
|
||||
baseName: fileName.replace(/\.rar$/i, ""),
|
||||
partNumber: -1,
|
||||
format: "RAR",
|
||||
pattern: "SINGLE",
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a filename looks like any archive attachment we should process.
|
||||
*/
|
||||
export function isArchiveAttachment(fileName: string): boolean {
|
||||
return detectArchive(fileName) !== null;
|
||||
}
|
||||
25
worker/src/archive/hash.ts
Normal file
25
worker/src/archive/hash.ts
Normal file
@@ -0,0 +1,25 @@
|
||||
import { createReadStream } from "fs";
|
||||
import { createHash } from "crypto";
|
||||
import { pipeline } from "stream/promises";
|
||||
import { PassThrough } from "stream";
|
||||
|
||||
/**
|
||||
* Compute SHA-256 hash of one or more files by streaming them in order.
|
||||
* Memory usage: O(1) — reads in 64KB chunks regardless of total size.
|
||||
* For multipart archives, pass all parts sorted by part number.
|
||||
*/
|
||||
export async function hashParts(filePaths: string[]): Promise<string> {
|
||||
const hash = createHash("sha256");
|
||||
for (const filePath of filePaths) {
|
||||
await pipeline(
|
||||
createReadStream(filePath),
|
||||
new PassThrough({
|
||||
transform(chunk, _encoding, callback) {
|
||||
hash.update(chunk);
|
||||
callback();
|
||||
},
|
||||
})
|
||||
);
|
||||
}
|
||||
return hash.digest("hex");
|
||||
}
|
||||
100
worker/src/archive/multipart.ts
Normal file
100
worker/src/archive/multipart.ts
Normal file
@@ -0,0 +1,100 @@
|
||||
import { detectArchive, type ArchiveFormat, type MultipartInfo } from "./detect.js";
|
||||
import { config } from "../util/config.js";
|
||||
import { childLogger } from "../util/logger.js";
|
||||
|
||||
const log = childLogger("multipart");
|
||||
|
||||
export interface TelegramMessage {
|
||||
id: bigint;
|
||||
fileName: string;
|
||||
fileId: string;
|
||||
fileSize: bigint;
|
||||
date: Date;
|
||||
}
|
||||
|
||||
export interface ArchiveSet {
|
||||
type: ArchiveFormat;
|
||||
baseName: string;
|
||||
parts: TelegramMessage[];
|
||||
isMultipart: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Group messages into archive sets (single files + multipart groups).
|
||||
* Messages should be pre-filtered to only include archive attachments.
|
||||
*/
|
||||
export function groupArchiveSets(messages: TelegramMessage[]): ArchiveSet[] {
|
||||
// Detect and annotate each message
|
||||
const annotated: { msg: TelegramMessage; info: MultipartInfo }[] = [];
|
||||
for (const msg of messages) {
|
||||
const info = detectArchive(msg.fileName);
|
||||
if (info) {
|
||||
annotated.push({ msg, info });
|
||||
}
|
||||
}
|
||||
|
||||
// Group by baseName + format
|
||||
const groups = new Map<string, { msg: TelegramMessage; info: MultipartInfo }[]>();
|
||||
for (const item of annotated) {
|
||||
const key = `${item.info.format}:${item.info.baseName.toLowerCase()}`;
|
||||
const group = groups.get(key) ?? [];
|
||||
group.push(item);
|
||||
groups.set(key, group);
|
||||
}
|
||||
|
||||
const results: ArchiveSet[] = [];
|
||||
|
||||
for (const [, group] of groups) {
|
||||
const format = group[0].info.format;
|
||||
const baseName = group[0].info.baseName;
|
||||
|
||||
// Separate explicit multipart entries from potential singles
|
||||
const multipartEntries = group.filter((g) => g.info.pattern !== "SINGLE");
|
||||
const singleEntries = group.filter((g) => g.info.pattern === "SINGLE");
|
||||
|
||||
if (multipartEntries.length > 0) {
|
||||
// This is a multipart set
|
||||
// Check if any single entry is the "final part" of a legacy split
|
||||
const allEntries = [...multipartEntries, ...singleEntries];
|
||||
|
||||
// Check time span — skip if parts span too long
|
||||
const dates = allEntries.map((e) => e.msg.date.getTime());
|
||||
const span = Math.max(...dates) - Math.min(...dates);
|
||||
const maxSpanMs = config.multipartTimeoutHours * 60 * 60 * 1000;
|
||||
|
||||
if (span > maxSpanMs) {
|
||||
log.warn(
|
||||
{ baseName, format, span: span / 3600000 },
|
||||
"Multipart set spans too long, skipping"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Sort by part number (singles get a very high number so they come last — they're the final part)
|
||||
allEntries.sort((a, b) => {
|
||||
const aNum = a.info.partNumber === -1 ? 999999 : a.info.partNumber;
|
||||
const bNum = b.info.partNumber === -1 ? 999999 : b.info.partNumber;
|
||||
return aNum - bNum;
|
||||
});
|
||||
|
||||
results.push({
|
||||
type: format,
|
||||
baseName,
|
||||
parts: allEntries.map((e) => e.msg),
|
||||
isMultipart: true,
|
||||
});
|
||||
} else {
|
||||
// All entries are singles — each is its own archive set
|
||||
for (const entry of singleEntries) {
|
||||
results.push({
|
||||
type: format,
|
||||
baseName: entry.info.baseName,
|
||||
parts: [entry.msg],
|
||||
isMultipart: false,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
90
worker/src/archive/rar-reader.ts
Normal file
90
worker/src/archive/rar-reader.ts
Normal file
@@ -0,0 +1,90 @@
|
||||
import { execFile } from "child_process";
|
||||
import { promisify } from "util";
|
||||
import path from "path";
|
||||
import { childLogger } from "../util/logger.js";
|
||||
import type { FileEntry } from "./zip-reader.js";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const log = childLogger("rar-reader");
|
||||
|
||||
/**
|
||||
* Parse output of `unrar l -v <file>` to extract file metadata.
|
||||
* unrar automatically discovers sibling parts when they're co-located.
|
||||
*/
|
||||
export async function readRarContents(
|
||||
firstPartPath: string
|
||||
): Promise<FileEntry[]> {
|
||||
try {
|
||||
const { stdout } = await execFileAsync("unrar", ["l", "-v", firstPartPath], {
|
||||
timeout: 30000,
|
||||
maxBuffer: 10 * 1024 * 1024, // 10MB for very large archives
|
||||
});
|
||||
|
||||
return parseUnrarOutput(stdout);
|
||||
} catch (err) {
|
||||
log.warn({ err, file: firstPartPath }, "Failed to read RAR contents");
|
||||
return []; // Fallback: return empty on error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the tabular output of `unrar l -v`.
|
||||
*
|
||||
* Example output format:
|
||||
* Archive: test.rar
|
||||
* Details: RAR 5
|
||||
*
|
||||
* Attributes Size Packed Ratio Date Time CRC-32 Name
|
||||
* ----------- --------- --------- ----- -------- ----- -------- ----
|
||||
* ...A.... 12345 10234 83% 2024-01-15 10:30 DEADBEEF folder/file.stl
|
||||
* ----------- --------- --------- ----- -------- ----- -------- ----
|
||||
*/
|
||||
function parseUnrarOutput(output: string): FileEntry[] {
|
||||
const entries: FileEntry[] = [];
|
||||
const lines = output.split("\n");
|
||||
|
||||
let inFileList = false;
|
||||
let separatorCount = 0;
|
||||
|
||||
for (const line of lines) {
|
||||
const trimmed = line.trim();
|
||||
|
||||
// Detect separator lines (------- pattern)
|
||||
if (/^-{5,}/.test(trimmed)) {
|
||||
separatorCount++;
|
||||
if (separatorCount === 1) {
|
||||
inFileList = true;
|
||||
} else if (separatorCount >= 2) {
|
||||
inFileList = false;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!inFileList) continue;
|
||||
|
||||
// Parse file entry line
|
||||
// Format: Attributes Size Packed Ratio Date Time CRC Name
|
||||
const match = trimmed.match(
|
||||
/^\S+\s+(\d+)\s+(\d+)\s+\d+%\s+\S+\s+\S+\s+([0-9A-Fa-f]+)\s+(.+)$/
|
||||
);
|
||||
|
||||
if (match) {
|
||||
const [, uncompressedStr, compressedStr, crc32, filePath] = match;
|
||||
|
||||
// Skip directory entries (typically end with / or have size 0 with dir attributes)
|
||||
if (filePath.endsWith("/") || filePath.endsWith("\\")) continue;
|
||||
|
||||
const ext = path.extname(filePath).toLowerCase();
|
||||
entries.push({
|
||||
path: filePath,
|
||||
fileName: path.basename(filePath),
|
||||
extension: ext ? ext.slice(1) : null,
|
||||
compressedSize: BigInt(compressedStr),
|
||||
uncompressedSize: BigInt(uncompressedStr),
|
||||
crc32: crc32.toLowerCase(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return entries;
|
||||
}
|
||||
48
worker/src/archive/split.ts
Normal file
48
worker/src/archive/split.ts
Normal file
@@ -0,0 +1,48 @@
|
||||
import { createReadStream, createWriteStream } from "fs";
|
||||
import { stat } from "fs/promises";
|
||||
import path from "path";
|
||||
import { pipeline } from "stream/promises";
|
||||
import { childLogger } from "../util/logger.js";
|
||||
|
||||
const log = childLogger("split");
|
||||
|
||||
/** 2GB in bytes — Telegram's file size limit */
|
||||
const MAX_PART_SIZE = 2n * 1024n * 1024n * 1024n;
|
||||
|
||||
/**
|
||||
* Split a file into ≤2GB parts using byte-level splitting.
|
||||
* Returns paths to the split parts. If the file is already ≤2GB, returns the original path.
|
||||
*/
|
||||
export async function byteLevelSplit(filePath: string): Promise<string[]> {
|
||||
const stats = await stat(filePath);
|
||||
const fileSize = BigInt(stats.size);
|
||||
|
||||
if (fileSize <= MAX_PART_SIZE) {
|
||||
return [filePath];
|
||||
}
|
||||
|
||||
const dir = path.dirname(filePath);
|
||||
const baseName = path.basename(filePath);
|
||||
const partSize = Number(MAX_PART_SIZE);
|
||||
const totalParts = Math.ceil(Number(fileSize) / partSize);
|
||||
const parts: string[] = [];
|
||||
|
||||
log.info({ filePath, fileSize: Number(fileSize), totalParts }, "Splitting file");
|
||||
|
||||
for (let i = 0; i < totalParts; i++) {
|
||||
const partNum = String(i + 1).padStart(3, "0");
|
||||
const partPath = path.join(dir, `${baseName}.${partNum}`);
|
||||
const start = i * partSize;
|
||||
const end = Math.min(start + partSize - 1, Number(fileSize) - 1);
|
||||
|
||||
await pipeline(
|
||||
createReadStream(filePath, { start, end }),
|
||||
createWriteStream(partPath)
|
||||
);
|
||||
|
||||
parts.push(partPath);
|
||||
}
|
||||
|
||||
log.info({ filePath, parts: parts.length }, "File split complete");
|
||||
return parts;
|
||||
}
|
||||
61
worker/src/archive/zip-reader.ts
Normal file
61
worker/src/archive/zip-reader.ts
Normal file
@@ -0,0 +1,61 @@
|
||||
import yauzl from "yauzl";
|
||||
import path from "path";
|
||||
import { childLogger } from "../util/logger.js";
|
||||
|
||||
const log = childLogger("zip-reader");
|
||||
|
||||
export interface FileEntry {
|
||||
path: string;
|
||||
fileName: string;
|
||||
extension: string | null;
|
||||
compressedSize: bigint;
|
||||
uncompressedSize: bigint;
|
||||
crc32: string | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the central directory of a ZIP file without extracting any contents.
|
||||
* For multipart ZIPs, pass the paths sorted by part order.
|
||||
* We attempt to read from the last part first (central directory is at the end).
|
||||
*/
|
||||
export async function readZipCentralDirectory(
|
||||
filePaths: string[]
|
||||
): Promise<FileEntry[]> {
|
||||
// The central directory lives at the end of the last file
|
||||
const targetFile = filePaths[filePaths.length - 1];
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
yauzl.open(targetFile, { lazyEntries: true, autoClose: true }, (err, zipFile) => {
|
||||
if (err) {
|
||||
log.warn({ err, file: targetFile }, "Failed to open ZIP for reading");
|
||||
resolve([]); // Fallback: return empty on error
|
||||
return;
|
||||
}
|
||||
|
||||
const entries: FileEntry[] = [];
|
||||
|
||||
zipFile.readEntry();
|
||||
zipFile.on("entry", (entry: yauzl.Entry) => {
|
||||
// Skip directories
|
||||
if (!entry.fileName.endsWith("/")) {
|
||||
const ext = path.extname(entry.fileName).toLowerCase();
|
||||
entries.push({
|
||||
path: entry.fileName,
|
||||
fileName: path.basename(entry.fileName),
|
||||
extension: ext ? ext.slice(1) : null, // Remove leading dot
|
||||
compressedSize: BigInt(entry.compressedSize),
|
||||
uncompressedSize: BigInt(entry.uncompressedSize),
|
||||
crc32: entry.crc32 !== 0 ? entry.crc32.toString(16).padStart(8, "0") : null,
|
||||
});
|
||||
}
|
||||
zipFile.readEntry();
|
||||
});
|
||||
|
||||
zipFile.on("end", () => resolve(entries));
|
||||
zipFile.on("error", (error) => {
|
||||
log.warn({ error, file: targetFile }, "Error reading ZIP entries");
|
||||
resolve(entries); // Return whatever we got
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
14
worker/src/db/client.ts
Normal file
14
worker/src/db/client.ts
Normal file
@@ -0,0 +1,14 @@
|
||||
import { PrismaClient } from "@prisma/client";
|
||||
import { PrismaPg } from "@prisma/adapter-pg";
|
||||
import pg from "pg";
|
||||
import { config } from "../util/config.js";
|
||||
|
||||
const pool = new pg.Pool({
|
||||
connectionString: config.databaseUrl,
|
||||
max: 5,
|
||||
});
|
||||
|
||||
const adapter = new PrismaPg(pool);
|
||||
export const db = new PrismaClient({ adapter });
|
||||
|
||||
export { pool };
|
||||
56
worker/src/db/locks.ts
Normal file
56
worker/src/db/locks.ts
Normal file
@@ -0,0 +1,56 @@
|
||||
import { pool } from "./client.js";
|
||||
import { childLogger } from "../util/logger.js";
|
||||
|
||||
const log = childLogger("locks");
|
||||
|
||||
/**
|
||||
* Derive a stable 32-bit integer lock ID from an account ID string.
|
||||
* PostgreSQL advisory locks use bigint, but we use 32-bit for safety.
|
||||
*/
|
||||
function hashToLockId(accountId: string): number {
|
||||
let hash = 0;
|
||||
for (let i = 0; i < accountId.length; i++) {
|
||||
const char = accountId.charCodeAt(i);
|
||||
hash = (hash << 5) - hash + char;
|
||||
hash |= 0; // Convert to 32-bit integer
|
||||
}
|
||||
return Math.abs(hash);
|
||||
}
|
||||
|
||||
/**
|
||||
* Try to acquire a PostgreSQL advisory lock for an account.
|
||||
* Returns true if acquired, false if already held by another session.
|
||||
*/
|
||||
export async function tryAcquireLock(accountId: string): Promise<boolean> {
|
||||
const lockId = hashToLockId(accountId);
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
const result = await client.query<{ pg_try_advisory_lock: boolean }>(
|
||||
"SELECT pg_try_advisory_lock($1)",
|
||||
[lockId]
|
||||
);
|
||||
const acquired = result.rows[0]?.pg_try_advisory_lock ?? false;
|
||||
if (acquired) {
|
||||
log.debug({ accountId, lockId }, "Advisory lock acquired");
|
||||
} else {
|
||||
log.debug({ accountId, lockId }, "Advisory lock already held");
|
||||
}
|
||||
return acquired;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Release the advisory lock for an account.
|
||||
*/
|
||||
export async function releaseLock(accountId: string): Promise<void> {
|
||||
const lockId = hashToLockId(accountId);
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
await client.query("SELECT pg_advisory_unlock($1)", [lockId]);
|
||||
log.debug({ accountId, lockId }, "Advisory lock released");
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
270
worker/src/db/queries.ts
Normal file
270
worker/src/db/queries.ts
Normal file
@@ -0,0 +1,270 @@
|
||||
import { db } from "./client.js";
|
||||
import type { ArchiveType } from "@prisma/client";
|
||||
|
||||
export async function getActiveAccounts() {
|
||||
return db.telegramAccount.findMany({
|
||||
where: { isActive: true, authState: "AUTHENTICATED" },
|
||||
});
|
||||
}
|
||||
|
||||
export async function getSourceChannelMappings(accountId: string) {
|
||||
return db.accountChannelMap.findMany({
|
||||
where: {
|
||||
accountId,
|
||||
role: "READER",
|
||||
channel: { type: "SOURCE", isActive: true },
|
||||
},
|
||||
include: { channel: true },
|
||||
});
|
||||
}
|
||||
|
||||
export async function getDestinationChannel(accountId: string) {
|
||||
const mapping = await db.accountChannelMap.findFirst({
|
||||
where: {
|
||||
accountId,
|
||||
role: "WRITER",
|
||||
channel: { type: "DESTINATION", isActive: true },
|
||||
},
|
||||
include: { channel: true },
|
||||
});
|
||||
return mapping?.channel ?? null;
|
||||
}
|
||||
|
||||
export async function packageExistsByHash(contentHash: string) {
|
||||
const pkg = await db.package.findUnique({
|
||||
where: { contentHash },
|
||||
select: { id: true },
|
||||
});
|
||||
return pkg !== null;
|
||||
}
|
||||
|
||||
export interface CreatePackageInput {
|
||||
contentHash: string;
|
||||
fileName: string;
|
||||
fileSize: bigint;
|
||||
archiveType: ArchiveType;
|
||||
sourceChannelId: string;
|
||||
sourceMessageId: bigint;
|
||||
sourceTopicId?: bigint | null;
|
||||
destChannelId?: string;
|
||||
destMessageId?: bigint;
|
||||
isMultipart: boolean;
|
||||
partCount: number;
|
||||
ingestionRunId: string;
|
||||
creator?: string | null;
|
||||
previewData?: Buffer | null;
|
||||
previewMsgId?: bigint | null;
|
||||
files: {
|
||||
path: string;
|
||||
fileName: string;
|
||||
extension: string | null;
|
||||
compressedSize: bigint;
|
||||
uncompressedSize: bigint;
|
||||
crc32: string | null;
|
||||
}[];
|
||||
}
|
||||
|
||||
export async function createPackageWithFiles(input: CreatePackageInput) {
|
||||
return db.package.create({
|
||||
data: {
|
||||
contentHash: input.contentHash,
|
||||
fileName: input.fileName,
|
||||
fileSize: input.fileSize,
|
||||
archiveType: input.archiveType,
|
||||
sourceChannelId: input.sourceChannelId,
|
||||
sourceMessageId: input.sourceMessageId,
|
||||
sourceTopicId: input.sourceTopicId ?? undefined,
|
||||
destChannelId: input.destChannelId,
|
||||
destMessageId: input.destMessageId,
|
||||
isMultipart: input.isMultipart,
|
||||
partCount: input.partCount,
|
||||
fileCount: input.files.length,
|
||||
ingestionRunId: input.ingestionRunId,
|
||||
creator: input.creator ?? undefined,
|
||||
previewData: input.previewData ? new Uint8Array(input.previewData) : undefined,
|
||||
previewMsgId: input.previewMsgId ?? undefined,
|
||||
files: {
|
||||
create: input.files,
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export async function createIngestionRun(accountId: string) {
|
||||
return db.ingestionRun.create({
|
||||
data: {
|
||||
accountId,
|
||||
status: "RUNNING",
|
||||
currentActivity: "Starting ingestion run",
|
||||
currentStep: "initializing",
|
||||
lastActivityAt: new Date(),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export interface ActivityUpdate {
|
||||
currentActivity: string;
|
||||
currentStep: string;
|
||||
currentChannel?: string | null;
|
||||
currentFile?: string | null;
|
||||
currentFileNum?: number | null;
|
||||
totalFiles?: number | null;
|
||||
downloadedBytes?: bigint | null;
|
||||
totalBytes?: bigint | null;
|
||||
downloadPercent?: number | null;
|
||||
messagesScanned?: number;
|
||||
zipsFound?: number;
|
||||
zipsDuplicate?: number;
|
||||
zipsIngested?: number;
|
||||
}
|
||||
|
||||
export async function updateRunActivity(
|
||||
runId: string,
|
||||
activity: ActivityUpdate
|
||||
) {
|
||||
return db.ingestionRun.update({
|
||||
where: { id: runId },
|
||||
data: {
|
||||
currentActivity: activity.currentActivity,
|
||||
currentStep: activity.currentStep,
|
||||
currentChannel: activity.currentChannel ?? undefined,
|
||||
currentFile: activity.currentFile ?? undefined,
|
||||
currentFileNum: activity.currentFileNum ?? undefined,
|
||||
totalFiles: activity.totalFiles ?? undefined,
|
||||
downloadedBytes: activity.downloadedBytes ?? undefined,
|
||||
totalBytes: activity.totalBytes ?? undefined,
|
||||
downloadPercent: activity.downloadPercent ?? undefined,
|
||||
lastActivityAt: new Date(),
|
||||
...(activity.messagesScanned !== undefined && { messagesScanned: activity.messagesScanned }),
|
||||
...(activity.zipsFound !== undefined && { zipsFound: activity.zipsFound }),
|
||||
...(activity.zipsDuplicate !== undefined && { zipsDuplicate: activity.zipsDuplicate }),
|
||||
...(activity.zipsIngested !== undefined && { zipsIngested: activity.zipsIngested }),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
const CLEAR_ACTIVITY = {
|
||||
currentActivity: null,
|
||||
currentStep: null,
|
||||
currentChannel: null,
|
||||
currentFile: null,
|
||||
currentFileNum: null,
|
||||
totalFiles: null,
|
||||
downloadedBytes: null,
|
||||
totalBytes: null,
|
||||
downloadPercent: null,
|
||||
lastActivityAt: new Date(),
|
||||
};
|
||||
|
||||
export async function completeIngestionRun(
|
||||
runId: string,
|
||||
counters: {
|
||||
messagesScanned: number;
|
||||
zipsFound: number;
|
||||
zipsDuplicate: number;
|
||||
zipsIngested: number;
|
||||
}
|
||||
) {
|
||||
return db.ingestionRun.update({
|
||||
where: { id: runId },
|
||||
data: {
|
||||
status: "COMPLETED",
|
||||
finishedAt: new Date(),
|
||||
...counters,
|
||||
...CLEAR_ACTIVITY,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export async function failIngestionRun(runId: string, errorMessage: string) {
|
||||
return db.ingestionRun.update({
|
||||
where: { id: runId },
|
||||
data: {
|
||||
status: "FAILED",
|
||||
finishedAt: new Date(),
|
||||
errorMessage,
|
||||
...CLEAR_ACTIVITY,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export async function updateLastProcessedMessage(
|
||||
mappingId: string,
|
||||
messageId: bigint
|
||||
) {
|
||||
return db.accountChannelMap.update({
|
||||
where: { id: mappingId },
|
||||
data: { lastProcessedMessageId: messageId },
|
||||
});
|
||||
}
|
||||
|
||||
export async function markStaleRunsAsFailed() {
|
||||
return db.ingestionRun.updateMany({
|
||||
where: { status: "RUNNING" },
|
||||
data: {
|
||||
status: "FAILED",
|
||||
finishedAt: new Date(),
|
||||
errorMessage: "Worker restarted — run was still marked as RUNNING",
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export async function updateAccountAuthState(
|
||||
accountId: string,
|
||||
authState: "PENDING" | "AWAITING_CODE" | "AWAITING_PASSWORD" | "AUTHENTICATED" | "EXPIRED",
|
||||
authCode?: string | null
|
||||
) {
|
||||
return db.telegramAccount.update({
|
||||
where: { id: accountId },
|
||||
data: { authState, authCode, lastSeenAt: authState === "AUTHENTICATED" ? new Date() : undefined },
|
||||
});
|
||||
}
|
||||
|
||||
export async function getAccountAuthCode(accountId: string) {
|
||||
const account = await db.telegramAccount.findUnique({
|
||||
where: { id: accountId },
|
||||
select: { authCode: true, authState: true },
|
||||
});
|
||||
return account;
|
||||
}
|
||||
|
||||
// ── Forum / Topic progress ──
|
||||
|
||||
export async function setChannelForum(channelId: string, isForum: boolean) {
|
||||
return db.telegramChannel.update({
|
||||
where: { id: channelId },
|
||||
data: { isForum },
|
||||
});
|
||||
}
|
||||
|
||||
export async function getTopicProgress(mappingId: string) {
|
||||
return db.topicProgress.findMany({
|
||||
where: { accountChannelMapId: mappingId },
|
||||
});
|
||||
}
|
||||
|
||||
export async function upsertTopicProgress(
|
||||
mappingId: string,
|
||||
topicId: bigint,
|
||||
topicName: string | null,
|
||||
lastProcessedMessageId: bigint
|
||||
) {
|
||||
return db.topicProgress.upsert({
|
||||
where: {
|
||||
accountChannelMapId_topicId: {
|
||||
accountChannelMapId: mappingId,
|
||||
topicId,
|
||||
},
|
||||
},
|
||||
create: {
|
||||
accountChannelMapId: mappingId,
|
||||
topicId,
|
||||
topicName,
|
||||
lastProcessedMessageId,
|
||||
},
|
||||
update: {
|
||||
topicName,
|
||||
lastProcessedMessageId,
|
||||
},
|
||||
});
|
||||
}
|
||||
50
worker/src/index.ts
Normal file
50
worker/src/index.ts
Normal file
@@ -0,0 +1,50 @@
|
||||
import { mkdir } from "fs/promises";
|
||||
import { config } from "./util/config.js";
|
||||
import { logger } from "./util/logger.js";
|
||||
import { markStaleRunsAsFailed } from "./db/queries.js";
|
||||
import { cleanupTempDir } from "./worker.js";
|
||||
import { startScheduler, stopScheduler } from "./scheduler.js";
|
||||
import { db, pool } from "./db/client.js";
|
||||
|
||||
const log = logger.child({ module: "main" });
|
||||
|
||||
async function main(): Promise<void> {
|
||||
log.info("DragonsStash Telegram Worker starting");
|
||||
log.info({ config: { ...config, databaseUrl: "***" } }, "Configuration loaded");
|
||||
|
||||
// Ensure temp directory exists
|
||||
await mkdir(config.tempDir, { recursive: true });
|
||||
await mkdir(config.tdlibStateDir, { recursive: true });
|
||||
|
||||
// Clean up stale state
|
||||
await cleanupTempDir();
|
||||
await markStaleRunsAsFailed();
|
||||
|
||||
// Start the scheduler
|
||||
await startScheduler();
|
||||
}
|
||||
|
||||
// Graceful shutdown
|
||||
function shutdown(signal: string): void {
|
||||
log.info({ signal }, "Shutdown signal received");
|
||||
stopScheduler();
|
||||
|
||||
// Close DB connections
|
||||
Promise.all([db.$disconnect(), pool.end()])
|
||||
.then(() => {
|
||||
log.info("Shutdown complete");
|
||||
process.exit(0);
|
||||
})
|
||||
.catch((err) => {
|
||||
log.error({ err }, "Error during shutdown");
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
|
||||
process.on("SIGTERM", () => shutdown("SIGTERM"));
|
||||
process.on("SIGINT", () => shutdown("SIGINT"));
|
||||
|
||||
main().catch((err) => {
|
||||
log.fatal({ err }, "Worker failed to start");
|
||||
process.exit(1);
|
||||
});
|
||||
86
worker/src/preview/match.ts
Normal file
86
worker/src/preview/match.ts
Normal file
@@ -0,0 +1,86 @@
|
||||
import { childLogger } from "../util/logger.js";
|
||||
|
||||
const log = childLogger("preview-match");
|
||||
|
||||
export interface TelegramPhoto {
|
||||
id: bigint;
|
||||
date: Date;
|
||||
/** Caption text on the photo message (if any). */
|
||||
caption: string;
|
||||
/** The smallest photo size available — used as thumbnail. */
|
||||
fileId: string;
|
||||
fileSize: number;
|
||||
}
|
||||
|
||||
export interface ArchiveRef {
|
||||
baseName: string;
|
||||
firstMessageId: bigint;
|
||||
firstMessageDate: Date;
|
||||
}
|
||||
|
||||
/**
|
||||
* Try to match a photo message to an archive by:
|
||||
* 1. Caption contains the archive baseName (without extension)
|
||||
* 2. Photo was posted within ±10 messages (time-window: ±6 hours)
|
||||
*
|
||||
* Returns the best match (closest in time), or null.
|
||||
*/
|
||||
export function matchPreviewToArchive(
|
||||
photos: TelegramPhoto[],
|
||||
archives: ArchiveRef[]
|
||||
): Map<string, TelegramPhoto> {
|
||||
const results = new Map<string, TelegramPhoto>();
|
||||
const TIME_WINDOW_MS = 6 * 60 * 60 * 1000; // 6 hours
|
||||
|
||||
for (const archive of archives) {
|
||||
// Normalize the archive base name for matching
|
||||
const normalizedBase = normalizeForMatch(archive.baseName);
|
||||
if (!normalizedBase) continue;
|
||||
|
||||
let bestMatch: TelegramPhoto | null = null;
|
||||
let bestTimeDiff = Infinity;
|
||||
|
||||
for (const photo of photos) {
|
||||
const timeDiff = Math.abs(
|
||||
photo.date.getTime() - archive.firstMessageDate.getTime()
|
||||
);
|
||||
|
||||
// Must be within time window
|
||||
if (timeDiff > TIME_WINDOW_MS) continue;
|
||||
|
||||
// Check if the photo caption contains the archive base name
|
||||
const normalizedCaption = normalizeForMatch(photo.caption);
|
||||
if (!normalizedCaption) continue;
|
||||
|
||||
const matches =
|
||||
normalizedCaption.includes(normalizedBase) ||
|
||||
normalizedBase.includes(normalizedCaption);
|
||||
|
||||
if (matches && timeDiff < bestTimeDiff) {
|
||||
bestMatch = photo;
|
||||
bestTimeDiff = timeDiff;
|
||||
}
|
||||
}
|
||||
|
||||
if (bestMatch) {
|
||||
log.debug(
|
||||
{ baseName: archive.baseName, photoId: bestMatch.id.toString() },
|
||||
"Matched preview photo to archive"
|
||||
);
|
||||
results.set(archive.baseName, bestMatch);
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Strip extension, punctuation, and normalize for fuzzy matching.
|
||||
*/
|
||||
function normalizeForMatch(input: string): string {
|
||||
return input
|
||||
.toLowerCase()
|
||||
.replace(/\.[a-z0-9]{1,5}$/i, "") // strip extension
|
||||
.replace(/[_\-.\s]+/g, " ") // normalize separators
|
||||
.trim();
|
||||
}
|
||||
92
worker/src/scheduler.ts
Normal file
92
worker/src/scheduler.ts
Normal file
@@ -0,0 +1,92 @@
|
||||
import { config } from "./util/config.js";
|
||||
import { childLogger } from "./util/logger.js";
|
||||
import { getActiveAccounts } from "./db/queries.js";
|
||||
import { runWorkerForAccount } from "./worker.js";
|
||||
|
||||
const log = childLogger("scheduler");
|
||||
|
||||
let running = false;
|
||||
let timer: ReturnType<typeof setTimeout> | null = null;
|
||||
|
||||
/**
|
||||
* Run one ingestion cycle: process all active, authenticated accounts sequentially.
|
||||
*/
|
||||
async function runCycle(): Promise<void> {
|
||||
if (running) {
|
||||
log.warn("Previous cycle still running, skipping");
|
||||
return;
|
||||
}
|
||||
|
||||
running = true;
|
||||
log.info("Starting ingestion cycle");
|
||||
|
||||
try {
|
||||
const accounts = await getActiveAccounts();
|
||||
|
||||
if (accounts.length === 0) {
|
||||
log.info("No active authenticated accounts, nothing to do");
|
||||
return;
|
||||
}
|
||||
|
||||
log.info({ accountCount: accounts.length }, "Processing accounts");
|
||||
|
||||
for (const account of accounts) {
|
||||
await runWorkerForAccount(account);
|
||||
}
|
||||
|
||||
log.info("Ingestion cycle complete");
|
||||
} catch (err) {
|
||||
log.error({ err }, "Ingestion cycle failed");
|
||||
} finally {
|
||||
running = false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Schedule the next cycle with jitter.
|
||||
*/
|
||||
function scheduleNext(): void {
|
||||
const intervalMs = config.workerIntervalMinutes * 60 * 1000;
|
||||
const jitterMs = Math.random() * config.jitterMinutes * 60 * 1000;
|
||||
const delay = intervalMs + jitterMs;
|
||||
|
||||
log.info(
|
||||
{ nextRunInMinutes: Math.round(delay / 60000) },
|
||||
"Next cycle scheduled"
|
||||
);
|
||||
|
||||
timer = setTimeout(async () => {
|
||||
await runCycle();
|
||||
scheduleNext();
|
||||
}, delay);
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the scheduler. Runs an immediate first cycle, then schedules subsequent ones.
|
||||
*/
|
||||
export async function startScheduler(): Promise<void> {
|
||||
log.info(
|
||||
{
|
||||
intervalMinutes: config.workerIntervalMinutes,
|
||||
jitterMinutes: config.jitterMinutes,
|
||||
},
|
||||
"Scheduler starting"
|
||||
);
|
||||
|
||||
// Run immediately on start
|
||||
await runCycle();
|
||||
|
||||
// Then schedule recurring cycles
|
||||
scheduleNext();
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop the scheduler gracefully.
|
||||
*/
|
||||
export function stopScheduler(): void {
|
||||
if (timer) {
|
||||
clearTimeout(timer);
|
||||
timer = null;
|
||||
}
|
||||
log.info("Scheduler stopped");
|
||||
}
|
||||
120
worker/src/tdlib/client.ts
Normal file
120
worker/src/tdlib/client.ts
Normal file
@@ -0,0 +1,120 @@
|
||||
import tdl, { createClient, type Client } from "tdl";
|
||||
import { getTdjson } from "prebuilt-tdlib";
|
||||
import path from "path";
|
||||
import { config } from "../util/config.js";
|
||||
import { childLogger } from "../util/logger.js";
|
||||
import {
|
||||
updateAccountAuthState,
|
||||
getAccountAuthCode,
|
||||
} from "../db/queries.js";
|
||||
|
||||
const log = childLogger("tdlib-client");
|
||||
|
||||
// Configure tdl to use the prebuilt tdjson shared library
|
||||
tdl.configure({ tdjson: getTdjson() });
|
||||
|
||||
interface AccountConfig {
|
||||
id: string;
|
||||
phone: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create and authenticate a TDLib client for a Telegram account.
|
||||
* Authentication flow communicates with the admin UI via the database:
|
||||
* - Worker sets authState to AWAITING_CODE when TDLib asks for phone code
|
||||
* - Admin enters the code via UI, which writes it to authCode field
|
||||
* - Worker polls DB for the code and feeds it to TDLib
|
||||
*/
|
||||
export async function createTdlibClient(
|
||||
account: AccountConfig
|
||||
): Promise<Client> {
|
||||
const dbPath = path.join(config.tdlibStateDir, account.id);
|
||||
|
||||
const client = createClient({
|
||||
apiId: config.telegramApiId,
|
||||
apiHash: config.telegramApiHash,
|
||||
databaseDirectory: dbPath,
|
||||
filesDirectory: path.join(dbPath, "files"),
|
||||
});
|
||||
|
||||
client.on("error", (err) => {
|
||||
log.error({ err, accountId: account.id }, "TDLib client error");
|
||||
});
|
||||
|
||||
try {
|
||||
await client.login(() => ({
|
||||
getPhoneNumber: async () => {
|
||||
log.info({ accountId: account.id }, "TDLib requesting phone number");
|
||||
return account.phone;
|
||||
},
|
||||
getAuthCode: async () => {
|
||||
log.info({ accountId: account.id }, "TDLib requesting auth code");
|
||||
await updateAccountAuthState(account.id, "AWAITING_CODE");
|
||||
|
||||
// Poll database for the code entered via admin UI
|
||||
const code = await pollForAuthCode(account.id);
|
||||
if (!code) {
|
||||
throw new Error("Auth code not provided within timeout");
|
||||
}
|
||||
|
||||
// Clear the code after reading
|
||||
await updateAccountAuthState(account.id, "AUTHENTICATED", null);
|
||||
return code;
|
||||
},
|
||||
getPassword: async () => {
|
||||
log.info({ accountId: account.id }, "TDLib requesting 2FA password");
|
||||
await updateAccountAuthState(account.id, "AWAITING_PASSWORD");
|
||||
|
||||
// Poll database for the password entered via admin UI
|
||||
const code = await pollForAuthCode(account.id);
|
||||
if (!code) {
|
||||
throw new Error("2FA password not provided within timeout");
|
||||
}
|
||||
|
||||
await updateAccountAuthState(account.id, "AUTHENTICATED", null);
|
||||
return code;
|
||||
},
|
||||
}));
|
||||
|
||||
await updateAccountAuthState(account.id, "AUTHENTICATED");
|
||||
log.info({ accountId: account.id }, "TDLib client authenticated");
|
||||
return client;
|
||||
} catch (err) {
|
||||
log.error({ err, accountId: account.id }, "TDLib authentication failed");
|
||||
await updateAccountAuthState(account.id, "EXPIRED");
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Poll the database every 5 seconds for an auth code, up to 5 minutes.
|
||||
*/
|
||||
async function pollForAuthCode(
|
||||
accountId: string,
|
||||
timeoutMs = 300_000
|
||||
): Promise<string | null> {
|
||||
const start = Date.now();
|
||||
while (Date.now() - start < timeoutMs) {
|
||||
const result = await getAccountAuthCode(accountId);
|
||||
if (result?.authCode) {
|
||||
return result.authCode;
|
||||
}
|
||||
await sleep(5000);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
/**
|
||||
* Close a TDLib client gracefully.
|
||||
*/
|
||||
export async function closeTdlibClient(client: Client): Promise<void> {
|
||||
try {
|
||||
await client.close();
|
||||
} catch (err) {
|
||||
log.warn({ err }, "Error closing TDLib client");
|
||||
}
|
||||
}
|
||||
389
worker/src/tdlib/download.ts
Normal file
389
worker/src/tdlib/download.ts
Normal file
@@ -0,0 +1,389 @@
|
||||
import type { Client } from "tdl";
|
||||
import { readFile, rename, stat } from "fs/promises";
|
||||
import { config } from "../util/config.js";
|
||||
import { childLogger } from "../util/logger.js";
|
||||
import { isArchiveAttachment } from "../archive/detect.js";
|
||||
import type { TelegramMessage } from "../archive/multipart.js";
|
||||
import type { TelegramPhoto } from "../preview/match.js";
|
||||
|
||||
const log = childLogger("download");
|
||||
|
||||
interface TdPhotoSize {
|
||||
type: string;
|
||||
photo: {
|
||||
id: number;
|
||||
size: number;
|
||||
expected_size: number;
|
||||
local?: {
|
||||
path?: string;
|
||||
is_downloading_active?: boolean;
|
||||
is_downloading_completed?: boolean;
|
||||
downloaded_size?: number;
|
||||
};
|
||||
};
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
interface TdMessage {
|
||||
id: number;
|
||||
date: number;
|
||||
content: {
|
||||
_: string;
|
||||
document?: {
|
||||
file_name?: string;
|
||||
document?: {
|
||||
id: number;
|
||||
size: number;
|
||||
local?: {
|
||||
path?: string;
|
||||
is_downloading_completed?: boolean;
|
||||
};
|
||||
};
|
||||
};
|
||||
photo?: {
|
||||
sizes?: TdPhotoSize[];
|
||||
};
|
||||
caption?: {
|
||||
text?: string;
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
interface TdFile {
|
||||
id: number;
|
||||
size: number;
|
||||
expected_size: number;
|
||||
local: {
|
||||
path: string;
|
||||
is_downloading_active: boolean;
|
||||
is_downloading_completed: boolean;
|
||||
downloaded_size: number;
|
||||
download_offset: number;
|
||||
};
|
||||
}
|
||||
|
||||
export interface ChannelScanResult {
|
||||
archives: TelegramMessage[];
|
||||
photos: TelegramPhoto[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch messages from a channel since a given message ID.
|
||||
* Collects both archive attachments AND photo messages (for preview matching).
|
||||
* Returns messages in chronological order (oldest first).
|
||||
*/
|
||||
export async function getChannelMessages(
|
||||
client: Client,
|
||||
chatId: bigint,
|
||||
fromMessageId?: bigint | null,
|
||||
limit = 100
|
||||
): Promise<ChannelScanResult> {
|
||||
const archives: TelegramMessage[] = [];
|
||||
const photos: TelegramPhoto[] = [];
|
||||
let currentFromId = fromMessageId ? Number(fromMessageId) : 0;
|
||||
|
||||
// eslint-disable-next-line no-constant-condition
|
||||
while (true) {
|
||||
const result = (await client.invoke({
|
||||
_: "getChatHistory",
|
||||
chat_id: Number(chatId),
|
||||
from_message_id: currentFromId,
|
||||
offset: 0,
|
||||
limit: Math.min(limit, 100),
|
||||
only_local: false,
|
||||
})) as { messages: TdMessage[] };
|
||||
|
||||
if (!result.messages || result.messages.length === 0) break;
|
||||
|
||||
for (const msg of result.messages) {
|
||||
// Check for archive documents
|
||||
const doc = msg.content?.document;
|
||||
if (doc?.file_name && doc.document && isArchiveAttachment(doc.file_name)) {
|
||||
archives.push({
|
||||
id: BigInt(msg.id),
|
||||
fileName: doc.file_name,
|
||||
fileId: String(doc.document.id),
|
||||
fileSize: BigInt(doc.document.size),
|
||||
date: new Date(msg.date * 1000),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check for photo messages (potential previews)
|
||||
const photo = msg.content?.photo;
|
||||
const caption = msg.content?.caption?.text ?? "";
|
||||
if (photo?.sizes && photo.sizes.length > 0) {
|
||||
// Pick the smallest size for thumbnail (type "s" or "m")
|
||||
// TDLib photo sizes are ordered from smallest to largest
|
||||
const smallest = photo.sizes[0];
|
||||
photos.push({
|
||||
id: BigInt(msg.id),
|
||||
date: new Date(msg.date * 1000),
|
||||
caption,
|
||||
fileId: String(smallest.photo.id),
|
||||
fileSize: smallest.photo.size || smallest.photo.expected_size,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
currentFromId = result.messages[result.messages.length - 1].id;
|
||||
if (result.messages.length < 100) break;
|
||||
|
||||
// Rate limit delay
|
||||
await sleep(config.apiDelayMs);
|
||||
}
|
||||
|
||||
// Return in chronological order (oldest first)
|
||||
return {
|
||||
archives: archives.reverse(),
|
||||
photos: photos.reverse(),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Download a photo thumbnail from Telegram and return its raw bytes.
|
||||
* Uses synchronous download (photos are small, typically < 100KB).
|
||||
* Returns null if download fails (non-critical).
|
||||
*/
|
||||
export async function downloadPhotoThumbnail(
|
||||
client: Client,
|
||||
fileId: string
|
||||
): Promise<Buffer | null> {
|
||||
const numericId = parseInt(fileId, 10);
|
||||
|
||||
try {
|
||||
const result = (await client.invoke({
|
||||
_: "downloadFile",
|
||||
file_id: numericId,
|
||||
priority: 1, // Low priority — thumbnails are nice-to-have
|
||||
offset: 0,
|
||||
limit: 0,
|
||||
synchronous: true, // Small file — wait for it
|
||||
})) as TdFile;
|
||||
|
||||
if (result?.local?.is_downloading_completed && result.local.path) {
|
||||
const data = await readFile(result.local.path);
|
||||
log.debug(
|
||||
{ fileId, bytes: data.length },
|
||||
"Downloaded photo thumbnail"
|
||||
);
|
||||
return data;
|
||||
}
|
||||
} catch (err) {
|
||||
log.warn({ fileId, err }, "Failed to download photo thumbnail");
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
export interface DownloadProgress {
|
||||
fileId: string;
|
||||
fileName: string;
|
||||
downloadedBytes: number;
|
||||
totalBytes: number;
|
||||
percent: number;
|
||||
isComplete: boolean;
|
||||
}
|
||||
|
||||
export type ProgressCallback = (progress: DownloadProgress) => void;
|
||||
|
||||
/**
|
||||
* Download a file from Telegram to a local path with progress tracking
|
||||
* and integrity verification.
|
||||
*
|
||||
* Progress flow:
|
||||
* 1. Starts async download via TDLib
|
||||
* 2. Listens for `updateFile` events to track download progress
|
||||
* 3. Logs progress at every 10% increment
|
||||
* 4. Once complete, verifies the local file size matches the expected size
|
||||
* 5. Moves the file from TDLib's cache to the destination path
|
||||
*
|
||||
* Verification:
|
||||
* - Compares actual file size on disk to the expected size from Telegram
|
||||
* - Throws on mismatch (partial/corrupt download)
|
||||
* - Throws on timeout (configurable, scales with file size)
|
||||
* - Throws if download stops without completing (network error, etc.)
|
||||
*/
|
||||
export async function downloadFile(
|
||||
client: Client,
|
||||
fileId: string,
|
||||
destPath: string,
|
||||
expectedSize: bigint,
|
||||
fileName: string,
|
||||
onProgress?: ProgressCallback
|
||||
): Promise<void> {
|
||||
const numericId = parseInt(fileId, 10);
|
||||
const totalBytes = Number(expectedSize);
|
||||
|
||||
log.info(
|
||||
{ fileId, fileName, destPath, totalBytes },
|
||||
"Starting file download"
|
||||
);
|
||||
|
||||
// Report initial progress
|
||||
onProgress?.({
|
||||
fileId,
|
||||
fileName,
|
||||
downloadedBytes: 0,
|
||||
totalBytes,
|
||||
percent: 0,
|
||||
isComplete: false,
|
||||
});
|
||||
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
let lastLoggedPercent = 0;
|
||||
let settled = false;
|
||||
|
||||
// Timeout: 10 minutes per GB, minimum 5 minutes
|
||||
const timeoutMs = Math.max(
|
||||
5 * 60_000,
|
||||
(totalBytes / (1024 * 1024 * 1024)) * 10 * 60_000
|
||||
);
|
||||
const timer = setTimeout(() => {
|
||||
if (!settled) {
|
||||
settled = true;
|
||||
cleanup();
|
||||
reject(
|
||||
new Error(
|
||||
`Download timed out after ${Math.round(timeoutMs / 60_000)}min for ${fileName}`
|
||||
)
|
||||
);
|
||||
}
|
||||
}, timeoutMs);
|
||||
|
||||
// Listen for file update events to track progress
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const handleUpdate = (update: any) => {
|
||||
if (update?._ !== "updateFile") return;
|
||||
const file = update.file as TdFile | undefined;
|
||||
if (!file || file.id !== numericId) return;
|
||||
|
||||
const downloaded = file.local.downloaded_size;
|
||||
const percent =
|
||||
totalBytes > 0 ? Math.round((downloaded / totalBytes) * 100) : 0;
|
||||
|
||||
// Log at every 10% increment
|
||||
if (percent >= lastLoggedPercent + 10) {
|
||||
lastLoggedPercent = percent - (percent % 10);
|
||||
log.info(
|
||||
{ fileId, fileName, downloaded, totalBytes, percent: `${percent}%` },
|
||||
"Download progress"
|
||||
);
|
||||
}
|
||||
|
||||
// Report to callback
|
||||
onProgress?.({
|
||||
fileId,
|
||||
fileName,
|
||||
downloadedBytes: downloaded,
|
||||
totalBytes,
|
||||
percent,
|
||||
isComplete: file.local.is_downloading_completed,
|
||||
});
|
||||
|
||||
// Download finished
|
||||
if (file.local.is_downloading_completed) {
|
||||
if (!settled) {
|
||||
settled = true;
|
||||
cleanup();
|
||||
verifyAndMove(file.local.path, destPath, totalBytes, fileName, fileId)
|
||||
.then(resolve)
|
||||
.catch(reject);
|
||||
}
|
||||
}
|
||||
|
||||
// Download stopped without completing (network error, cancelled, etc.)
|
||||
if (
|
||||
!file.local.is_downloading_active &&
|
||||
!file.local.is_downloading_completed
|
||||
) {
|
||||
if (!settled) {
|
||||
settled = true;
|
||||
cleanup();
|
||||
reject(
|
||||
new Error(
|
||||
`Download stopped unexpectedly for ${fileName} ` +
|
||||
`(${downloaded}/${totalBytes} bytes, ${percent}%)`
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const cleanup = () => {
|
||||
clearTimeout(timer);
|
||||
client.off("update", handleUpdate);
|
||||
};
|
||||
|
||||
// Subscribe to updates BEFORE starting download
|
||||
client.on("update", handleUpdate);
|
||||
|
||||
// Start async download (non-blocking — progress via updateFile events)
|
||||
client
|
||||
.invoke({
|
||||
_: "downloadFile",
|
||||
file_id: numericId,
|
||||
priority: 32,
|
||||
offset: 0,
|
||||
limit: 0,
|
||||
synchronous: false,
|
||||
})
|
||||
.then((result: unknown) => {
|
||||
// If the file was already cached locally, invoke returns immediately
|
||||
const file = result as TdFile | undefined;
|
||||
if (file?.local?.is_downloading_completed && !settled) {
|
||||
settled = true;
|
||||
cleanup();
|
||||
verifyAndMove(file.local.path, destPath, totalBytes, fileName, fileId)
|
||||
.then(resolve)
|
||||
.catch(reject);
|
||||
}
|
||||
})
|
||||
.catch((err: unknown) => {
|
||||
if (!settled) {
|
||||
settled = true;
|
||||
cleanup();
|
||||
reject(err);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify the downloaded file's size matches the expected size,
|
||||
* then move it to the destination path.
|
||||
*/
|
||||
async function verifyAndMove(
|
||||
localPath: string,
|
||||
destPath: string,
|
||||
expectedBytes: number,
|
||||
fileName: string,
|
||||
fileId: string
|
||||
): Promise<void> {
|
||||
const stats = await stat(localPath);
|
||||
const actualBytes = stats.size;
|
||||
|
||||
if (expectedBytes > 0 && actualBytes !== expectedBytes) {
|
||||
log.error(
|
||||
{ fileId, fileName, expectedBytes, actualBytes },
|
||||
"Download size mismatch — file is incomplete or corrupted"
|
||||
);
|
||||
throw new Error(
|
||||
`Download verification failed for ${fileName}: ` +
|
||||
`expected ${expectedBytes} bytes, got ${actualBytes} bytes`
|
||||
);
|
||||
}
|
||||
|
||||
log.info(
|
||||
{ fileId, fileName, bytes: actualBytes, destPath },
|
||||
"File verified and complete"
|
||||
);
|
||||
|
||||
// Move from TDLib's cache to our temp directory
|
||||
await rename(localPath, destPath);
|
||||
}
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
222
worker/src/tdlib/topics.ts
Normal file
222
worker/src/tdlib/topics.ts
Normal file
@@ -0,0 +1,222 @@
|
||||
import type { Client } from "tdl";
|
||||
import { config } from "../util/config.js";
|
||||
import { childLogger } from "../util/logger.js";
|
||||
import { isArchiveAttachment } from "../archive/detect.js";
|
||||
import type { TelegramMessage } from "../archive/multipart.js";
|
||||
import type { TelegramPhoto } from "../preview/match.js";
|
||||
import type { ChannelScanResult } from "./download.js";
|
||||
|
||||
const log = childLogger("topics");
|
||||
|
||||
export interface ForumTopic {
|
||||
topicId: bigint;
|
||||
name: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a chat is a forum supergroup (topics enabled).
|
||||
*/
|
||||
export async function isChatForum(
|
||||
client: Client,
|
||||
chatId: bigint
|
||||
): Promise<boolean> {
|
||||
try {
|
||||
const chat = (await client.invoke({
|
||||
_: "getChat",
|
||||
chat_id: Number(chatId),
|
||||
})) as {
|
||||
type?: {
|
||||
_: string;
|
||||
supergroup_id?: number;
|
||||
is_forum?: boolean;
|
||||
};
|
||||
};
|
||||
|
||||
if (chat.type?._ === "chatTypeSupergroup" && chat.type.is_forum) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Also check via getSupergroup for older TDLib versions
|
||||
if (chat.type?._ === "chatTypeSupergroup" && chat.type.supergroup_id) {
|
||||
const sg = (await client.invoke({
|
||||
_: "getSupergroup",
|
||||
supergroup_id: chat.type.supergroup_id,
|
||||
})) as { is_forum?: boolean };
|
||||
return sg.is_forum === true;
|
||||
}
|
||||
|
||||
return false;
|
||||
} catch (err) {
|
||||
log.warn({ err, chatId: chatId.toString() }, "Failed to check if chat is forum");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all forum topics in a supergroup.
|
||||
*/
|
||||
export async function getForumTopicList(
|
||||
client: Client,
|
||||
chatId: bigint
|
||||
): Promise<ForumTopic[]> {
|
||||
const topics: ForumTopic[] = [];
|
||||
let offsetDate = 0;
|
||||
let offsetMessageId = 0;
|
||||
let offsetMessageThreadId = 0;
|
||||
|
||||
// eslint-disable-next-line no-constant-condition
|
||||
while (true) {
|
||||
const result = (await client.invoke({
|
||||
_: "getForumTopics",
|
||||
chat_id: Number(chatId),
|
||||
query: "",
|
||||
offset_date: offsetDate,
|
||||
offset_message_id: offsetMessageId,
|
||||
offset_message_thread_id: offsetMessageThreadId,
|
||||
limit: 100,
|
||||
})) as {
|
||||
topics?: {
|
||||
info?: {
|
||||
message_thread_id?: number;
|
||||
name?: string;
|
||||
is_general?: boolean;
|
||||
};
|
||||
}[];
|
||||
next_offset_date?: number;
|
||||
next_offset_message_id?: number;
|
||||
next_offset_message_thread_id?: number;
|
||||
};
|
||||
|
||||
if (!result.topics || result.topics.length === 0) break;
|
||||
|
||||
for (const t of result.topics) {
|
||||
if (!t.info?.message_thread_id) continue;
|
||||
// Skip the "General" topic — it's not creator-specific
|
||||
if (t.info.is_general) continue;
|
||||
|
||||
topics.push({
|
||||
topicId: BigInt(t.info.message_thread_id),
|
||||
name: t.info.name ?? "Unnamed",
|
||||
});
|
||||
}
|
||||
|
||||
// Check if there are more pages
|
||||
if (
|
||||
!result.next_offset_date &&
|
||||
!result.next_offset_message_id &&
|
||||
!result.next_offset_message_thread_id
|
||||
) {
|
||||
break;
|
||||
}
|
||||
|
||||
offsetDate = result.next_offset_date ?? 0;
|
||||
offsetMessageId = result.next_offset_message_id ?? 0;
|
||||
offsetMessageThreadId = result.next_offset_message_thread_id ?? 0;
|
||||
|
||||
await sleep(config.apiDelayMs);
|
||||
}
|
||||
|
||||
log.info(
|
||||
{ chatId: chatId.toString(), topicCount: topics.length },
|
||||
"Enumerated forum topics"
|
||||
);
|
||||
|
||||
return topics;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch messages from a specific forum topic (thread).
|
||||
* Uses getMessageThreadHistory to scan within a topic.
|
||||
*/
|
||||
export async function getTopicMessages(
|
||||
client: Client,
|
||||
chatId: bigint,
|
||||
topicId: bigint,
|
||||
fromMessageId?: bigint | null,
|
||||
limit = 100
|
||||
): Promise<ChannelScanResult> {
|
||||
const archives: TelegramMessage[] = [];
|
||||
const photos: TelegramPhoto[] = [];
|
||||
let currentFromId = fromMessageId ? Number(fromMessageId) : 0;
|
||||
|
||||
// eslint-disable-next-line no-constant-condition
|
||||
while (true) {
|
||||
const result = (await client.invoke({
|
||||
_: "getMessageThreadHistory",
|
||||
chat_id: Number(chatId),
|
||||
message_id: Number(topicId),
|
||||
from_message_id: currentFromId,
|
||||
offset: 0,
|
||||
limit: Math.min(limit, 100),
|
||||
})) as {
|
||||
messages?: {
|
||||
id: number;
|
||||
date: number;
|
||||
content: {
|
||||
_: string;
|
||||
document?: {
|
||||
file_name?: string;
|
||||
document?: {
|
||||
id: number;
|
||||
size: number;
|
||||
};
|
||||
};
|
||||
photo?: {
|
||||
sizes?: {
|
||||
type: string;
|
||||
photo: { id: number; size: number; expected_size: number };
|
||||
width: number;
|
||||
height: number;
|
||||
}[];
|
||||
};
|
||||
caption?: { text?: string };
|
||||
};
|
||||
}[];
|
||||
};
|
||||
|
||||
if (!result.messages || result.messages.length === 0) break;
|
||||
|
||||
for (const msg of result.messages) {
|
||||
// Check for archive documents
|
||||
const doc = msg.content?.document;
|
||||
if (doc?.file_name && doc.document && isArchiveAttachment(doc.file_name)) {
|
||||
archives.push({
|
||||
id: BigInt(msg.id),
|
||||
fileName: doc.file_name,
|
||||
fileId: String(doc.document.id),
|
||||
fileSize: BigInt(doc.document.size),
|
||||
date: new Date(msg.date * 1000),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check for photo messages (potential previews)
|
||||
const photo = msg.content?.photo;
|
||||
const caption = msg.content?.caption?.text ?? "";
|
||||
if (photo?.sizes && photo.sizes.length > 0) {
|
||||
const smallest = photo.sizes[0];
|
||||
photos.push({
|
||||
id: BigInt(msg.id),
|
||||
date: new Date(msg.date * 1000),
|
||||
caption,
|
||||
fileId: String(smallest.photo.id),
|
||||
fileSize: smallest.photo.size || smallest.photo.expected_size,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
currentFromId = result.messages[result.messages.length - 1].id;
|
||||
if (result.messages.length < 100) break;
|
||||
|
||||
await sleep(config.apiDelayMs);
|
||||
}
|
||||
|
||||
return {
|
||||
archives: archives.reverse(),
|
||||
photos: photos.reverse(),
|
||||
};
|
||||
}
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
76
worker/src/upload/channel.ts
Normal file
76
worker/src/upload/channel.ts
Normal file
@@ -0,0 +1,76 @@
|
||||
import type { Client } from "tdl";
|
||||
import { config } from "../util/config.js";
|
||||
import { childLogger } from "../util/logger.js";
|
||||
|
||||
const log = childLogger("upload");
|
||||
|
||||
export interface UploadResult {
|
||||
messageId: bigint;
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload one or more files to a destination Telegram channel.
|
||||
* For multipart archives, each file is sent as a separate message.
|
||||
* Returns the message ID of the first uploaded message.
|
||||
*/
|
||||
export async function uploadToChannel(
|
||||
client: Client,
|
||||
chatId: bigint,
|
||||
filePaths: string[],
|
||||
caption?: string
|
||||
): Promise<UploadResult> {
|
||||
let firstMessageId: bigint | null = null;
|
||||
|
||||
for (let i = 0; i < filePaths.length; i++) {
|
||||
const filePath = filePaths[i];
|
||||
const fileCaption =
|
||||
i === 0 && caption ? caption : undefined;
|
||||
|
||||
log.debug(
|
||||
{ chatId: Number(chatId), filePath, part: i + 1, total: filePaths.length },
|
||||
"Uploading file to channel"
|
||||
);
|
||||
|
||||
const result = (await client.invoke({
|
||||
_: "sendMessage",
|
||||
chat_id: Number(chatId),
|
||||
input_message_content: {
|
||||
_: "inputMessageDocument",
|
||||
document: {
|
||||
_: "inputFileLocal",
|
||||
path: filePath,
|
||||
},
|
||||
caption: fileCaption
|
||||
? {
|
||||
_: "formattedText",
|
||||
text: fileCaption,
|
||||
}
|
||||
: undefined,
|
||||
},
|
||||
})) as { id: number };
|
||||
|
||||
if (i === 0) {
|
||||
firstMessageId = BigInt(result.id);
|
||||
}
|
||||
|
||||
// Rate limit delay between uploads
|
||||
if (i < filePaths.length - 1) {
|
||||
await sleep(config.apiDelayMs);
|
||||
}
|
||||
}
|
||||
|
||||
if (firstMessageId === null) {
|
||||
throw new Error("Upload failed: no messages sent");
|
||||
}
|
||||
|
||||
log.info(
|
||||
{ chatId: Number(chatId), messageId: Number(firstMessageId), files: filePaths.length },
|
||||
"Upload complete"
|
||||
);
|
||||
|
||||
return { messageId: firstMessageId };
|
||||
}
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
18
worker/src/util/config.ts
Normal file
18
worker/src/util/config.ts
Normal file
@@ -0,0 +1,18 @@
|
||||
export const config = {
|
||||
databaseUrl: process.env.DATABASE_URL ?? "",
|
||||
workerIntervalMinutes: parseInt(process.env.WORKER_INTERVAL_MINUTES ?? "60", 10),
|
||||
tempDir: process.env.WORKER_TEMP_DIR ?? "/tmp/zips",
|
||||
tdlibStateDir: process.env.TDLIB_STATE_DIR ?? "/data/tdlib",
|
||||
maxZipSizeMB: parseInt(process.env.WORKER_MAX_ZIP_SIZE_MB ?? "4096", 10),
|
||||
logLevel: (process.env.LOG_LEVEL ?? "info") as "debug" | "info" | "warn" | "error",
|
||||
telegramApiId: parseInt(process.env.TELEGRAM_API_ID ?? "0", 10),
|
||||
telegramApiHash: process.env.TELEGRAM_API_HASH ?? "",
|
||||
/** Maximum jitter added to scheduler interval (in minutes) */
|
||||
jitterMinutes: 5,
|
||||
/** Maximum time between multipart archive parts (in hours) */
|
||||
multipartTimeoutHours: 24,
|
||||
/** Delay between Telegram API calls (in ms) to avoid rate limits */
|
||||
apiDelayMs: 1000,
|
||||
/** Max retries for rate-limited requests */
|
||||
maxRetries: 5,
|
||||
} as const;
|
||||
14
worker/src/util/logger.ts
Normal file
14
worker/src/util/logger.ts
Normal file
@@ -0,0 +1,14 @@
|
||||
import pino from "pino";
|
||||
import { config } from "./config.js";
|
||||
|
||||
export const logger = pino({
|
||||
level: config.logLevel,
|
||||
transport:
|
||||
config.logLevel === "debug"
|
||||
? { target: "pino/file", options: { destination: 1 } }
|
||||
: undefined,
|
||||
});
|
||||
|
||||
export function childLogger(name: string, extra?: Record<string, unknown>) {
|
||||
return logger.child({ module: name, ...extra });
|
||||
}
|
||||
665
worker/src/worker.ts
Normal file
665
worker/src/worker.ts
Normal file
@@ -0,0 +1,665 @@
|
||||
import path from "path";
|
||||
import { unlink, readdir } from "fs/promises";
|
||||
import { config } from "./util/config.js";
|
||||
import { childLogger } from "./util/logger.js";
|
||||
import { tryAcquireLock, releaseLock } from "./db/locks.js";
|
||||
import {
|
||||
getSourceChannelMappings,
|
||||
getDestinationChannel,
|
||||
packageExistsByHash,
|
||||
createPackageWithFiles,
|
||||
createIngestionRun,
|
||||
completeIngestionRun,
|
||||
failIngestionRun,
|
||||
updateLastProcessedMessage,
|
||||
updateRunActivity,
|
||||
setChannelForum,
|
||||
getTopicProgress,
|
||||
upsertTopicProgress,
|
||||
} from "./db/queries.js";
|
||||
import type { ActivityUpdate } from "./db/queries.js";
|
||||
import { createTdlibClient, closeTdlibClient } from "./tdlib/client.js";
|
||||
import { getChannelMessages, downloadFile, downloadPhotoThumbnail } from "./tdlib/download.js";
|
||||
import type { DownloadProgress, ChannelScanResult } from "./tdlib/download.js";
|
||||
import { isChatForum, getForumTopicList, getTopicMessages } from "./tdlib/topics.js";
|
||||
import { matchPreviewToArchive } from "./preview/match.js";
|
||||
import { groupArchiveSets } from "./archive/multipart.js";
|
||||
import type { ArchiveSet } from "./archive/multipart.js";
|
||||
import { extractCreatorFromFileName } from "./archive/creator.js";
|
||||
import { hashParts } from "./archive/hash.js";
|
||||
import { readZipCentralDirectory } from "./archive/zip-reader.js";
|
||||
import { readRarContents } from "./archive/rar-reader.js";
|
||||
import { byteLevelSplit } from "./archive/split.js";
|
||||
import { uploadToChannel } from "./upload/channel.js";
|
||||
import type { TelegramAccount, TelegramChannel } from "@prisma/client";
|
||||
import type { Client } from "tdl";
|
||||
|
||||
const log = childLogger("worker");
|
||||
|
||||
/**
|
||||
* Throttle DB writes for download progress to avoid hammering the DB.
|
||||
* Only writes if at least 2 seconds have passed since the last write.
|
||||
*/
|
||||
function createThrottledActivityUpdater(runId: string, minIntervalMs = 2000) {
|
||||
let lastWriteTime = 0;
|
||||
let pendingUpdate: ActivityUpdate | null = null;
|
||||
let flushTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
|
||||
const flush = async () => {
|
||||
if (pendingUpdate) {
|
||||
const update = pendingUpdate;
|
||||
pendingUpdate = null;
|
||||
lastWriteTime = Date.now();
|
||||
await updateRunActivity(runId, update).catch(() => {});
|
||||
}
|
||||
};
|
||||
|
||||
return {
|
||||
update: (activity: ActivityUpdate) => {
|
||||
pendingUpdate = activity;
|
||||
const elapsed = Date.now() - lastWriteTime;
|
||||
if (elapsed >= minIntervalMs) {
|
||||
if (flushTimer) clearTimeout(flushTimer);
|
||||
flush();
|
||||
} else if (!flushTimer) {
|
||||
flushTimer = setTimeout(() => {
|
||||
flushTimer = null;
|
||||
flush();
|
||||
}, minIntervalMs - elapsed);
|
||||
}
|
||||
},
|
||||
flush,
|
||||
};
|
||||
}
|
||||
|
||||
/** Shared context passed to the archive processing pipeline. */
|
||||
interface PipelineContext {
|
||||
client: Client;
|
||||
runId: string;
|
||||
channelTitle: string;
|
||||
channel: TelegramChannel;
|
||||
destChannelTelegramId: bigint;
|
||||
destChannelId: string;
|
||||
throttled: ReturnType<typeof createThrottledActivityUpdater>;
|
||||
counters: {
|
||||
messagesScanned: number;
|
||||
zipsFound: number;
|
||||
zipsDuplicate: number;
|
||||
zipsIngested: number;
|
||||
};
|
||||
/** Creator from forum topic name (null for non-forum). */
|
||||
topicCreator: string | null;
|
||||
/** Forum topic ID (null for non-forum). */
|
||||
sourceTopicId: bigint | null;
|
||||
accountLog: ReturnType<typeof childLogger>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a full ingestion cycle for a single Telegram account.
|
||||
* Every step writes live activity to the DB so the admin UI can display it.
|
||||
*/
|
||||
export async function runWorkerForAccount(
|
||||
account: TelegramAccount
|
||||
): Promise<void> {
|
||||
const accountLog = childLogger("worker", { accountId: account.id, phone: account.phone });
|
||||
|
||||
// 1. Acquire advisory lock
|
||||
const acquired = await tryAcquireLock(account.id);
|
||||
if (!acquired) {
|
||||
accountLog.info("Account already locked, skipping");
|
||||
return;
|
||||
}
|
||||
|
||||
let runId: string | undefined;
|
||||
|
||||
try {
|
||||
// 2. Create ingestion run
|
||||
const run = await createIngestionRun(account.id);
|
||||
runId = run.id;
|
||||
const activeRunId = runId;
|
||||
accountLog.info({ runId }, "Ingestion run started");
|
||||
|
||||
const throttled = createThrottledActivityUpdater(activeRunId);
|
||||
|
||||
// 3. Initialize TDLib client
|
||||
await updateRunActivity(activeRunId, {
|
||||
currentActivity: "Connecting to Telegram",
|
||||
currentStep: "connecting",
|
||||
});
|
||||
|
||||
const client = await createTdlibClient({
|
||||
id: account.id,
|
||||
phone: account.phone,
|
||||
});
|
||||
|
||||
const counters = {
|
||||
messagesScanned: 0,
|
||||
zipsFound: 0,
|
||||
zipsDuplicate: 0,
|
||||
zipsIngested: 0,
|
||||
};
|
||||
|
||||
try {
|
||||
// 4. Get assigned source channels and destination
|
||||
const channelMappings = await getSourceChannelMappings(account.id);
|
||||
const destChannel = await getDestinationChannel(account.id);
|
||||
|
||||
if (!destChannel) {
|
||||
throw new Error("No active destination channel configured");
|
||||
}
|
||||
|
||||
for (const mapping of channelMappings) {
|
||||
const channel = mapping.channel;
|
||||
|
||||
// ── Check if channel is a forum ──
|
||||
const forum = await isChatForum(client, channel.telegramId);
|
||||
if (forum !== channel.isForum) {
|
||||
await setChannelForum(channel.id, forum);
|
||||
accountLog.info(
|
||||
{ channelId: channel.id, title: channel.title, isForum: forum },
|
||||
"Updated channel forum status"
|
||||
);
|
||||
}
|
||||
|
||||
const pipelineCtx: PipelineContext = {
|
||||
client,
|
||||
runId: activeRunId,
|
||||
channelTitle: channel.title,
|
||||
channel,
|
||||
destChannelTelegramId: destChannel.telegramId,
|
||||
destChannelId: destChannel.id,
|
||||
throttled,
|
||||
counters,
|
||||
topicCreator: null,
|
||||
sourceTopicId: null,
|
||||
accountLog,
|
||||
};
|
||||
|
||||
if (forum) {
|
||||
// ── Forum channel: scan per-topic ──
|
||||
await updateRunActivity(activeRunId, {
|
||||
currentActivity: `Enumerating topics in "${channel.title}"`,
|
||||
currentStep: "scanning",
|
||||
currentChannel: channel.title,
|
||||
currentFile: null,
|
||||
currentFileNum: null,
|
||||
totalFiles: null,
|
||||
downloadedBytes: null,
|
||||
totalBytes: null,
|
||||
downloadPercent: null,
|
||||
});
|
||||
|
||||
const topics = await getForumTopicList(client, channel.telegramId);
|
||||
const topicProgressList = await getTopicProgress(mapping.id);
|
||||
|
||||
accountLog.info(
|
||||
{ channelId: channel.id, title: channel.title, topicCount: topics.length },
|
||||
"Scanning forum channel by topic"
|
||||
);
|
||||
|
||||
for (const topic of topics) {
|
||||
const progress = topicProgressList.find(
|
||||
(tp) => tp.topicId === topic.topicId
|
||||
);
|
||||
|
||||
await updateRunActivity(activeRunId, {
|
||||
currentActivity: `Scanning topic "${topic.name}" in "${channel.title}"`,
|
||||
currentStep: "scanning",
|
||||
currentChannel: `${channel.title} › ${topic.name}`,
|
||||
currentFile: null,
|
||||
currentFileNum: null,
|
||||
totalFiles: null,
|
||||
downloadedBytes: null,
|
||||
totalBytes: null,
|
||||
downloadPercent: null,
|
||||
});
|
||||
|
||||
const scanResult = await getTopicMessages(
|
||||
client,
|
||||
channel.telegramId,
|
||||
topic.topicId,
|
||||
progress?.lastProcessedMessageId
|
||||
);
|
||||
|
||||
if (scanResult.archives.length === 0) {
|
||||
accountLog.debug(
|
||||
{ channelId: channel.id, topic: topic.name },
|
||||
"No new archives in topic"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
accountLog.info(
|
||||
{ topic: topic.name, archives: scanResult.archives.length, photos: scanResult.photos.length },
|
||||
"Found messages in topic"
|
||||
);
|
||||
|
||||
// Process archives with topic creator
|
||||
pipelineCtx.topicCreator = topic.name;
|
||||
pipelineCtx.sourceTopicId = topic.topicId;
|
||||
pipelineCtx.channelTitle = `${channel.title} › ${topic.name}`;
|
||||
|
||||
await processArchiveSets(pipelineCtx, scanResult, run.id);
|
||||
|
||||
// Update topic progress
|
||||
const allMsgIds = [
|
||||
...scanResult.archives.map((m) => m.id),
|
||||
...scanResult.photos.map((p) => p.id),
|
||||
];
|
||||
if (allMsgIds.length > 0) {
|
||||
const maxId = allMsgIds.reduce((a, b) => (a > b ? a : b));
|
||||
await upsertTopicProgress(
|
||||
mapping.id,
|
||||
topic.topicId,
|
||||
topic.name,
|
||||
maxId
|
||||
);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// ── Non-forum channel: flat scan (existing behavior) ──
|
||||
await updateRunActivity(activeRunId, {
|
||||
currentActivity: `Scanning "${channel.title}" for new archives`,
|
||||
currentStep: "scanning",
|
||||
currentChannel: channel.title,
|
||||
currentFile: null,
|
||||
currentFileNum: null,
|
||||
totalFiles: null,
|
||||
downloadedBytes: null,
|
||||
totalBytes: null,
|
||||
downloadPercent: null,
|
||||
});
|
||||
|
||||
accountLog.info(
|
||||
{ channelId: channel.id, title: channel.title },
|
||||
"Processing source channel"
|
||||
);
|
||||
|
||||
const scanResult = await getChannelMessages(
|
||||
client,
|
||||
channel.telegramId,
|
||||
mapping.lastProcessedMessageId
|
||||
);
|
||||
|
||||
if (scanResult.archives.length === 0) {
|
||||
accountLog.debug({ channelId: channel.id }, "No new archives");
|
||||
continue;
|
||||
}
|
||||
|
||||
accountLog.info(
|
||||
{ archives: scanResult.archives.length, photos: scanResult.photos.length },
|
||||
"Found messages in channel"
|
||||
);
|
||||
|
||||
// For non-forum, creator comes from filename (set to null, resolved per-archive)
|
||||
pipelineCtx.topicCreator = null;
|
||||
pipelineCtx.sourceTopicId = null;
|
||||
pipelineCtx.channelTitle = channel.title;
|
||||
|
||||
await processArchiveSets(pipelineCtx, scanResult, run.id);
|
||||
|
||||
// Update last processed message
|
||||
const allMsgIds = [
|
||||
...scanResult.archives.map((m) => m.id),
|
||||
...scanResult.photos.map((p) => p.id),
|
||||
];
|
||||
if (allMsgIds.length > 0) {
|
||||
const maxId = allMsgIds.reduce((a, b) => (a > b ? a : b));
|
||||
await updateLastProcessedMessage(mapping.id, maxId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Done ──
|
||||
await completeIngestionRun(activeRunId, counters);
|
||||
accountLog.info({ counters }, "Ingestion run completed");
|
||||
} finally {
|
||||
await closeTdlibClient(client);
|
||||
}
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
accountLog.error({ err }, "Ingestion run failed");
|
||||
if (runId) {
|
||||
await failIngestionRun(runId, message).catch((e) =>
|
||||
accountLog.error({ e }, "Failed to mark run as failed")
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
await releaseLock(account.id);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a scan result through the archive pipeline:
|
||||
* group → download → hash → dedup → metadata → split → upload → preview → index.
|
||||
*/
|
||||
async function processArchiveSets(
|
||||
ctx: PipelineContext,
|
||||
scanResult: ChannelScanResult,
|
||||
ingestionRunId: string
|
||||
): Promise<void> {
|
||||
const { client, runId, channelTitle, channel, throttled, counters, accountLog } = ctx;
|
||||
|
||||
// Group into archive sets
|
||||
const archiveSets = groupArchiveSets(scanResult.archives);
|
||||
counters.zipsFound += archiveSets.length;
|
||||
|
||||
// Match preview photos to archive sets
|
||||
const previewMatches = matchPreviewToArchive(
|
||||
scanResult.photos,
|
||||
archiveSets.map((s) => ({
|
||||
baseName: s.baseName,
|
||||
firstMessageId: s.parts[0].id,
|
||||
firstMessageDate: s.parts[0].date,
|
||||
}))
|
||||
);
|
||||
|
||||
if (previewMatches.size > 0) {
|
||||
accountLog.info(
|
||||
{ matched: previewMatches.size, total: archiveSets.length },
|
||||
"Matched preview photos to archives"
|
||||
);
|
||||
}
|
||||
|
||||
await updateRunActivity(runId, {
|
||||
currentActivity: `Found ${archiveSets.length} archive(s) in "${channelTitle}"`,
|
||||
currentStep: "scanning",
|
||||
currentChannel: channelTitle,
|
||||
totalFiles: archiveSets.length,
|
||||
zipsFound: counters.zipsFound,
|
||||
});
|
||||
|
||||
for (let setIdx = 0; setIdx < archiveSets.length; setIdx++) {
|
||||
await processOneArchiveSet(
|
||||
ctx,
|
||||
archiveSets[setIdx],
|
||||
setIdx,
|
||||
archiveSets.length,
|
||||
previewMatches,
|
||||
ingestionRunId
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a single archive set through the full pipeline.
|
||||
*/
|
||||
async function processOneArchiveSet(
|
||||
ctx: PipelineContext,
|
||||
archiveSet: ArchiveSet,
|
||||
setIdx: number,
|
||||
totalSets: number,
|
||||
previewMatches: Map<string, { id: bigint; fileId: string }>,
|
||||
ingestionRunId: string
|
||||
): Promise<void> {
|
||||
const {
|
||||
client, runId, channelTitle, channel,
|
||||
destChannelTelegramId, destChannelId,
|
||||
throttled, counters, topicCreator, sourceTopicId, accountLog,
|
||||
} = ctx;
|
||||
|
||||
counters.messagesScanned += archiveSet.parts.length;
|
||||
const archiveName = archiveSet.parts[0].fileName;
|
||||
const tempPaths: string[] = [];
|
||||
let splitPaths: string[] = [];
|
||||
|
||||
try {
|
||||
// ── Downloading ──
|
||||
for (let partIdx = 0; partIdx < archiveSet.parts.length; partIdx++) {
|
||||
const part = archiveSet.parts[partIdx];
|
||||
const tempPath = path.join(
|
||||
config.tempDir,
|
||||
`${ingestionRunId}_${part.id}_${part.fileName}`
|
||||
);
|
||||
|
||||
const partLabel = archiveSet.parts.length > 1
|
||||
? ` (part ${partIdx + 1}/${archiveSet.parts.length})`
|
||||
: "";
|
||||
|
||||
await updateRunActivity(runId, {
|
||||
currentActivity: `Downloading ${part.fileName}${partLabel}`,
|
||||
currentStep: "downloading",
|
||||
currentChannel: channelTitle,
|
||||
currentFile: part.fileName,
|
||||
currentFileNum: setIdx + 1,
|
||||
totalFiles: totalSets,
|
||||
downloadedBytes: 0n,
|
||||
totalBytes: part.fileSize,
|
||||
downloadPercent: 0,
|
||||
messagesScanned: counters.messagesScanned,
|
||||
});
|
||||
|
||||
accountLog.info(
|
||||
{
|
||||
fileName: part.fileName,
|
||||
fileSize: Number(part.fileSize),
|
||||
part: partIdx + 1,
|
||||
totalParts: archiveSet.parts.length,
|
||||
},
|
||||
"Downloading archive part"
|
||||
);
|
||||
|
||||
await downloadFile(
|
||||
client,
|
||||
part.fileId,
|
||||
tempPath,
|
||||
part.fileSize,
|
||||
part.fileName,
|
||||
(progress: DownloadProgress) => {
|
||||
throttled.update({
|
||||
currentActivity: `Downloading ${part.fileName}${partLabel} — ${progress.percent}%`,
|
||||
currentStep: "downloading",
|
||||
currentChannel: channelTitle,
|
||||
currentFile: part.fileName,
|
||||
currentFileNum: setIdx + 1,
|
||||
totalFiles: totalSets,
|
||||
downloadedBytes: BigInt(progress.downloadedBytes),
|
||||
totalBytes: BigInt(progress.totalBytes),
|
||||
downloadPercent: progress.percent,
|
||||
});
|
||||
}
|
||||
);
|
||||
await throttled.flush();
|
||||
tempPaths.push(tempPath);
|
||||
}
|
||||
|
||||
// ── Hashing ──
|
||||
await updateRunActivity(runId, {
|
||||
currentActivity: `Computing hash for ${archiveName}`,
|
||||
currentStep: "hashing",
|
||||
currentChannel: channelTitle,
|
||||
currentFile: archiveName,
|
||||
currentFileNum: setIdx + 1,
|
||||
totalFiles: totalSets,
|
||||
downloadedBytes: null,
|
||||
totalBytes: null,
|
||||
downloadPercent: null,
|
||||
});
|
||||
|
||||
const contentHash = await hashParts(tempPaths);
|
||||
|
||||
// ── Deduplicating ──
|
||||
await updateRunActivity(runId, {
|
||||
currentActivity: `Checking if ${archiveName} is a duplicate`,
|
||||
currentStep: "deduplicating",
|
||||
currentChannel: channelTitle,
|
||||
currentFile: archiveName,
|
||||
currentFileNum: setIdx + 1,
|
||||
totalFiles: totalSets,
|
||||
});
|
||||
|
||||
const exists = await packageExistsByHash(contentHash);
|
||||
if (exists) {
|
||||
counters.zipsDuplicate++;
|
||||
accountLog.debug({ contentHash }, "Duplicate archive, skipping");
|
||||
|
||||
await updateRunActivity(runId, {
|
||||
currentActivity: `Skipped ${archiveName} (duplicate)`,
|
||||
currentStep: "deduplicating",
|
||||
currentChannel: channelTitle,
|
||||
currentFile: archiveName,
|
||||
currentFileNum: setIdx + 1,
|
||||
totalFiles: totalSets,
|
||||
zipsDuplicate: counters.zipsDuplicate,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// ── Reading metadata ──
|
||||
await updateRunActivity(runId, {
|
||||
currentActivity: `Reading file list from ${archiveName}`,
|
||||
currentStep: "reading_metadata",
|
||||
currentChannel: channelTitle,
|
||||
currentFile: archiveName,
|
||||
currentFileNum: setIdx + 1,
|
||||
totalFiles: totalSets,
|
||||
});
|
||||
|
||||
let entries: { path: string; fileName: string; extension: string | null; compressedSize: bigint; uncompressedSize: bigint; crc32: string | null }[] = [];
|
||||
try {
|
||||
if (archiveSet.type === "ZIP") {
|
||||
entries = await readZipCentralDirectory(tempPaths);
|
||||
} else {
|
||||
entries = await readRarContents(tempPaths[0]);
|
||||
}
|
||||
} catch (err) {
|
||||
accountLog.warn({ err, baseName: archiveSet.baseName }, "Failed to read archive metadata, ingesting without file list");
|
||||
}
|
||||
|
||||
// ── Splitting (if needed) ──
|
||||
let uploadPaths = tempPaths;
|
||||
const totalSize = archiveSet.parts.reduce(
|
||||
(sum, p) => sum + p.fileSize,
|
||||
0n
|
||||
);
|
||||
|
||||
if (!archiveSet.isMultipart && totalSize > 2n * 1024n * 1024n * 1024n) {
|
||||
await updateRunActivity(runId, {
|
||||
currentActivity: `Splitting ${archiveName} for upload (>2GB)`,
|
||||
currentStep: "splitting",
|
||||
currentChannel: channelTitle,
|
||||
currentFile: archiveName,
|
||||
currentFileNum: setIdx + 1,
|
||||
totalFiles: totalSets,
|
||||
});
|
||||
splitPaths = await byteLevelSplit(tempPaths[0]);
|
||||
uploadPaths = splitPaths;
|
||||
}
|
||||
|
||||
// ── Uploading ──
|
||||
const uploadLabel = uploadPaths.length > 1
|
||||
? ` (${uploadPaths.length} parts)`
|
||||
: "";
|
||||
await updateRunActivity(runId, {
|
||||
currentActivity: `Uploading ${archiveName} to archive channel${uploadLabel}`,
|
||||
currentStep: "uploading",
|
||||
currentChannel: channelTitle,
|
||||
currentFile: archiveName,
|
||||
currentFileNum: setIdx + 1,
|
||||
totalFiles: totalSets,
|
||||
});
|
||||
|
||||
const destResult = await uploadToChannel(
|
||||
client,
|
||||
destChannelTelegramId,
|
||||
uploadPaths
|
||||
);
|
||||
|
||||
// ── Preview thumbnail ──
|
||||
let previewData: Buffer | null = null;
|
||||
let previewMsgId: bigint | null = null;
|
||||
const matchedPhoto = previewMatches.get(archiveSet.baseName);
|
||||
if (matchedPhoto) {
|
||||
await updateRunActivity(runId, {
|
||||
currentActivity: `Downloading preview image for ${archiveName}`,
|
||||
currentStep: "preview",
|
||||
currentChannel: channelTitle,
|
||||
currentFile: archiveName,
|
||||
currentFileNum: setIdx + 1,
|
||||
totalFiles: totalSets,
|
||||
});
|
||||
previewData = await downloadPhotoThumbnail(client, matchedPhoto.fileId);
|
||||
previewMsgId = matchedPhoto.id;
|
||||
}
|
||||
|
||||
// ── Resolve creator: topic name > filename extraction > null ──
|
||||
const creator = topicCreator ?? extractCreatorFromFileName(archiveName) ?? null;
|
||||
|
||||
// ── Indexing ──
|
||||
await updateRunActivity(runId, {
|
||||
currentActivity: `Saving metadata for ${archiveName} (${entries.length} files)`,
|
||||
currentStep: "indexing",
|
||||
currentChannel: channelTitle,
|
||||
currentFile: archiveName,
|
||||
currentFileNum: setIdx + 1,
|
||||
totalFiles: totalSets,
|
||||
});
|
||||
|
||||
await createPackageWithFiles({
|
||||
contentHash,
|
||||
fileName: archiveName,
|
||||
fileSize: totalSize,
|
||||
archiveType: archiveSet.type,
|
||||
sourceChannelId: channel.id,
|
||||
sourceMessageId: archiveSet.parts[0].id,
|
||||
sourceTopicId,
|
||||
destChannelId,
|
||||
destMessageId: destResult.messageId,
|
||||
isMultipart:
|
||||
archiveSet.parts.length > 1 || uploadPaths.length > 1,
|
||||
partCount: uploadPaths.length,
|
||||
ingestionRunId,
|
||||
creator,
|
||||
previewData,
|
||||
previewMsgId,
|
||||
files: entries,
|
||||
});
|
||||
|
||||
counters.zipsIngested++;
|
||||
|
||||
await updateRunActivity(runId, {
|
||||
currentActivity: `Ingested ${archiveName} (${entries.length} files indexed)`,
|
||||
currentStep: "complete",
|
||||
currentChannel: channelTitle,
|
||||
currentFile: archiveName,
|
||||
currentFileNum: setIdx + 1,
|
||||
totalFiles: totalSets,
|
||||
zipsIngested: counters.zipsIngested,
|
||||
});
|
||||
|
||||
accountLog.info(
|
||||
{ fileName: archiveName, contentHash, fileCount: entries.length, creator },
|
||||
"Archive ingested"
|
||||
);
|
||||
} finally {
|
||||
// ALWAYS delete temp files
|
||||
await deleteFiles([...tempPaths, ...splitPaths]);
|
||||
}
|
||||
}
|
||||
|
||||
async function deleteFiles(paths: string[]): Promise<void> {
|
||||
for (const p of paths) {
|
||||
try {
|
||||
await unlink(p);
|
||||
} catch {
|
||||
// File may already be deleted or never created
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up any leftover temp files from previous runs.
|
||||
*/
|
||||
export async function cleanupTempDir(): Promise<void> {
|
||||
try {
|
||||
const files = await readdir(config.tempDir);
|
||||
for (const file of files) {
|
||||
await unlink(path.join(config.tempDir, file)).catch(() => {});
|
||||
}
|
||||
if (files.length > 0) {
|
||||
log.info({ count: files.length }, "Cleaned up stale temp files");
|
||||
}
|
||||
} catch {
|
||||
// Directory might not exist yet
|
||||
}
|
||||
}
|
||||
19
worker/tsconfig.json
Normal file
19
worker/tsconfig.json
Normal file
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "bundler",
|
||||
"lib": ["ES2022"],
|
||||
"outDir": "dist",
|
||||
"rootDir": "src",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"resolveJsonModule": true,
|
||||
"isolatedModules": true,
|
||||
"declaration": true,
|
||||
"sourceMap": true
|
||||
},
|
||||
"include": ["src/**/*.ts"],
|
||||
"exclude": ["node_modules", "dist"]
|
||||
}
|
||||
Reference in New Issue
Block a user