mirror of
https://github.com/xCyanGrizzly/DragonsStash.git
synced 2026-05-11 06:11:15 +00:00
feat: add Telegram integration with forum topic support and creator tracking
Adds full Telegram ZIP ingestion pipeline: TDLib worker service scans source channels for archive files, deduplicates by content hash, extracts metadata, uploads to archive channel, and indexes in Postgres. Forum supergroups are scanned per-topic with topic names used as creator. Filename-based creator extraction (e.g. "Mammoth Factory - 2026-01.zip") serves as fallback. Includes admin UI for managing accounts/channels, simplified account setup (API credentials via env vars), auth code/password submission dialog, package browser with creator column, and live ingestion activity tracking. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
11
.env.example
11
.env.example
@@ -11,3 +11,14 @@ AUTH_GITHUB_SECRET=""
|
|||||||
|
|
||||||
# App
|
# App
|
||||||
NEXT_PUBLIC_APP_URL="http://localhost:3000"
|
NEXT_PUBLIC_APP_URL="http://localhost:3000"
|
||||||
|
|
||||||
|
# Telegram integration (get from https://my.telegram.org/apps)
|
||||||
|
TELEGRAM_API_ID=""
|
||||||
|
TELEGRAM_API_HASH=""
|
||||||
|
|
||||||
|
# Worker (only needed when running worker container)
|
||||||
|
WORKER_INTERVAL_MINUTES=60
|
||||||
|
WORKER_TEMP_DIR="/tmp/zips"
|
||||||
|
TDLIB_STATE_DIR="/data/tdlib"
|
||||||
|
WORKER_MAX_ZIP_SIZE_MB=4096
|
||||||
|
LOG_LEVEL="info"
|
||||||
|
|||||||
5
.gitignore
vendored
5
.gitignore
vendored
@@ -1,5 +1,6 @@
|
|||||||
# dependencies
|
# dependencies
|
||||||
/node_modules
|
/node_modules
|
||||||
|
worker/node_modules
|
||||||
/.pnp
|
/.pnp
|
||||||
.pnp.*
|
.pnp.*
|
||||||
.yarn/*
|
.yarn/*
|
||||||
@@ -48,3 +49,7 @@ src/generated
|
|||||||
# ide
|
# ide
|
||||||
.idea
|
.idea
|
||||||
.vscode
|
.vscode
|
||||||
|
|
||||||
|
# temp files
|
||||||
|
nul
|
||||||
|
tmpclaude-*
|
||||||
|
|||||||
@@ -15,5 +15,27 @@ services:
|
|||||||
timeout: 5s
|
timeout: 5s
|
||||||
retries: 5
|
retries: 5
|
||||||
|
|
||||||
|
worker:
|
||||||
|
build:
|
||||||
|
context: .
|
||||||
|
dockerfile: worker/Dockerfile
|
||||||
|
environment:
|
||||||
|
- DATABASE_URL=postgresql://dragons:stash@db:5432/dragonsstash
|
||||||
|
- WORKER_INTERVAL_MINUTES=5
|
||||||
|
- WORKER_TEMP_DIR=/tmp/zips
|
||||||
|
- TDLIB_STATE_DIR=/data/tdlib
|
||||||
|
- WORKER_MAX_ZIP_SIZE_MB=4096
|
||||||
|
- LOG_LEVEL=debug
|
||||||
|
- TELEGRAM_API_ID=${TELEGRAM_API_ID}
|
||||||
|
- TELEGRAM_API_HASH=${TELEGRAM_API_HASH}
|
||||||
|
volumes:
|
||||||
|
- tdlib_dev_state:/data/tdlib
|
||||||
|
- tmp_dev_zips:/tmp/zips
|
||||||
|
depends_on:
|
||||||
|
db:
|
||||||
|
condition: service_healthy
|
||||||
|
|
||||||
volumes:
|
volumes:
|
||||||
postgres_dev_data:
|
postgres_dev_data:
|
||||||
|
tdlib_dev_state:
|
||||||
|
tmp_dev_zips:
|
||||||
|
|||||||
@@ -10,11 +10,37 @@ services:
|
|||||||
- AUTH_SECRET=change-me-to-a-random-secret-in-production
|
- AUTH_SECRET=change-me-to-a-random-secret-in-production
|
||||||
- AUTH_TRUST_HOST=true
|
- AUTH_TRUST_HOST=true
|
||||||
- NEXT_PUBLIC_APP_URL=http://localhost:3000
|
- NEXT_PUBLIC_APP_URL=http://localhost:3000
|
||||||
|
- TELEGRAM_API_KEY=${TELEGRAM_API_KEY:-}
|
||||||
depends_on:
|
depends_on:
|
||||||
db:
|
db:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
|
|
||||||
|
worker:
|
||||||
|
build:
|
||||||
|
context: .
|
||||||
|
dockerfile: worker/Dockerfile
|
||||||
|
environment:
|
||||||
|
- DATABASE_URL=postgresql://dragons:stash@db:5432/dragonsstash
|
||||||
|
- WORKER_INTERVAL_MINUTES=60
|
||||||
|
- WORKER_TEMP_DIR=/tmp/zips
|
||||||
|
- TDLIB_STATE_DIR=/data/tdlib
|
||||||
|
- WORKER_MAX_ZIP_SIZE_MB=4096
|
||||||
|
- LOG_LEVEL=info
|
||||||
|
volumes:
|
||||||
|
- tdlib_state:/data/tdlib
|
||||||
|
- tmp_zips:/tmp/zips
|
||||||
|
depends_on:
|
||||||
|
db:
|
||||||
|
condition: service_healthy
|
||||||
|
restart: unless-stopped
|
||||||
|
deploy:
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
memory: 1G
|
||||||
|
reservations:
|
||||||
|
memory: 256M
|
||||||
|
|
||||||
db:
|
db:
|
||||||
image: postgres:16-alpine
|
image: postgres:16-alpine
|
||||||
ports:
|
ports:
|
||||||
@@ -34,3 +60,5 @@ services:
|
|||||||
|
|
||||||
volumes:
|
volumes:
|
||||||
postgres_data:
|
postgres_data:
|
||||||
|
tdlib_state:
|
||||||
|
tmp_zips:
|
||||||
|
|||||||
@@ -0,0 +1,183 @@
|
|||||||
|
-- CreateEnum
|
||||||
|
CREATE TYPE "AuthState" AS ENUM ('PENDING', 'AWAITING_CODE', 'AWAITING_PASSWORD', 'AUTHENTICATED', 'EXPIRED');
|
||||||
|
|
||||||
|
-- CreateEnum
|
||||||
|
CREATE TYPE "ChannelType" AS ENUM ('SOURCE', 'DESTINATION');
|
||||||
|
|
||||||
|
-- CreateEnum
|
||||||
|
CREATE TYPE "ChannelRole" AS ENUM ('READER', 'WRITER');
|
||||||
|
|
||||||
|
-- CreateEnum
|
||||||
|
CREATE TYPE "ArchiveType" AS ENUM ('ZIP', 'RAR');
|
||||||
|
|
||||||
|
-- CreateEnum
|
||||||
|
CREATE TYPE "IngestionStatus" AS ENUM ('RUNNING', 'COMPLETED', 'FAILED', 'CANCELLED');
|
||||||
|
|
||||||
|
-- CreateTable
|
||||||
|
CREATE TABLE "telegram_accounts" (
|
||||||
|
"id" TEXT NOT NULL,
|
||||||
|
"phone" TEXT NOT NULL,
|
||||||
|
"displayName" TEXT,
|
||||||
|
"apiId" INTEGER NOT NULL,
|
||||||
|
"apiHash" TEXT NOT NULL,
|
||||||
|
"sessionPath" TEXT NOT NULL,
|
||||||
|
"isActive" BOOLEAN NOT NULL DEFAULT true,
|
||||||
|
"authState" "AuthState" NOT NULL DEFAULT 'PENDING',
|
||||||
|
"authCode" TEXT,
|
||||||
|
"lastSeenAt" TIMESTAMP(3),
|
||||||
|
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
"updatedAt" TIMESTAMP(3) NOT NULL,
|
||||||
|
|
||||||
|
CONSTRAINT "telegram_accounts_pkey" PRIMARY KEY ("id")
|
||||||
|
);
|
||||||
|
|
||||||
|
-- CreateTable
|
||||||
|
CREATE TABLE "telegram_channels" (
|
||||||
|
"id" TEXT NOT NULL,
|
||||||
|
"telegramId" BIGINT NOT NULL,
|
||||||
|
"title" TEXT NOT NULL,
|
||||||
|
"type" "ChannelType" NOT NULL,
|
||||||
|
"isActive" BOOLEAN NOT NULL DEFAULT true,
|
||||||
|
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
"updatedAt" TIMESTAMP(3) NOT NULL,
|
||||||
|
|
||||||
|
CONSTRAINT "telegram_channels_pkey" PRIMARY KEY ("id")
|
||||||
|
);
|
||||||
|
|
||||||
|
-- CreateTable
|
||||||
|
CREATE TABLE "account_channel_map" (
|
||||||
|
"id" TEXT NOT NULL,
|
||||||
|
"accountId" TEXT NOT NULL,
|
||||||
|
"channelId" TEXT NOT NULL,
|
||||||
|
"role" "ChannelRole" NOT NULL DEFAULT 'READER',
|
||||||
|
"lastProcessedMessageId" BIGINT,
|
||||||
|
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
|
||||||
|
CONSTRAINT "account_channel_map_pkey" PRIMARY KEY ("id")
|
||||||
|
);
|
||||||
|
|
||||||
|
-- CreateTable
|
||||||
|
CREATE TABLE "packages" (
|
||||||
|
"id" TEXT NOT NULL,
|
||||||
|
"contentHash" TEXT NOT NULL,
|
||||||
|
"fileName" TEXT NOT NULL,
|
||||||
|
"fileSize" BIGINT NOT NULL,
|
||||||
|
"archiveType" "ArchiveType" NOT NULL,
|
||||||
|
"sourceChannelId" TEXT NOT NULL,
|
||||||
|
"sourceMessageId" BIGINT NOT NULL,
|
||||||
|
"destChannelId" TEXT,
|
||||||
|
"destMessageId" BIGINT,
|
||||||
|
"isMultipart" BOOLEAN NOT NULL DEFAULT false,
|
||||||
|
"partCount" INTEGER NOT NULL DEFAULT 1,
|
||||||
|
"fileCount" INTEGER NOT NULL DEFAULT 0,
|
||||||
|
"indexedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
"ingestionRunId" TEXT,
|
||||||
|
|
||||||
|
CONSTRAINT "packages_pkey" PRIMARY KEY ("id")
|
||||||
|
);
|
||||||
|
|
||||||
|
-- CreateTable
|
||||||
|
CREATE TABLE "package_files" (
|
||||||
|
"id" TEXT NOT NULL,
|
||||||
|
"packageId" TEXT NOT NULL,
|
||||||
|
"path" TEXT NOT NULL,
|
||||||
|
"fileName" TEXT NOT NULL,
|
||||||
|
"extension" TEXT,
|
||||||
|
"compressedSize" BIGINT NOT NULL DEFAULT 0,
|
||||||
|
"uncompressedSize" BIGINT NOT NULL DEFAULT 0,
|
||||||
|
"crc32" TEXT,
|
||||||
|
|
||||||
|
CONSTRAINT "package_files_pkey" PRIMARY KEY ("id")
|
||||||
|
);
|
||||||
|
|
||||||
|
-- CreateTable
|
||||||
|
CREATE TABLE "ingestion_runs" (
|
||||||
|
"id" TEXT NOT NULL,
|
||||||
|
"accountId" TEXT NOT NULL,
|
||||||
|
"status" "IngestionStatus" NOT NULL DEFAULT 'RUNNING',
|
||||||
|
"startedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
"finishedAt" TIMESTAMP(3),
|
||||||
|
"messagesScanned" INTEGER NOT NULL DEFAULT 0,
|
||||||
|
"zipsFound" INTEGER NOT NULL DEFAULT 0,
|
||||||
|
"zipsDuplicate" INTEGER NOT NULL DEFAULT 0,
|
||||||
|
"zipsIngested" INTEGER NOT NULL DEFAULT 0,
|
||||||
|
"errorMessage" TEXT,
|
||||||
|
|
||||||
|
CONSTRAINT "ingestion_runs_pkey" PRIMARY KEY ("id")
|
||||||
|
);
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE UNIQUE INDEX "telegram_accounts_phone_key" ON "telegram_accounts"("phone");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE INDEX "telegram_accounts_isActive_idx" ON "telegram_accounts"("isActive");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE UNIQUE INDEX "telegram_channels_telegramId_key" ON "telegram_channels"("telegramId");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE INDEX "telegram_channels_type_isActive_idx" ON "telegram_channels"("type", "isActive");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE INDEX "account_channel_map_accountId_idx" ON "account_channel_map"("accountId");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE INDEX "account_channel_map_channelId_idx" ON "account_channel_map"("channelId");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE UNIQUE INDEX "account_channel_map_accountId_channelId_key" ON "account_channel_map"("accountId", "channelId");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE UNIQUE INDEX "packages_contentHash_key" ON "packages"("contentHash");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE INDEX "packages_sourceChannelId_idx" ON "packages"("sourceChannelId");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE INDEX "packages_destChannelId_idx" ON "packages"("destChannelId");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE INDEX "packages_fileName_idx" ON "packages"("fileName");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE INDEX "packages_indexedAt_idx" ON "packages"("indexedAt");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE INDEX "packages_archiveType_idx" ON "packages"("archiveType");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE INDEX "package_files_packageId_idx" ON "package_files"("packageId");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE INDEX "package_files_extension_idx" ON "package_files"("extension");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE INDEX "package_files_fileName_idx" ON "package_files"("fileName");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE INDEX "ingestion_runs_accountId_idx" ON "ingestion_runs"("accountId");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE INDEX "ingestion_runs_status_idx" ON "ingestion_runs"("status");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE INDEX "ingestion_runs_startedAt_idx" ON "ingestion_runs"("startedAt");
|
||||||
|
|
||||||
|
-- AddForeignKey
|
||||||
|
ALTER TABLE "account_channel_map" ADD CONSTRAINT "account_channel_map_accountId_fkey" FOREIGN KEY ("accountId") REFERENCES "telegram_accounts"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||||
|
|
||||||
|
-- AddForeignKey
|
||||||
|
ALTER TABLE "account_channel_map" ADD CONSTRAINT "account_channel_map_channelId_fkey" FOREIGN KEY ("channelId") REFERENCES "telegram_channels"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||||
|
|
||||||
|
-- AddForeignKey
|
||||||
|
ALTER TABLE "packages" ADD CONSTRAINT "packages_sourceChannelId_fkey" FOREIGN KEY ("sourceChannelId") REFERENCES "telegram_channels"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
|
||||||
|
|
||||||
|
-- AddForeignKey
|
||||||
|
ALTER TABLE "packages" ADD CONSTRAINT "packages_ingestionRunId_fkey" FOREIGN KEY ("ingestionRunId") REFERENCES "ingestion_runs"("id") ON DELETE SET NULL ON UPDATE CASCADE;
|
||||||
|
|
||||||
|
-- AddForeignKey
|
||||||
|
ALTER TABLE "package_files" ADD CONSTRAINT "package_files_packageId_fkey" FOREIGN KEY ("packageId") REFERENCES "packages"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||||
|
|
||||||
|
-- AddForeignKey
|
||||||
|
ALTER TABLE "ingestion_runs" ADD CONSTRAINT "ingestion_runs_accountId_fkey" FOREIGN KEY ("accountId") REFERENCES "telegram_accounts"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
-- AlterTable
|
||||||
|
ALTER TABLE "ingestion_runs" ADD COLUMN "currentActivity" TEXT,
|
||||||
|
ADD COLUMN "currentChannel" TEXT,
|
||||||
|
ADD COLUMN "currentFile" TEXT,
|
||||||
|
ADD COLUMN "currentFileNum" INTEGER,
|
||||||
|
ADD COLUMN "currentStep" TEXT,
|
||||||
|
ADD COLUMN "downloadPercent" INTEGER,
|
||||||
|
ADD COLUMN "downloadedBytes" BIGINT,
|
||||||
|
ADD COLUMN "lastActivityAt" TIMESTAMP(3),
|
||||||
|
ADD COLUMN "totalBytes" BIGINT,
|
||||||
|
ADD COLUMN "totalFiles" INTEGER;
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
-- AlterTable
|
||||||
|
ALTER TABLE "packages" ADD COLUMN "previewData" BYTEA,
|
||||||
|
ADD COLUMN "previewMsgId" BIGINT;
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
/*
|
||||||
|
Warnings:
|
||||||
|
|
||||||
|
- You are about to drop the column `apiHash` on the `telegram_accounts` table. All the data in the column will be lost.
|
||||||
|
- You are about to drop the column `apiId` on the `telegram_accounts` table. All the data in the column will be lost.
|
||||||
|
- You are about to drop the column `sessionPath` on the `telegram_accounts` table. All the data in the column will be lost.
|
||||||
|
|
||||||
|
*/
|
||||||
|
-- AlterTable
|
||||||
|
ALTER TABLE "telegram_accounts" DROP COLUMN "apiHash",
|
||||||
|
DROP COLUMN "apiId",
|
||||||
|
DROP COLUMN "sessionPath";
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
-- AlterTable
|
||||||
|
ALTER TABLE "packages" ADD COLUMN "creator" TEXT,
|
||||||
|
ADD COLUMN "sourceTopicId" BIGINT;
|
||||||
|
|
||||||
|
-- AlterTable
|
||||||
|
ALTER TABLE "telegram_channels" ADD COLUMN "isForum" BOOLEAN NOT NULL DEFAULT false;
|
||||||
|
|
||||||
|
-- CreateTable
|
||||||
|
CREATE TABLE "topic_progress" (
|
||||||
|
"id" TEXT NOT NULL,
|
||||||
|
"accountChannelMapId" TEXT NOT NULL,
|
||||||
|
"topicId" BIGINT NOT NULL,
|
||||||
|
"topicName" TEXT,
|
||||||
|
"lastProcessedMessageId" BIGINT,
|
||||||
|
|
||||||
|
CONSTRAINT "topic_progress_pkey" PRIMARY KEY ("id")
|
||||||
|
);
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE INDEX "topic_progress_accountChannelMapId_idx" ON "topic_progress"("accountChannelMapId");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE UNIQUE INDEX "topic_progress_accountChannelMapId_topicId_key" ON "topic_progress"("accountChannelMapId", "topicId");
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE INDEX "packages_creator_idx" ON "packages"("creator");
|
||||||
|
|
||||||
|
-- AddForeignKey
|
||||||
|
ALTER TABLE "topic_progress" ADD CONSTRAINT "topic_progress_accountChannelMapId_fkey" FOREIGN KEY ("accountChannelMapId") REFERENCES "account_channel_map"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||||
@@ -349,3 +349,189 @@ model UserSettings {
|
|||||||
|
|
||||||
user User @relation(fields: [userId], references: [id], onDelete: Cascade)
|
user User @relation(fields: [userId], references: [id], onDelete: Cascade)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ───────────────────────────────────────
|
||||||
|
// Telegram ingestion models
|
||||||
|
// ───────────────────────────────────────
|
||||||
|
|
||||||
|
enum AuthState {
|
||||||
|
PENDING
|
||||||
|
AWAITING_CODE
|
||||||
|
AWAITING_PASSWORD
|
||||||
|
AUTHENTICATED
|
||||||
|
EXPIRED
|
||||||
|
}
|
||||||
|
|
||||||
|
enum ChannelType {
|
||||||
|
SOURCE
|
||||||
|
DESTINATION
|
||||||
|
}
|
||||||
|
|
||||||
|
enum ChannelRole {
|
||||||
|
READER
|
||||||
|
WRITER
|
||||||
|
}
|
||||||
|
|
||||||
|
enum ArchiveType {
|
||||||
|
ZIP
|
||||||
|
RAR
|
||||||
|
}
|
||||||
|
|
||||||
|
enum IngestionStatus {
|
||||||
|
RUNNING
|
||||||
|
COMPLETED
|
||||||
|
FAILED
|
||||||
|
CANCELLED
|
||||||
|
}
|
||||||
|
|
||||||
|
model TelegramAccount {
|
||||||
|
id String @id @default(cuid())
|
||||||
|
phone String @unique
|
||||||
|
displayName String?
|
||||||
|
isActive Boolean @default(true)
|
||||||
|
authState AuthState @default(PENDING)
|
||||||
|
authCode String?
|
||||||
|
lastSeenAt DateTime?
|
||||||
|
createdAt DateTime @default(now())
|
||||||
|
updatedAt DateTime @updatedAt
|
||||||
|
|
||||||
|
channelMaps AccountChannelMap[]
|
||||||
|
ingestionRuns IngestionRun[]
|
||||||
|
|
||||||
|
@@index([isActive])
|
||||||
|
@@map("telegram_accounts")
|
||||||
|
}
|
||||||
|
|
||||||
|
model TelegramChannel {
|
||||||
|
id String @id @default(cuid())
|
||||||
|
telegramId BigInt @unique
|
||||||
|
title String
|
||||||
|
type ChannelType
|
||||||
|
isForum Boolean @default(false)
|
||||||
|
isActive Boolean @default(true)
|
||||||
|
createdAt DateTime @default(now())
|
||||||
|
updatedAt DateTime @updatedAt
|
||||||
|
|
||||||
|
accountMaps AccountChannelMap[]
|
||||||
|
packages Package[]
|
||||||
|
|
||||||
|
@@index([type, isActive])
|
||||||
|
@@map("telegram_channels")
|
||||||
|
}
|
||||||
|
|
||||||
|
model AccountChannelMap {
|
||||||
|
id String @id @default(cuid())
|
||||||
|
accountId String
|
||||||
|
channelId String
|
||||||
|
role ChannelRole @default(READER)
|
||||||
|
lastProcessedMessageId BigInt?
|
||||||
|
createdAt DateTime @default(now())
|
||||||
|
|
||||||
|
account TelegramAccount @relation(fields: [accountId], references: [id], onDelete: Cascade)
|
||||||
|
channel TelegramChannel @relation(fields: [channelId], references: [id], onDelete: Cascade)
|
||||||
|
topicProgress TopicProgress[]
|
||||||
|
|
||||||
|
@@unique([accountId, channelId])
|
||||||
|
@@index([accountId])
|
||||||
|
@@index([channelId])
|
||||||
|
@@map("account_channel_map")
|
||||||
|
}
|
||||||
|
|
||||||
|
model Package {
|
||||||
|
id String @id @default(cuid())
|
||||||
|
contentHash String @unique
|
||||||
|
fileName String
|
||||||
|
fileSize BigInt
|
||||||
|
archiveType ArchiveType
|
||||||
|
creator String?
|
||||||
|
sourceChannelId String
|
||||||
|
sourceMessageId BigInt
|
||||||
|
sourceTopicId BigInt?
|
||||||
|
destChannelId String?
|
||||||
|
destMessageId BigInt?
|
||||||
|
isMultipart Boolean @default(false)
|
||||||
|
partCount Int @default(1)
|
||||||
|
fileCount Int @default(0)
|
||||||
|
previewData Bytes? // JPEG thumbnail from nearby Telegram photo (stored as raw bytes)
|
||||||
|
previewMsgId BigInt? // Telegram message ID of the matched photo
|
||||||
|
indexedAt DateTime @default(now())
|
||||||
|
createdAt DateTime @default(now())
|
||||||
|
|
||||||
|
sourceChannel TelegramChannel @relation(fields: [sourceChannelId], references: [id])
|
||||||
|
files PackageFile[]
|
||||||
|
ingestionRun IngestionRun? @relation(fields: [ingestionRunId], references: [id])
|
||||||
|
ingestionRunId String?
|
||||||
|
|
||||||
|
@@index([sourceChannelId])
|
||||||
|
@@index([destChannelId])
|
||||||
|
@@index([fileName])
|
||||||
|
@@index([indexedAt])
|
||||||
|
@@index([archiveType])
|
||||||
|
@@index([creator])
|
||||||
|
@@map("packages")
|
||||||
|
}
|
||||||
|
|
||||||
|
model PackageFile {
|
||||||
|
id String @id @default(cuid())
|
||||||
|
packageId String
|
||||||
|
path String
|
||||||
|
fileName String
|
||||||
|
extension String?
|
||||||
|
compressedSize BigInt @default(0)
|
||||||
|
uncompressedSize BigInt @default(0)
|
||||||
|
crc32 String?
|
||||||
|
|
||||||
|
package Package @relation(fields: [packageId], references: [id], onDelete: Cascade)
|
||||||
|
|
||||||
|
@@index([packageId])
|
||||||
|
@@index([extension])
|
||||||
|
@@index([fileName])
|
||||||
|
@@map("package_files")
|
||||||
|
}
|
||||||
|
|
||||||
|
model IngestionRun {
|
||||||
|
id String @id @default(cuid())
|
||||||
|
accountId String
|
||||||
|
status IngestionStatus @default(RUNNING)
|
||||||
|
startedAt DateTime @default(now())
|
||||||
|
finishedAt DateTime?
|
||||||
|
messagesScanned Int @default(0)
|
||||||
|
zipsFound Int @default(0)
|
||||||
|
zipsDuplicate Int @default(0)
|
||||||
|
zipsIngested Int @default(0)
|
||||||
|
errorMessage String?
|
||||||
|
|
||||||
|
// Live activity tracking — written by worker in real-time
|
||||||
|
currentActivity String? // Human-readable: "Downloading pack.zip (part 2/5)"
|
||||||
|
currentStep String? // Machine-readable step key
|
||||||
|
currentChannel String? // Channel title being processed
|
||||||
|
currentFile String? // File currently being processed
|
||||||
|
currentFileNum Int? // Which archive set (1-indexed)
|
||||||
|
totalFiles Int? // Total archive sets found
|
||||||
|
downloadedBytes BigInt? // Current download progress in bytes
|
||||||
|
totalBytes BigInt? // Total size of current download
|
||||||
|
downloadPercent Int? // 0-100
|
||||||
|
lastActivityAt DateTime? // When activity was last updated
|
||||||
|
|
||||||
|
account TelegramAccount @relation(fields: [accountId], references: [id])
|
||||||
|
packages Package[]
|
||||||
|
|
||||||
|
@@index([accountId])
|
||||||
|
@@index([status])
|
||||||
|
@@index([startedAt])
|
||||||
|
@@map("ingestion_runs")
|
||||||
|
}
|
||||||
|
|
||||||
|
model TopicProgress {
|
||||||
|
id String @id @default(cuid())
|
||||||
|
accountChannelMapId String
|
||||||
|
topicId BigInt
|
||||||
|
topicName String?
|
||||||
|
lastProcessedMessageId BigInt?
|
||||||
|
|
||||||
|
accountChannelMap AccountChannelMap @relation(fields: [accountChannelMapId], references: [id], onDelete: Cascade)
|
||||||
|
|
||||||
|
@@unique([accountChannelMapId, topicId])
|
||||||
|
@@index([accountChannelMapId])
|
||||||
|
@@map("topic_progress")
|
||||||
|
}
|
||||||
|
|||||||
149
src/app/(app)/stls/_components/ingestion-status.tsx
Normal file
149
src/app/(app)/stls/_components/ingestion-status.tsx
Normal file
@@ -0,0 +1,149 @@
|
|||||||
|
"use client";
|
||||||
|
|
||||||
|
import { useEffect, useState } from "react";
|
||||||
|
import { Loader2, CheckCircle2, XCircle, CloudOff } from "lucide-react";
|
||||||
|
import { cn } from "@/lib/utils";
|
||||||
|
import type { IngestionAccountStatus } from "@/lib/telegram/types";
|
||||||
|
|
||||||
|
interface IngestionStatusProps {
|
||||||
|
initialStatus: IngestionAccountStatus[];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Polls /api/ingestion/status every 3 seconds while a run is active,
|
||||||
|
* or every 30 seconds when idle. Shows a compact status banner with
|
||||||
|
* a spinning throbber when ingestion is running.
|
||||||
|
*/
|
||||||
|
export function IngestionStatus({ initialStatus }: IngestionStatusProps) {
|
||||||
|
const [accounts, setAccounts] = useState(initialStatus);
|
||||||
|
const [error, setError] = useState(false);
|
||||||
|
|
||||||
|
// Determine if any account is currently running
|
||||||
|
const activeRun = accounts.find((a) => a.currentRun);
|
||||||
|
const isRunning = !!activeRun;
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
let timer: ReturnType<typeof setTimeout>;
|
||||||
|
let mounted = true;
|
||||||
|
|
||||||
|
const poll = async () => {
|
||||||
|
try {
|
||||||
|
const res = await fetch("/api/ingestion/status");
|
||||||
|
if (!res.ok) throw new Error("fetch failed");
|
||||||
|
const data = await res.json();
|
||||||
|
if (mounted) {
|
||||||
|
setAccounts(data.accounts ?? []);
|
||||||
|
setError(false);
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
if (mounted) setError(true);
|
||||||
|
}
|
||||||
|
if (mounted) {
|
||||||
|
// Poll fast while running, slow when idle
|
||||||
|
const interval = accounts.some((a) => a.currentRun) ? 3_000 : 30_000;
|
||||||
|
timer = setTimeout(poll, interval);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// Start polling after a short delay to avoid double-fetching on mount
|
||||||
|
timer = setTimeout(poll, 3_000);
|
||||||
|
|
||||||
|
return () => {
|
||||||
|
mounted = false;
|
||||||
|
clearTimeout(timer);
|
||||||
|
};
|
||||||
|
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||||
|
}, [isRunning]);
|
||||||
|
|
||||||
|
// Nothing to show if no accounts configured
|
||||||
|
if (accounts.length === 0 && !error) return null;
|
||||||
|
|
||||||
|
// If we can't reach the API, show a muted offline badge
|
||||||
|
if (error) {
|
||||||
|
return (
|
||||||
|
<div className="flex items-center gap-2 rounded-lg border border-border bg-card px-3 py-2 text-xs text-muted-foreground">
|
||||||
|
<CloudOff className="h-3.5 w-3.5" />
|
||||||
|
<span>Sync status unavailable</span>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Active run — show throbber with live activity
|
||||||
|
if (activeRun?.currentRun) {
|
||||||
|
const run = activeRun.currentRun;
|
||||||
|
return (
|
||||||
|
<div className="flex items-center gap-3 rounded-lg border border-primary/20 bg-primary/5 px-3 py-2">
|
||||||
|
<Loader2 className="h-4 w-4 shrink-0 animate-spin text-primary" />
|
||||||
|
<div className="min-w-0 flex-1">
|
||||||
|
<p className="truncate text-sm font-medium text-primary">
|
||||||
|
{run.currentActivity ?? "Syncing..."}
|
||||||
|
</p>
|
||||||
|
{run.downloadPercent != null && run.downloadPercent > 0 && (
|
||||||
|
<div className="mt-1 flex items-center gap-2">
|
||||||
|
<div className="h-1.5 w-24 rounded-full bg-primary/20">
|
||||||
|
<div
|
||||||
|
className="h-full rounded-full bg-primary transition-all duration-300"
|
||||||
|
style={{ width: `${Math.min(100, run.downloadPercent)}%` }}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<span className="text-[10px] text-primary/70">{run.downloadPercent}%</span>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
{run.totalFiles != null && run.currentFileNum != null && (
|
||||||
|
<span className="shrink-0 text-[10px] text-primary/60">
|
||||||
|
{run.currentFileNum}/{run.totalFiles}
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// All idle — show last run summary
|
||||||
|
const lastCompleted = accounts
|
||||||
|
.filter((a) => a.lastRun)
|
||||||
|
.sort(
|
||||||
|
(a, b) =>
|
||||||
|
new Date(b.lastRun!.finishedAt ?? b.lastRun!.startedAt).getTime() -
|
||||||
|
new Date(a.lastRun!.finishedAt ?? a.lastRun!.startedAt).getTime()
|
||||||
|
)[0];
|
||||||
|
|
||||||
|
if (!lastCompleted?.lastRun) return null;
|
||||||
|
|
||||||
|
const last = lastCompleted.lastRun;
|
||||||
|
const isFailed = last.status === "FAILED";
|
||||||
|
const timeAgo = getTimeAgo(last.finishedAt ?? last.startedAt);
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div
|
||||||
|
className={cn(
|
||||||
|
"flex items-center gap-2 rounded-lg border px-3 py-2 text-xs",
|
||||||
|
isFailed
|
||||||
|
? "border-red-500/20 bg-red-500/5 text-red-400"
|
||||||
|
: "border-border bg-card text-muted-foreground"
|
||||||
|
)}
|
||||||
|
>
|
||||||
|
{isFailed ? (
|
||||||
|
<XCircle className="h-3.5 w-3.5 shrink-0" />
|
||||||
|
) : (
|
||||||
|
<CheckCircle2 className="h-3.5 w-3.5 shrink-0 text-emerald-400" />
|
||||||
|
)}
|
||||||
|
<span className="truncate">
|
||||||
|
{isFailed
|
||||||
|
? `Last sync failed ${timeAgo}`
|
||||||
|
: `Last sync ${timeAgo} — ${last.zipsIngested} new, ${last.zipsDuplicate} skipped`}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function getTimeAgo(dateStr: string): string {
|
||||||
|
const diff = Date.now() - new Date(dateStr).getTime();
|
||||||
|
const mins = Math.floor(diff / 60_000);
|
||||||
|
if (mins < 1) return "just now";
|
||||||
|
if (mins < 60) return `${mins}m ago`;
|
||||||
|
const hours = Math.floor(mins / 60);
|
||||||
|
if (hours < 24) return `${hours}h ago`;
|
||||||
|
const days = Math.floor(hours / 24);
|
||||||
|
return `${days}d ago`;
|
||||||
|
}
|
||||||
154
src/app/(app)/stls/_components/package-columns.tsx
Normal file
154
src/app/(app)/stls/_components/package-columns.tsx
Normal file
@@ -0,0 +1,154 @@
|
|||||||
|
"use client";
|
||||||
|
|
||||||
|
import { type ColumnDef } from "@tanstack/react-table";
|
||||||
|
import { FileArchive, Eye, ImageIcon } from "lucide-react";
|
||||||
|
import { DataTableColumnHeader } from "@/components/shared/data-table-column-header";
|
||||||
|
import { Badge } from "@/components/ui/badge";
|
||||||
|
import { Button } from "@/components/ui/button";
|
||||||
|
|
||||||
|
export interface PackageRow {
|
||||||
|
id: string;
|
||||||
|
fileName: string;
|
||||||
|
fileSize: string;
|
||||||
|
contentHash: string;
|
||||||
|
archiveType: "ZIP" | "RAR";
|
||||||
|
fileCount: number;
|
||||||
|
isMultipart: boolean;
|
||||||
|
hasPreview: boolean;
|
||||||
|
creator: string | null;
|
||||||
|
indexedAt: string;
|
||||||
|
sourceChannel: {
|
||||||
|
id: string;
|
||||||
|
title: string;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
interface PackageColumnsProps {
|
||||||
|
onViewFiles: (pkg: PackageRow) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatBytes(bytesStr: string): string {
|
||||||
|
const bytes = Number(bytesStr);
|
||||||
|
if (bytes === 0) return "0 B";
|
||||||
|
const k = 1024;
|
||||||
|
const sizes = ["B", "KB", "MB", "GB", "TB"];
|
||||||
|
const i = Math.floor(Math.log(bytes) / Math.log(k));
|
||||||
|
return `${parseFloat((bytes / Math.pow(k, i)).toFixed(1))} ${sizes[i]}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function PreviewCell({ pkg }: { pkg: PackageRow }) {
|
||||||
|
if (pkg.hasPreview) {
|
||||||
|
return (
|
||||||
|
<img
|
||||||
|
src={`/api/zips/${pkg.id}/preview`}
|
||||||
|
alt=""
|
||||||
|
className="h-9 w-9 rounded-md object-cover bg-muted"
|
||||||
|
loading="lazy"
|
||||||
|
/>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return (
|
||||||
|
<div className="flex h-9 w-9 items-center justify-center rounded-md bg-muted">
|
||||||
|
<FileArchive className="h-4 w-4 text-muted-foreground" />
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getPackageColumns({
|
||||||
|
onViewFiles,
|
||||||
|
}: PackageColumnsProps): ColumnDef<PackageRow, unknown>[] {
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
id: "preview",
|
||||||
|
header: "",
|
||||||
|
cell: ({ row }) => <PreviewCell pkg={row.original} />,
|
||||||
|
enableHiding: false,
|
||||||
|
enableSorting: false,
|
||||||
|
size: 52,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
accessorKey: "fileName",
|
||||||
|
header: ({ column }) => <DataTableColumnHeader column={column} title="File Name" />,
|
||||||
|
cell: ({ row }) => (
|
||||||
|
<div className="flex items-center gap-2 min-w-0">
|
||||||
|
<span className="font-medium truncate max-w-[300px]">{row.original.fileName}</span>
|
||||||
|
{row.original.isMultipart && (
|
||||||
|
<Badge variant="outline" className="text-[10px] shrink-0">
|
||||||
|
Multi
|
||||||
|
</Badge>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
),
|
||||||
|
enableHiding: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
accessorKey: "archiveType",
|
||||||
|
header: ({ column }) => <DataTableColumnHeader column={column} title="Type" />,
|
||||||
|
cell: ({ row }) => (
|
||||||
|
<Badge variant="secondary" className="text-[10px]">
|
||||||
|
{row.original.archiveType}
|
||||||
|
</Badge>
|
||||||
|
),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
accessorKey: "fileSize",
|
||||||
|
header: ({ column }) => <DataTableColumnHeader column={column} title="Size" />,
|
||||||
|
cell: ({ row }) => (
|
||||||
|
<span className="text-sm text-muted-foreground">
|
||||||
|
{formatBytes(row.original.fileSize)}
|
||||||
|
</span>
|
||||||
|
),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
accessorKey: "fileCount",
|
||||||
|
header: ({ column }) => <DataTableColumnHeader column={column} title="Files" />,
|
||||||
|
cell: ({ row }) => (
|
||||||
|
<span className="text-sm">
|
||||||
|
{row.original.fileCount.toLocaleString()}
|
||||||
|
</span>
|
||||||
|
),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
accessorKey: "creator",
|
||||||
|
header: ({ column }) => <DataTableColumnHeader column={column} title="Creator" />,
|
||||||
|
cell: ({ row }) => (
|
||||||
|
<span className="text-sm text-muted-foreground truncate max-w-[160px] block">
|
||||||
|
{row.original.creator ?? "\u2014"}
|
||||||
|
</span>
|
||||||
|
),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "channel",
|
||||||
|
header: ({ column }) => <DataTableColumnHeader column={column} title="Source" />,
|
||||||
|
cell: ({ row }) => (
|
||||||
|
<span className="text-sm text-muted-foreground truncate max-w-[160px] block">
|
||||||
|
{row.original.sourceChannel.title}
|
||||||
|
</span>
|
||||||
|
),
|
||||||
|
accessorFn: (row) => row.sourceChannel.title,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
accessorKey: "indexedAt",
|
||||||
|
header: ({ column }) => <DataTableColumnHeader column={column} title="Indexed" />,
|
||||||
|
cell: ({ row }) => (
|
||||||
|
<span className="text-sm text-muted-foreground">
|
||||||
|
{new Date(row.original.indexedAt).toLocaleDateString()}
|
||||||
|
</span>
|
||||||
|
),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "actions",
|
||||||
|
cell: ({ row }) => (
|
||||||
|
<Button
|
||||||
|
variant="ghost"
|
||||||
|
size="icon"
|
||||||
|
className="h-8 w-8"
|
||||||
|
onClick={() => onViewFiles(row.original)}
|
||||||
|
>
|
||||||
|
<Eye className="h-4 w-4" />
|
||||||
|
</Button>
|
||||||
|
),
|
||||||
|
enableHiding: false,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
}
|
||||||
412
src/app/(app)/stls/_components/package-files-drawer.tsx
Normal file
412
src/app/(app)/stls/_components/package-files-drawer.tsx
Normal file
@@ -0,0 +1,412 @@
|
|||||||
|
"use client";
|
||||||
|
|
||||||
|
import { useEffect, useState, useCallback, useMemo } from "react";
|
||||||
|
import {
|
||||||
|
FileText,
|
||||||
|
Folder,
|
||||||
|
FolderOpen,
|
||||||
|
Loader2,
|
||||||
|
Search,
|
||||||
|
ChevronDown,
|
||||||
|
ChevronRight,
|
||||||
|
} from "lucide-react";
|
||||||
|
import {
|
||||||
|
Dialog,
|
||||||
|
DialogContent,
|
||||||
|
DialogHeader,
|
||||||
|
DialogTitle,
|
||||||
|
DialogDescription,
|
||||||
|
} from "@/components/ui/dialog";
|
||||||
|
import { ScrollArea } from "@/components/ui/scroll-area";
|
||||||
|
import { Badge } from "@/components/ui/badge";
|
||||||
|
import { Input } from "@/components/ui/input";
|
||||||
|
import { Button } from "@/components/ui/button";
|
||||||
|
import { cn } from "@/lib/utils";
|
||||||
|
import type { PackageRow } from "./package-columns";
|
||||||
|
|
||||||
|
interface FileItem {
|
||||||
|
id: string;
|
||||||
|
path: string;
|
||||||
|
fileName: string;
|
||||||
|
extension: string | null;
|
||||||
|
compressedSize: string;
|
||||||
|
uncompressedSize: string;
|
||||||
|
crc32: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface TreeNode {
|
||||||
|
name: string;
|
||||||
|
isFolder: boolean;
|
||||||
|
children: Map<string, TreeNode>;
|
||||||
|
file?: FileItem;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface PackageFilesDrawerProps {
|
||||||
|
pkg: PackageRow | null;
|
||||||
|
open: boolean;
|
||||||
|
onOpenChange: (open: boolean) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatBytes(bytesStr: string): string {
|
||||||
|
const bytes = Number(bytesStr);
|
||||||
|
if (bytes === 0) return "0 B";
|
||||||
|
const k = 1024;
|
||||||
|
const sizes = ["B", "KB", "MB", "GB", "TB"];
|
||||||
|
const i = Math.floor(Math.log(bytes) / Math.log(k));
|
||||||
|
return `${parseFloat((bytes / Math.pow(k, i)).toFixed(1))} ${sizes[i]}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
const EXTENSION_COLORS: Record<string, string> = {
|
||||||
|
stl: "bg-blue-500/15 text-blue-400 border-blue-500/30",
|
||||||
|
obj: "bg-violet-500/15 text-violet-400 border-violet-500/30",
|
||||||
|
"3mf": "bg-cyan-500/15 text-cyan-400 border-cyan-500/30",
|
||||||
|
gcode: "bg-amber-500/15 text-amber-400 border-amber-500/30",
|
||||||
|
png: "bg-emerald-500/15 text-emerald-400 border-emerald-500/30",
|
||||||
|
jpg: "bg-emerald-500/15 text-emerald-400 border-emerald-500/30",
|
||||||
|
jpeg: "bg-emerald-500/15 text-emerald-400 border-emerald-500/30",
|
||||||
|
pdf: "bg-red-500/15 text-red-400 border-red-500/30",
|
||||||
|
txt: "bg-zinc-500/15 text-zinc-400 border-zinc-500/30",
|
||||||
|
lys: "bg-pink-500/15 text-pink-400 border-pink-500/30",
|
||||||
|
};
|
||||||
|
|
||||||
|
function getExtBadgeClass(ext: string | null): string {
|
||||||
|
if (!ext) return "bg-zinc-500/15 text-zinc-400 border-zinc-500/30";
|
||||||
|
return EXTENSION_COLORS[ext.toLowerCase()] ?? "bg-zinc-500/15 text-zinc-400 border-zinc-500/30";
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build a tree structure from flat file paths.
|
||||||
|
*/
|
||||||
|
function buildFileTree(files: FileItem[]): TreeNode {
|
||||||
|
const root: TreeNode = { name: "", isFolder: true, children: new Map() };
|
||||||
|
|
||||||
|
for (const file of files) {
|
||||||
|
// Normalize path separators (Windows RAR archives may use backslashes)
|
||||||
|
const parts = file.path.replace(/\\/g, "/").split("/").filter(Boolean);
|
||||||
|
let current = root;
|
||||||
|
|
||||||
|
for (let i = 0; i < parts.length; i++) {
|
||||||
|
const part = parts[i];
|
||||||
|
const isLast = i === parts.length - 1;
|
||||||
|
|
||||||
|
if (!current.children.has(part)) {
|
||||||
|
current.children.set(part, {
|
||||||
|
name: part,
|
||||||
|
isFolder: !isLast,
|
||||||
|
children: new Map(),
|
||||||
|
file: isLast ? file : undefined,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
current = current.children.get(part)!;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return root;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Recursively renders a file tree node with indentation.
|
||||||
|
*/
|
||||||
|
function TreeNodeView({
|
||||||
|
node,
|
||||||
|
depth,
|
||||||
|
search,
|
||||||
|
defaultOpen,
|
||||||
|
}: {
|
||||||
|
node: TreeNode;
|
||||||
|
depth: number;
|
||||||
|
search: string;
|
||||||
|
defaultOpen: boolean;
|
||||||
|
}) {
|
||||||
|
const [open, setOpen] = useState(defaultOpen);
|
||||||
|
|
||||||
|
// Sort children: folders first, then files, alphabetical within each group
|
||||||
|
const sortedChildren = useMemo(() => {
|
||||||
|
const arr = Array.from(node.children.values());
|
||||||
|
return arr.sort((a, b) => {
|
||||||
|
if (a.isFolder !== b.isFolder) return a.isFolder ? -1 : 1;
|
||||||
|
return a.name.localeCompare(b.name);
|
||||||
|
});
|
||||||
|
}, [node.children]);
|
||||||
|
|
||||||
|
// If searching, force all open
|
||||||
|
useEffect(() => {
|
||||||
|
if (search) setOpen(true);
|
||||||
|
}, [search]);
|
||||||
|
|
||||||
|
if (node.isFolder && node.children.size > 0) {
|
||||||
|
return (
|
||||||
|
<div>
|
||||||
|
{/* Don't render a row for the root node */}
|
||||||
|
{depth >= 0 && (
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
className="flex w-full items-center gap-1.5 rounded-md px-1 py-1 text-sm hover:bg-muted/50 transition-colors"
|
||||||
|
style={{ paddingLeft: `${Math.max(0, depth) * 16 + 4}px` }}
|
||||||
|
onClick={() => setOpen(!open)}
|
||||||
|
>
|
||||||
|
{open ? (
|
||||||
|
<ChevronDown className="h-3 w-3 shrink-0 text-muted-foreground" />
|
||||||
|
) : (
|
||||||
|
<ChevronRight className="h-3 w-3 shrink-0 text-muted-foreground" />
|
||||||
|
)}
|
||||||
|
{open ? (
|
||||||
|
<FolderOpen className="h-3.5 w-3.5 shrink-0 text-primary/70" />
|
||||||
|
) : (
|
||||||
|
<Folder className="h-3.5 w-3.5 shrink-0 text-primary/70" />
|
||||||
|
)}
|
||||||
|
<span className="text-sm font-medium truncate">{node.name}</span>
|
||||||
|
<span className="text-[10px] text-muted-foreground ml-auto shrink-0">
|
||||||
|
{countFiles(node)}
|
||||||
|
</span>
|
||||||
|
</button>
|
||||||
|
)}
|
||||||
|
{open &&
|
||||||
|
sortedChildren.map((child) => (
|
||||||
|
<TreeNodeView
|
||||||
|
key={child.name}
|
||||||
|
node={child}
|
||||||
|
depth={depth + 1}
|
||||||
|
search={search}
|
||||||
|
defaultOpen={depth < 1} // Auto-expand first 2 levels
|
||||||
|
/>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// File node
|
||||||
|
if (node.file) {
|
||||||
|
return (
|
||||||
|
<div
|
||||||
|
className="flex items-center gap-2 rounded-md px-1 py-1 hover:bg-muted/50 transition-colors"
|
||||||
|
style={{ paddingLeft: `${Math.max(0, depth) * 16 + 4}px` }}
|
||||||
|
>
|
||||||
|
<FileText className="h-3.5 w-3.5 shrink-0 text-muted-foreground" />
|
||||||
|
<span className="text-sm truncate flex-1 min-w-0" title={node.file.path}>
|
||||||
|
{node.name}
|
||||||
|
</span>
|
||||||
|
{node.file.extension && (
|
||||||
|
<Badge
|
||||||
|
variant="outline"
|
||||||
|
className={`text-[10px] shrink-0 ${getExtBadgeClass(node.file.extension)}`}
|
||||||
|
>
|
||||||
|
.{node.file.extension}
|
||||||
|
</Badge>
|
||||||
|
)}
|
||||||
|
<span className="text-[11px] text-muted-foreground shrink-0 tabular-nums">
|
||||||
|
{formatBytes(node.file.uncompressedSize)}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function countFiles(node: TreeNode): number {
|
||||||
|
if (!node.isFolder) return 1;
|
||||||
|
let count = 0;
|
||||||
|
for (const child of node.children.values()) {
|
||||||
|
count += countFiles(child);
|
||||||
|
}
|
||||||
|
return count;
|
||||||
|
}
|
||||||
|
|
||||||
|
const PAGE_SIZE = 100;
|
||||||
|
|
||||||
|
export function PackageFilesDrawer({ pkg, open, onOpenChange }: PackageFilesDrawerProps) {
|
||||||
|
const [files, setFiles] = useState<FileItem[]>([]);
|
||||||
|
const [total, setTotal] = useState(0);
|
||||||
|
const [loading, setLoading] = useState(false);
|
||||||
|
const [loadingMore, setLoadingMore] = useState(false);
|
||||||
|
const [search, setSearch] = useState("");
|
||||||
|
const [page, setPage] = useState(1);
|
||||||
|
|
||||||
|
const fetchFiles = useCallback(
|
||||||
|
async (pageNum: number, append: boolean) => {
|
||||||
|
if (!pkg) return;
|
||||||
|
if (pageNum === 1) setLoading(true);
|
||||||
|
else setLoadingMore(true);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const params = new URLSearchParams({
|
||||||
|
page: String(pageNum),
|
||||||
|
limit: String(PAGE_SIZE),
|
||||||
|
});
|
||||||
|
const res = await fetch(`/api/zips/${pkg.id}/files?${params}`);
|
||||||
|
if (!res.ok) throw new Error("fetch failed");
|
||||||
|
const data = await res.json();
|
||||||
|
setFiles((prev) => (append ? [...prev, ...data.items] : data.items));
|
||||||
|
setTotal(data.pagination.total);
|
||||||
|
} catch {
|
||||||
|
// Silently handle
|
||||||
|
} finally {
|
||||||
|
setLoading(false);
|
||||||
|
setLoadingMore(false);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
[pkg]
|
||||||
|
);
|
||||||
|
|
||||||
|
// Reset and fetch when package changes
|
||||||
|
useEffect(() => {
|
||||||
|
if (open && pkg) {
|
||||||
|
setFiles([]);
|
||||||
|
setTotal(0);
|
||||||
|
setSearch("");
|
||||||
|
setPage(1);
|
||||||
|
fetchFiles(1, false);
|
||||||
|
}
|
||||||
|
}, [open, pkg, fetchFiles]);
|
||||||
|
|
||||||
|
const loadMore = () => {
|
||||||
|
const nextPage = page + 1;
|
||||||
|
setPage(nextPage);
|
||||||
|
fetchFiles(nextPage, true);
|
||||||
|
};
|
||||||
|
|
||||||
|
const hasMore = files.length < total;
|
||||||
|
|
||||||
|
// Client-side search filter (over loaded files)
|
||||||
|
const filtered = search
|
||||||
|
? files.filter(
|
||||||
|
(f) =>
|
||||||
|
f.fileName.toLowerCase().includes(search.toLowerCase()) ||
|
||||||
|
f.path.toLowerCase().includes(search.toLowerCase())
|
||||||
|
)
|
||||||
|
: files;
|
||||||
|
|
||||||
|
// Build tree from filtered files
|
||||||
|
const tree = useMemo(() => buildFileTree(filtered), [filtered]);
|
||||||
|
|
||||||
|
// If all files are in root (no folders), skip the tree and show flat list
|
||||||
|
const hasNesting = useMemo(() => {
|
||||||
|
return filtered.some((f) => f.path.replace(/\\/g, "/").includes("/"));
|
||||||
|
}, [filtered]);
|
||||||
|
|
||||||
|
return (
|
||||||
|
<Dialog open={open} onOpenChange={onOpenChange}>
|
||||||
|
<DialogContent className="sm:max-w-2xl max-h-[80vh] flex flex-col gap-0 p-0">
|
||||||
|
<DialogHeader className="px-6 pt-6 pb-4 border-b border-border space-y-3">
|
||||||
|
{/* Preview image + title row */}
|
||||||
|
<div className="flex gap-4">
|
||||||
|
{pkg?.hasPreview && (
|
||||||
|
<img
|
||||||
|
src={`/api/zips/${pkg.id}/preview`}
|
||||||
|
alt=""
|
||||||
|
className="h-20 w-20 rounded-lg object-cover bg-muted shrink-0"
|
||||||
|
/>
|
||||||
|
)}
|
||||||
|
<div className="min-w-0 flex-1">
|
||||||
|
<DialogTitle className="truncate pr-8">
|
||||||
|
{pkg?.fileName ?? "Package Files"}
|
||||||
|
</DialogTitle>
|
||||||
|
<DialogDescription className="mt-1">
|
||||||
|
{total.toLocaleString()} file{total !== 1 ? "s" : ""} in archive
|
||||||
|
</DialogDescription>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Search within file list */}
|
||||||
|
{files.length > 0 && (
|
||||||
|
<div className="relative">
|
||||||
|
<Search className="absolute left-2.5 top-2.5 h-4 w-4 text-muted-foreground" />
|
||||||
|
<Input
|
||||||
|
placeholder="Filter files..."
|
||||||
|
value={search}
|
||||||
|
onChange={(e) => setSearch(e.target.value)}
|
||||||
|
className="pl-9 h-9"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</DialogHeader>
|
||||||
|
|
||||||
|
<ScrollArea className="flex-1 min-h-0">
|
||||||
|
<div className="px-4 py-3">
|
||||||
|
{loading ? (
|
||||||
|
<div className="flex flex-col items-center justify-center gap-2 py-12">
|
||||||
|
<Loader2 className="h-5 w-5 animate-spin text-muted-foreground" />
|
||||||
|
<span className="text-sm text-muted-foreground">Loading files...</span>
|
||||||
|
</div>
|
||||||
|
) : filtered.length === 0 ? (
|
||||||
|
<div className="flex flex-col items-center justify-center gap-2 py-12">
|
||||||
|
<FileText className="h-6 w-6 text-muted-foreground/50" />
|
||||||
|
<span className="text-sm text-muted-foreground">
|
||||||
|
{search ? "No matching files" : "No files indexed"}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
) : hasNesting ? (
|
||||||
|
<>
|
||||||
|
{/* Render as folder tree */}
|
||||||
|
{Array.from(tree.children.values())
|
||||||
|
.sort((a, b) => {
|
||||||
|
if (a.isFolder !== b.isFolder) return a.isFolder ? -1 : 1;
|
||||||
|
return a.name.localeCompare(b.name);
|
||||||
|
})
|
||||||
|
.map((child) => (
|
||||||
|
<TreeNodeView
|
||||||
|
key={child.name}
|
||||||
|
node={child}
|
||||||
|
depth={0}
|
||||||
|
search={search}
|
||||||
|
defaultOpen={true}
|
||||||
|
/>
|
||||||
|
))}
|
||||||
|
</>
|
||||||
|
) : (
|
||||||
|
<>
|
||||||
|
{/* Flat list for archives without folders */}
|
||||||
|
{filtered.map((file) => (
|
||||||
|
<div
|
||||||
|
key={file.id}
|
||||||
|
className="flex items-center gap-3 rounded-md px-2 py-1.5 hover:bg-muted/50 transition-colors"
|
||||||
|
>
|
||||||
|
<FileText className="h-3.5 w-3.5 shrink-0 text-muted-foreground" />
|
||||||
|
<div className="min-w-0 flex-1">
|
||||||
|
<p className="text-sm truncate" title={file.path}>
|
||||||
|
{file.fileName}
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
{file.extension && (
|
||||||
|
<Badge
|
||||||
|
variant="outline"
|
||||||
|
className={`text-[10px] shrink-0 ${getExtBadgeClass(file.extension)}`}
|
||||||
|
>
|
||||||
|
.{file.extension}
|
||||||
|
</Badge>
|
||||||
|
)}
|
||||||
|
<span className="text-[11px] text-muted-foreground shrink-0 tabular-nums">
|
||||||
|
{formatBytes(file.uncompressedSize)}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
))}
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{/* Load more button */}
|
||||||
|
{hasMore && !search && (
|
||||||
|
<div className="flex justify-center pt-3 pb-1">
|
||||||
|
<Button
|
||||||
|
variant="outline"
|
||||||
|
size="sm"
|
||||||
|
onClick={loadMore}
|
||||||
|
disabled={loadingMore}
|
||||||
|
className="gap-1"
|
||||||
|
>
|
||||||
|
{loadingMore ? (
|
||||||
|
<Loader2 className="h-3.5 w-3.5 animate-spin" />
|
||||||
|
) : (
|
||||||
|
<ChevronDown className="h-3.5 w-3.5" />
|
||||||
|
)}
|
||||||
|
Load more ({files.length} of {total.toLocaleString()})
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</ScrollArea>
|
||||||
|
</DialogContent>
|
||||||
|
</Dialog>
|
||||||
|
);
|
||||||
|
}
|
||||||
95
src/app/(app)/stls/_components/stl-table.tsx
Normal file
95
src/app/(app)/stls/_components/stl-table.tsx
Normal file
@@ -0,0 +1,95 @@
|
|||||||
|
"use client";
|
||||||
|
|
||||||
|
import { useState, useCallback } from "react";
|
||||||
|
import { useRouter, usePathname, useSearchParams } from "next/navigation";
|
||||||
|
import { Search, FileBox } from "lucide-react";
|
||||||
|
import { useDataTable } from "@/hooks/use-data-table";
|
||||||
|
import { getPackageColumns, type PackageRow } from "./package-columns";
|
||||||
|
import { PackageFilesDrawer } from "./package-files-drawer";
|
||||||
|
import { IngestionStatus } from "./ingestion-status";
|
||||||
|
import { DataTable } from "@/components/shared/data-table";
|
||||||
|
import { DataTablePagination } from "@/components/shared/data-table-pagination";
|
||||||
|
import { DataTableViewOptions } from "@/components/shared/data-table-view-options";
|
||||||
|
import { PageHeader } from "@/components/shared/page-header";
|
||||||
|
import { Input } from "@/components/ui/input";
|
||||||
|
import type { IngestionAccountStatus } from "@/lib/telegram/types";
|
||||||
|
|
||||||
|
interface StlTableProps {
|
||||||
|
data: PackageRow[];
|
||||||
|
pageCount: number;
|
||||||
|
totalCount: number;
|
||||||
|
ingestionStatus: IngestionAccountStatus[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export function StlTable({
|
||||||
|
data,
|
||||||
|
pageCount,
|
||||||
|
totalCount,
|
||||||
|
ingestionStatus,
|
||||||
|
}: StlTableProps) {
|
||||||
|
const router = useRouter();
|
||||||
|
const pathname = usePathname();
|
||||||
|
const searchParams = useSearchParams();
|
||||||
|
|
||||||
|
const [searchValue, setSearchValue] = useState(searchParams.get("search") ?? "");
|
||||||
|
const [viewPkg, setViewPkg] = useState<PackageRow | null>(null);
|
||||||
|
|
||||||
|
const updateSearch = useCallback(
|
||||||
|
(value: string) => {
|
||||||
|
setSearchValue(value);
|
||||||
|
const params = new URLSearchParams(searchParams.toString());
|
||||||
|
if (value) {
|
||||||
|
params.set("search", value);
|
||||||
|
params.set("page", "1");
|
||||||
|
} else {
|
||||||
|
params.delete("search");
|
||||||
|
}
|
||||||
|
router.push(`${pathname}?${params.toString()}`, { scroll: false });
|
||||||
|
},
|
||||||
|
[router, pathname, searchParams]
|
||||||
|
);
|
||||||
|
|
||||||
|
const columns = getPackageColumns({
|
||||||
|
onViewFiles: (pkg) => setViewPkg(pkg),
|
||||||
|
});
|
||||||
|
|
||||||
|
const { table } = useDataTable({ data, columns, pageCount });
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="space-y-4">
|
||||||
|
<PageHeader
|
||||||
|
title="STL Files"
|
||||||
|
description="Browse indexed archive packages from Telegram channels"
|
||||||
|
>
|
||||||
|
<IngestionStatus initialStatus={ingestionStatus} />
|
||||||
|
</PageHeader>
|
||||||
|
|
||||||
|
<div className="flex flex-wrap items-center gap-2">
|
||||||
|
<div className="relative flex-1 min-w-[200px] max-w-sm">
|
||||||
|
<Search className="absolute left-2.5 top-2.5 h-4 w-4 text-muted-foreground" />
|
||||||
|
<Input
|
||||||
|
placeholder="Search packages or files..."
|
||||||
|
value={searchValue}
|
||||||
|
onChange={(e) => updateSearch(e.target.value)}
|
||||||
|
className="pl-9 h-9"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<DataTableViewOptions table={table} />
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<DataTable
|
||||||
|
table={table}
|
||||||
|
emptyMessage="No packages found. Archives will appear here after ingestion."
|
||||||
|
/>
|
||||||
|
<DataTablePagination table={table} totalCount={totalCount} />
|
||||||
|
|
||||||
|
<PackageFilesDrawer
|
||||||
|
pkg={viewPkg}
|
||||||
|
open={!!viewPkg}
|
||||||
|
onOpenChange={(open) => {
|
||||||
|
if (!open) setViewPkg(null);
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
50
src/app/(app)/stls/page.tsx
Normal file
50
src/app/(app)/stls/page.tsx
Normal file
@@ -0,0 +1,50 @@
|
|||||||
|
import { auth } from "@/lib/auth";
|
||||||
|
import { redirect } from "next/navigation";
|
||||||
|
import { listPackages, searchPackages, getIngestionStatus } from "@/lib/telegram/queries";
|
||||||
|
import { StlTable } from "./_components/stl-table";
|
||||||
|
|
||||||
|
interface Props {
|
||||||
|
searchParams: Promise<Record<string, string | string[] | undefined>>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export default async function StlFilesPage({ searchParams }: Props) {
|
||||||
|
const session = await auth();
|
||||||
|
if (!session?.user?.id) redirect("/login");
|
||||||
|
|
||||||
|
const params = await searchParams;
|
||||||
|
|
||||||
|
const page = Number(params.page) || 1;
|
||||||
|
const perPage = Number(params.perPage) || 20;
|
||||||
|
const sort = (params.sort as string) ?? "indexedAt";
|
||||||
|
const order = (params.order as "asc" | "desc") ?? "desc";
|
||||||
|
const search = (params.search as string) ?? "";
|
||||||
|
const creator = (params.creator as string) || undefined;
|
||||||
|
|
||||||
|
// Fetch packages and ingestion status in parallel
|
||||||
|
const [result, ingestionStatus] = await Promise.all([
|
||||||
|
search
|
||||||
|
? searchPackages({
|
||||||
|
query: search,
|
||||||
|
page,
|
||||||
|
limit: perPage,
|
||||||
|
searchIn: "both",
|
||||||
|
})
|
||||||
|
: listPackages({
|
||||||
|
page,
|
||||||
|
limit: perPage,
|
||||||
|
creator,
|
||||||
|
sortBy: sort as "indexedAt" | "fileName" | "fileSize",
|
||||||
|
order,
|
||||||
|
}),
|
||||||
|
getIngestionStatus(),
|
||||||
|
]);
|
||||||
|
|
||||||
|
return (
|
||||||
|
<StlTable
|
||||||
|
data={result.items}
|
||||||
|
pageCount={result.pagination.totalPages}
|
||||||
|
totalCount={result.pagination.total}
|
||||||
|
ingestionStatus={ingestionStatus}
|
||||||
|
/>
|
||||||
|
);
|
||||||
|
}
|
||||||
184
src/app/(app)/telegram/_components/account-columns.tsx
Normal file
184
src/app/(app)/telegram/_components/account-columns.tsx
Normal file
@@ -0,0 +1,184 @@
|
|||||||
|
"use client";
|
||||||
|
|
||||||
|
import { type ColumnDef } from "@tanstack/react-table";
|
||||||
|
import {
|
||||||
|
MoreHorizontal,
|
||||||
|
Pencil,
|
||||||
|
Trash2,
|
||||||
|
Power,
|
||||||
|
Link2,
|
||||||
|
Play,
|
||||||
|
KeyRound,
|
||||||
|
} from "lucide-react";
|
||||||
|
import { Badge } from "@/components/ui/badge";
|
||||||
|
import { Button } from "@/components/ui/button";
|
||||||
|
import {
|
||||||
|
DropdownMenu,
|
||||||
|
DropdownMenuContent,
|
||||||
|
DropdownMenuItem,
|
||||||
|
DropdownMenuSeparator,
|
||||||
|
DropdownMenuTrigger,
|
||||||
|
} from "@/components/ui/dropdown-menu";
|
||||||
|
import type { AccountRow } from "@/lib/telegram/admin-queries";
|
||||||
|
|
||||||
|
const authStateColors: Record<string, string> = {
|
||||||
|
PENDING: "bg-yellow-500/10 text-yellow-600 border-yellow-500/20",
|
||||||
|
AWAITING_CODE: "bg-orange-500/10 text-orange-600 border-orange-500/20",
|
||||||
|
AWAITING_PASSWORD: "bg-orange-500/10 text-orange-600 border-orange-500/20",
|
||||||
|
AUTHENTICATED: "bg-green-500/10 text-green-600 border-green-500/20",
|
||||||
|
EXPIRED: "bg-red-500/10 text-red-600 border-red-500/20",
|
||||||
|
};
|
||||||
|
|
||||||
|
interface AccountColumnsProps {
|
||||||
|
onEdit: (account: AccountRow) => void;
|
||||||
|
onToggleActive: (id: string) => void;
|
||||||
|
onDelete: (id: string) => void;
|
||||||
|
onViewLinks: (id: string) => void;
|
||||||
|
onTriggerSync: (id: string) => void;
|
||||||
|
onEnterCode: (account: AccountRow) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getAccountColumns({
|
||||||
|
onEdit,
|
||||||
|
onToggleActive,
|
||||||
|
onDelete,
|
||||||
|
onViewLinks,
|
||||||
|
onTriggerSync,
|
||||||
|
onEnterCode,
|
||||||
|
}: AccountColumnsProps): ColumnDef<AccountRow, unknown>[] {
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
accessorKey: "displayName",
|
||||||
|
header: "Account",
|
||||||
|
cell: ({ row }) => (
|
||||||
|
<div className="flex flex-col">
|
||||||
|
<span className="font-medium">
|
||||||
|
{row.original.displayName || row.original.phone}
|
||||||
|
</span>
|
||||||
|
{row.original.displayName && (
|
||||||
|
<span className="text-xs text-muted-foreground">
|
||||||
|
{row.original.phone}
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
),
|
||||||
|
enableHiding: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
accessorKey: "authState",
|
||||||
|
header: "Auth State",
|
||||||
|
cell: ({ row }) => {
|
||||||
|
const needsCode =
|
||||||
|
row.original.authState === "AWAITING_CODE" ||
|
||||||
|
row.original.authState === "AWAITING_PASSWORD";
|
||||||
|
return (
|
||||||
|
<div className="flex items-center gap-2">
|
||||||
|
<Badge
|
||||||
|
variant="outline"
|
||||||
|
className={authStateColors[row.original.authState] ?? ""}
|
||||||
|
>
|
||||||
|
{row.original.authState.replace(/_/g, " ")}
|
||||||
|
</Badge>
|
||||||
|
{needsCode && (
|
||||||
|
<Button
|
||||||
|
variant="outline"
|
||||||
|
size="sm"
|
||||||
|
className="h-6 gap-1 px-2 text-xs"
|
||||||
|
onClick={() => onEnterCode(row.original)}
|
||||||
|
>
|
||||||
|
<KeyRound className="h-3 w-3" />
|
||||||
|
Enter Code
|
||||||
|
</Button>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
accessorKey: "isActive",
|
||||||
|
header: "Status",
|
||||||
|
cell: ({ row }) => (
|
||||||
|
<Badge variant={row.original.isActive ? "default" : "secondary"}>
|
||||||
|
{row.original.isActive ? "Active" : "Disabled"}
|
||||||
|
</Badge>
|
||||||
|
),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "channels",
|
||||||
|
header: "Channels",
|
||||||
|
cell: ({ row }) => (
|
||||||
|
<Button
|
||||||
|
variant="ghost"
|
||||||
|
size="sm"
|
||||||
|
className="h-7 gap-1 px-2 text-xs"
|
||||||
|
onClick={() => onViewLinks(row.original.id)}
|
||||||
|
>
|
||||||
|
<Link2 className="h-3 w-3" />
|
||||||
|
{row.original.channelCount}
|
||||||
|
</Button>
|
||||||
|
),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "runs",
|
||||||
|
header: "Runs",
|
||||||
|
cell: ({ row }) => (
|
||||||
|
<span className="text-sm text-muted-foreground">
|
||||||
|
{row.original.runCount}
|
||||||
|
</span>
|
||||||
|
),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
accessorKey: "lastSeenAt",
|
||||||
|
header: "Last Seen",
|
||||||
|
cell: ({ row }) =>
|
||||||
|
row.original.lastSeenAt ? (
|
||||||
|
<span className="text-sm text-muted-foreground">
|
||||||
|
{new Date(row.original.lastSeenAt).toLocaleDateString()}
|
||||||
|
</span>
|
||||||
|
) : (
|
||||||
|
<span className="text-sm text-muted-foreground">Never</span>
|
||||||
|
),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "actions",
|
||||||
|
cell: ({ row }) => (
|
||||||
|
<DropdownMenu>
|
||||||
|
<DropdownMenuTrigger asChild>
|
||||||
|
<Button variant="ghost" size="icon" className="h-8 w-8">
|
||||||
|
<MoreHorizontal className="h-4 w-4" />
|
||||||
|
</Button>
|
||||||
|
</DropdownMenuTrigger>
|
||||||
|
<DropdownMenuContent align="end">
|
||||||
|
<DropdownMenuItem onClick={() => onEdit(row.original)}>
|
||||||
|
<Pencil className="mr-2 h-3.5 w-3.5" />
|
||||||
|
Edit
|
||||||
|
</DropdownMenuItem>
|
||||||
|
<DropdownMenuItem onClick={() => onViewLinks(row.original.id)}>
|
||||||
|
<Link2 className="mr-2 h-3.5 w-3.5" />
|
||||||
|
Manage Channels
|
||||||
|
</DropdownMenuItem>
|
||||||
|
<DropdownMenuItem onClick={() => onTriggerSync(row.original.id)}>
|
||||||
|
<Play className="mr-2 h-3.5 w-3.5" />
|
||||||
|
Sync Now
|
||||||
|
</DropdownMenuItem>
|
||||||
|
<DropdownMenuItem
|
||||||
|
onClick={() => onToggleActive(row.original.id)}
|
||||||
|
>
|
||||||
|
<Power className="mr-2 h-3.5 w-3.5" />
|
||||||
|
{row.original.isActive ? "Disable" : "Enable"}
|
||||||
|
</DropdownMenuItem>
|
||||||
|
<DropdownMenuSeparator />
|
||||||
|
<DropdownMenuItem
|
||||||
|
onClick={() => onDelete(row.original.id)}
|
||||||
|
className="text-destructive focus:text-destructive"
|
||||||
|
>
|
||||||
|
<Trash2 className="mr-2 h-3.5 w-3.5" />
|
||||||
|
Delete
|
||||||
|
</DropdownMenuItem>
|
||||||
|
</DropdownMenuContent>
|
||||||
|
</DropdownMenu>
|
||||||
|
),
|
||||||
|
enableHiding: false,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
}
|
||||||
102
src/app/(app)/telegram/_components/account-form.tsx
Normal file
102
src/app/(app)/telegram/_components/account-form.tsx
Normal file
@@ -0,0 +1,102 @@
|
|||||||
|
"use client";
|
||||||
|
|
||||||
|
import { useTransition } from "react";
|
||||||
|
import { useForm } from "react-hook-form";
|
||||||
|
import { zodResolver } from "@hookform/resolvers/zod";
|
||||||
|
import { toast } from "sonner";
|
||||||
|
import {
|
||||||
|
telegramAccountSchema,
|
||||||
|
type TelegramAccountInput,
|
||||||
|
} from "@/schemas/telegram";
|
||||||
|
import { createAccount, updateAccount } from "../actions";
|
||||||
|
import { Button } from "@/components/ui/button";
|
||||||
|
import { Input } from "@/components/ui/input";
|
||||||
|
import {
|
||||||
|
Form,
|
||||||
|
FormControl,
|
||||||
|
FormDescription,
|
||||||
|
FormField,
|
||||||
|
FormItem,
|
||||||
|
FormLabel,
|
||||||
|
FormMessage,
|
||||||
|
} from "@/components/ui/form";
|
||||||
|
import type { AccountRow } from "@/lib/telegram/admin-queries";
|
||||||
|
|
||||||
|
interface AccountFormProps {
|
||||||
|
account?: AccountRow;
|
||||||
|
onSuccess: () => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function AccountForm({ account, onSuccess }: AccountFormProps) {
|
||||||
|
const [isPending, startTransition] = useTransition();
|
||||||
|
const isEditing = !!account;
|
||||||
|
|
||||||
|
const form = useForm<TelegramAccountInput>({
|
||||||
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||||
|
resolver: zodResolver(telegramAccountSchema) as any,
|
||||||
|
defaultValues: {
|
||||||
|
phone: account?.phone ?? "",
|
||||||
|
displayName: account?.displayName ?? "",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
function onSubmit(values: TelegramAccountInput) {
|
||||||
|
startTransition(async () => {
|
||||||
|
const result = isEditing
|
||||||
|
? await updateAccount(account!.id, values)
|
||||||
|
: await createAccount(values);
|
||||||
|
|
||||||
|
if (!result.success) {
|
||||||
|
toast.error(result.error);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
toast.success(isEditing ? "Account updated" : "Account created");
|
||||||
|
form.reset();
|
||||||
|
onSuccess();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<Form {...form}>
|
||||||
|
<form onSubmit={form.handleSubmit(onSubmit)} className="space-y-4">
|
||||||
|
<FormField
|
||||||
|
control={form.control}
|
||||||
|
name="phone"
|
||||||
|
render={({ field }) => (
|
||||||
|
<FormItem>
|
||||||
|
<FormLabel>Phone Number</FormLabel>
|
||||||
|
<FormControl>
|
||||||
|
<Input placeholder="+31612345678" {...field} />
|
||||||
|
</FormControl>
|
||||||
|
<FormDescription>
|
||||||
|
International format with country code
|
||||||
|
</FormDescription>
|
||||||
|
<FormMessage />
|
||||||
|
</FormItem>
|
||||||
|
)}
|
||||||
|
/>
|
||||||
|
|
||||||
|
<FormField
|
||||||
|
control={form.control}
|
||||||
|
name="displayName"
|
||||||
|
render={({ field }) => (
|
||||||
|
<FormItem>
|
||||||
|
<FormLabel>Display Name</FormLabel>
|
||||||
|
<FormControl>
|
||||||
|
<Input placeholder="My Bot Account" {...field} />
|
||||||
|
</FormControl>
|
||||||
|
<FormMessage />
|
||||||
|
</FormItem>
|
||||||
|
)}
|
||||||
|
/>
|
||||||
|
|
||||||
|
<div className="flex justify-end gap-2">
|
||||||
|
<Button type="submit" disabled={isPending}>
|
||||||
|
{isPending ? "Saving..." : isEditing ? "Update" : "Create"}
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
</form>
|
||||||
|
</Form>
|
||||||
|
);
|
||||||
|
}
|
||||||
233
src/app/(app)/telegram/_components/account-links-drawer.tsx
Normal file
233
src/app/(app)/telegram/_components/account-links-drawer.tsx
Normal file
@@ -0,0 +1,233 @@
|
|||||||
|
"use client";
|
||||||
|
|
||||||
|
import { useState, useEffect, useTransition, useCallback } from "react";
|
||||||
|
import { Link2Off, Plus } from "lucide-react";
|
||||||
|
import { toast } from "sonner";
|
||||||
|
import { linkChannel, unlinkChannel } from "../actions";
|
||||||
|
import {
|
||||||
|
Dialog,
|
||||||
|
DialogContent,
|
||||||
|
DialogDescription,
|
||||||
|
DialogHeader,
|
||||||
|
DialogTitle,
|
||||||
|
} from "@/components/ui/dialog";
|
||||||
|
import {
|
||||||
|
Select,
|
||||||
|
SelectContent,
|
||||||
|
SelectItem,
|
||||||
|
SelectTrigger,
|
||||||
|
SelectValue,
|
||||||
|
} from "@/components/ui/select";
|
||||||
|
import { Badge } from "@/components/ui/badge";
|
||||||
|
import { Button } from "@/components/ui/button";
|
||||||
|
import { Separator } from "@/components/ui/separator";
|
||||||
|
|
||||||
|
interface ChannelLink {
|
||||||
|
id: string;
|
||||||
|
channelId: string;
|
||||||
|
role: string;
|
||||||
|
lastProcessedMessageId: string | null;
|
||||||
|
channel: {
|
||||||
|
id: string;
|
||||||
|
title: string;
|
||||||
|
type: string;
|
||||||
|
telegramId: string;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
interface UnlinkedChannel {
|
||||||
|
id: string;
|
||||||
|
title: string;
|
||||||
|
type: string;
|
||||||
|
telegramId: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface AccountLinksDrawerProps {
|
||||||
|
accountId: string | null;
|
||||||
|
open: boolean;
|
||||||
|
onOpenChange: (open: boolean) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function AccountLinksDrawer({
|
||||||
|
accountId,
|
||||||
|
open,
|
||||||
|
onOpenChange,
|
||||||
|
}: AccountLinksDrawerProps) {
|
||||||
|
const [isPending, startTransition] = useTransition();
|
||||||
|
const [links, setLinks] = useState<ChannelLink[]>([]);
|
||||||
|
const [unlinked, setUnlinked] = useState<UnlinkedChannel[]>([]);
|
||||||
|
const [selectedChannelId, setSelectedChannelId] = useState("");
|
||||||
|
const [selectedRole, setSelectedRole] = useState<"READER" | "WRITER">("READER");
|
||||||
|
const [loading, setLoading] = useState(false);
|
||||||
|
|
||||||
|
const fetchLinks = useCallback(async () => {
|
||||||
|
if (!accountId) return;
|
||||||
|
setLoading(true);
|
||||||
|
try {
|
||||||
|
const [linksRes, unlinkedRes] = await Promise.all([
|
||||||
|
fetch(`/api/telegram/accounts/${accountId}/links`),
|
||||||
|
fetch(`/api/telegram/accounts/${accountId}/unlinked-channels`),
|
||||||
|
]);
|
||||||
|
if (linksRes.ok) setLinks(await linksRes.json());
|
||||||
|
if (unlinkedRes.ok) setUnlinked(await unlinkedRes.json());
|
||||||
|
} catch {
|
||||||
|
toast.error("Failed to load channel links");
|
||||||
|
}
|
||||||
|
setLoading(false);
|
||||||
|
}, [accountId]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (open && accountId) {
|
||||||
|
fetchLinks();
|
||||||
|
}
|
||||||
|
}, [open, accountId, fetchLinks]);
|
||||||
|
|
||||||
|
const handleLink = () => {
|
||||||
|
if (!accountId || !selectedChannelId) return;
|
||||||
|
startTransition(async () => {
|
||||||
|
const result = await linkChannel({
|
||||||
|
accountId,
|
||||||
|
channelId: selectedChannelId,
|
||||||
|
role: selectedRole,
|
||||||
|
});
|
||||||
|
if (result.success) {
|
||||||
|
toast.success("Channel linked");
|
||||||
|
setSelectedChannelId("");
|
||||||
|
await fetchLinks();
|
||||||
|
} else {
|
||||||
|
toast.error(result.error);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
const handleUnlink = (linkId: string) => {
|
||||||
|
startTransition(async () => {
|
||||||
|
const result = await unlinkChannel(linkId);
|
||||||
|
if (result.success) {
|
||||||
|
toast.success("Channel unlinked");
|
||||||
|
await fetchLinks();
|
||||||
|
} else {
|
||||||
|
toast.error(result.error);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<Dialog open={open} onOpenChange={onOpenChange}>
|
||||||
|
<DialogContent className="sm:max-w-lg">
|
||||||
|
<DialogHeader>
|
||||||
|
<DialogTitle>Manage Channel Links</DialogTitle>
|
||||||
|
<DialogDescription>
|
||||||
|
Link channels to this account. The account will read from Source
|
||||||
|
channels and write to Destination channels.
|
||||||
|
</DialogDescription>
|
||||||
|
</DialogHeader>
|
||||||
|
|
||||||
|
{/* Add new link */}
|
||||||
|
{unlinked.length > 0 && (
|
||||||
|
<div className="space-y-3">
|
||||||
|
<h4 className="text-sm font-medium">Link a Channel</h4>
|
||||||
|
<div className="flex items-end gap-2">
|
||||||
|
<div className="flex-1">
|
||||||
|
<Select
|
||||||
|
value={selectedChannelId}
|
||||||
|
onValueChange={setSelectedChannelId}
|
||||||
|
>
|
||||||
|
<SelectTrigger>
|
||||||
|
<SelectValue placeholder="Select channel" />
|
||||||
|
</SelectTrigger>
|
||||||
|
<SelectContent>
|
||||||
|
{unlinked.map((ch) => (
|
||||||
|
<SelectItem key={ch.id} value={ch.id}>
|
||||||
|
{ch.title} ({ch.type})
|
||||||
|
</SelectItem>
|
||||||
|
))}
|
||||||
|
</SelectContent>
|
||||||
|
</Select>
|
||||||
|
</div>
|
||||||
|
<Select
|
||||||
|
value={selectedRole}
|
||||||
|
onValueChange={(v) => setSelectedRole(v as "READER" | "WRITER")}
|
||||||
|
>
|
||||||
|
<SelectTrigger className="w-28">
|
||||||
|
<SelectValue />
|
||||||
|
</SelectTrigger>
|
||||||
|
<SelectContent>
|
||||||
|
<SelectItem value="READER">Reader</SelectItem>
|
||||||
|
<SelectItem value="WRITER">Writer</SelectItem>
|
||||||
|
</SelectContent>
|
||||||
|
</Select>
|
||||||
|
<Button
|
||||||
|
size="sm"
|
||||||
|
disabled={!selectedChannelId || isPending}
|
||||||
|
onClick={handleLink}
|
||||||
|
>
|
||||||
|
<Plus className="mr-1 h-3.5 w-3.5" />
|
||||||
|
Link
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
<Separator />
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{/* Existing links */}
|
||||||
|
<div className="space-y-2">
|
||||||
|
<h4 className="text-sm font-medium">
|
||||||
|
Linked Channels ({links.length})
|
||||||
|
</h4>
|
||||||
|
{loading ? (
|
||||||
|
<p className="text-sm text-muted-foreground">Loading...</p>
|
||||||
|
) : links.length === 0 ? (
|
||||||
|
<p className="text-sm text-muted-foreground">
|
||||||
|
No channels linked to this account.
|
||||||
|
</p>
|
||||||
|
) : (
|
||||||
|
<div className="space-y-2">
|
||||||
|
{links.map((link) => (
|
||||||
|
<div
|
||||||
|
key={link.id}
|
||||||
|
className="flex items-center justify-between rounded-md border p-3"
|
||||||
|
>
|
||||||
|
<div className="flex flex-col gap-1">
|
||||||
|
<div className="flex items-center gap-2">
|
||||||
|
<span className="text-sm font-medium">
|
||||||
|
{link.channel.title}
|
||||||
|
</span>
|
||||||
|
<Badge
|
||||||
|
variant="outline"
|
||||||
|
className={
|
||||||
|
link.channel.type === "SOURCE"
|
||||||
|
? "bg-blue-500/10 text-blue-600 border-blue-500/20"
|
||||||
|
: "bg-purple-500/10 text-purple-600 border-purple-500/20"
|
||||||
|
}
|
||||||
|
>
|
||||||
|
{link.channel.type}
|
||||||
|
</Badge>
|
||||||
|
<Badge variant="secondary" className="text-[10px]">
|
||||||
|
{link.role}
|
||||||
|
</Badge>
|
||||||
|
</div>
|
||||||
|
<span className="text-xs text-muted-foreground">
|
||||||
|
ID: {link.channel.telegramId}
|
||||||
|
{link.lastProcessedMessageId &&
|
||||||
|
` | Last msg: ${link.lastProcessedMessageId}`}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
<Button
|
||||||
|
variant="ghost"
|
||||||
|
size="icon"
|
||||||
|
className="h-8 w-8 text-muted-foreground hover:text-destructive"
|
||||||
|
disabled={isPending}
|
||||||
|
onClick={() => handleUnlink(link.id)}
|
||||||
|
>
|
||||||
|
<Link2Off className="h-4 w-4" />
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</DialogContent>
|
||||||
|
</Dialog>
|
||||||
|
);
|
||||||
|
}
|
||||||
44
src/app/(app)/telegram/_components/account-modal.tsx
Normal file
44
src/app/(app)/telegram/_components/account-modal.tsx
Normal file
@@ -0,0 +1,44 @@
|
|||||||
|
"use client";
|
||||||
|
|
||||||
|
import {
|
||||||
|
Dialog,
|
||||||
|
DialogContent,
|
||||||
|
DialogDescription,
|
||||||
|
DialogHeader,
|
||||||
|
DialogTitle,
|
||||||
|
} from "@/components/ui/dialog";
|
||||||
|
import { AccountForm } from "./account-form";
|
||||||
|
import type { AccountRow } from "@/lib/telegram/admin-queries";
|
||||||
|
|
||||||
|
interface AccountModalProps {
|
||||||
|
open: boolean;
|
||||||
|
onOpenChange: (open: boolean) => void;
|
||||||
|
account?: AccountRow;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function AccountModal({
|
||||||
|
open,
|
||||||
|
onOpenChange,
|
||||||
|
account,
|
||||||
|
}: AccountModalProps) {
|
||||||
|
return (
|
||||||
|
<Dialog open={open} onOpenChange={onOpenChange}>
|
||||||
|
<DialogContent className="sm:max-w-lg">
|
||||||
|
<DialogHeader>
|
||||||
|
<DialogTitle>
|
||||||
|
{account ? "Edit Account" : "Add Telegram Account"}
|
||||||
|
</DialogTitle>
|
||||||
|
<DialogDescription>
|
||||||
|
{account
|
||||||
|
? "Update the account details below."
|
||||||
|
: "Configure a new Telegram account for ingestion. You'll need an API ID and hash from my.telegram.org."}
|
||||||
|
</DialogDescription>
|
||||||
|
</DialogHeader>
|
||||||
|
<AccountForm
|
||||||
|
account={account}
|
||||||
|
onSuccess={() => onOpenChange(false)}
|
||||||
|
/>
|
||||||
|
</DialogContent>
|
||||||
|
</Dialog>
|
||||||
|
);
|
||||||
|
}
|
||||||
140
src/app/(app)/telegram/_components/accounts-tab.tsx
Normal file
140
src/app/(app)/telegram/_components/accounts-tab.tsx
Normal file
@@ -0,0 +1,140 @@
|
|||||||
|
"use client";
|
||||||
|
|
||||||
|
import { useState, useTransition } from "react";
|
||||||
|
import { Plus, Play } from "lucide-react";
|
||||||
|
import { toast } from "sonner";
|
||||||
|
import { getAccountColumns } from "./account-columns";
|
||||||
|
import { AccountModal } from "./account-modal";
|
||||||
|
import { AccountLinksDrawer } from "./account-links-drawer";
|
||||||
|
import { AuthCodeDialog } from "./auth-code-dialog";
|
||||||
|
import { deleteAccount, toggleAccountActive, triggerIngestion } from "../actions";
|
||||||
|
import { DataTable } from "@/components/shared/data-table";
|
||||||
|
import { DeleteDialog } from "@/components/shared/delete-dialog";
|
||||||
|
import { Button } from "@/components/ui/button";
|
||||||
|
import type { AccountRow } from "@/lib/telegram/admin-queries";
|
||||||
|
import { useDataTable } from "@/hooks/use-data-table";
|
||||||
|
|
||||||
|
interface AccountsTabProps {
|
||||||
|
accounts: AccountRow[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export function AccountsTab({ accounts }: AccountsTabProps) {
|
||||||
|
const [isPending, startTransition] = useTransition();
|
||||||
|
const [modalOpen, setModalOpen] = useState(false);
|
||||||
|
const [editAccount, setEditAccount] = useState<AccountRow | undefined>();
|
||||||
|
const [deleteId, setDeleteId] = useState<string | null>(null);
|
||||||
|
const [linksAccountId, setLinksAccountId] = useState<string | null>(null);
|
||||||
|
const [authCodeAccount, setAuthCodeAccount] = useState<AccountRow | null>(null);
|
||||||
|
|
||||||
|
const columns = getAccountColumns({
|
||||||
|
onEdit: (account) => {
|
||||||
|
setEditAccount(account);
|
||||||
|
setModalOpen(true);
|
||||||
|
},
|
||||||
|
onToggleActive: (id) => {
|
||||||
|
startTransition(async () => {
|
||||||
|
const result = await toggleAccountActive(id);
|
||||||
|
if (result.success) toast.success("Account toggled");
|
||||||
|
else toast.error(result.error);
|
||||||
|
});
|
||||||
|
},
|
||||||
|
onDelete: (id) => setDeleteId(id),
|
||||||
|
onViewLinks: (id) => setLinksAccountId(id),
|
||||||
|
onEnterCode: (account) => setAuthCodeAccount(account),
|
||||||
|
onTriggerSync: (id) => {
|
||||||
|
startTransition(async () => {
|
||||||
|
const result = await triggerIngestion(id);
|
||||||
|
if (result.success) toast.success("Ingestion triggered");
|
||||||
|
else toast.error(result.error);
|
||||||
|
});
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const { table } = useDataTable({
|
||||||
|
data: accounts,
|
||||||
|
columns,
|
||||||
|
pageCount: 1,
|
||||||
|
});
|
||||||
|
|
||||||
|
const handleDelete = () => {
|
||||||
|
if (!deleteId) return;
|
||||||
|
startTransition(async () => {
|
||||||
|
const result = await deleteAccount(deleteId);
|
||||||
|
if (result.success) {
|
||||||
|
toast.success("Account deleted");
|
||||||
|
setDeleteId(null);
|
||||||
|
} else {
|
||||||
|
toast.error(result.error);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="space-y-4">
|
||||||
|
<div className="flex items-center gap-2">
|
||||||
|
<Button
|
||||||
|
onClick={() => {
|
||||||
|
setEditAccount(undefined);
|
||||||
|
setModalOpen(true);
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<Plus className="mr-2 h-4 w-4" />
|
||||||
|
Add Account
|
||||||
|
</Button>
|
||||||
|
<Button
|
||||||
|
variant="outline"
|
||||||
|
disabled={isPending}
|
||||||
|
onClick={() => {
|
||||||
|
startTransition(async () => {
|
||||||
|
const result = await triggerIngestion();
|
||||||
|
if (result.success) toast.success("Ingestion triggered for all accounts");
|
||||||
|
else toast.error(result.error);
|
||||||
|
});
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<Play className="mr-2 h-4 w-4" />
|
||||||
|
Sync All
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<DataTable
|
||||||
|
table={table}
|
||||||
|
emptyMessage="No accounts configured. Add your first Telegram account."
|
||||||
|
/>
|
||||||
|
|
||||||
|
<AccountModal
|
||||||
|
open={modalOpen}
|
||||||
|
onOpenChange={(open) => {
|
||||||
|
setModalOpen(open);
|
||||||
|
if (!open) setEditAccount(undefined);
|
||||||
|
}}
|
||||||
|
account={editAccount}
|
||||||
|
/>
|
||||||
|
|
||||||
|
<DeleteDialog
|
||||||
|
open={!!deleteId}
|
||||||
|
onOpenChange={(open) => !open && setDeleteId(null)}
|
||||||
|
title="Delete Account"
|
||||||
|
description="This will permanently delete this Telegram account and all its channel links. Existing packages will NOT be deleted."
|
||||||
|
onConfirm={handleDelete}
|
||||||
|
isLoading={isPending}
|
||||||
|
/>
|
||||||
|
|
||||||
|
<AccountLinksDrawer
|
||||||
|
accountId={linksAccountId}
|
||||||
|
open={!!linksAccountId}
|
||||||
|
onOpenChange={(open) => {
|
||||||
|
if (!open) setLinksAccountId(null);
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
|
||||||
|
<AuthCodeDialog
|
||||||
|
account={authCodeAccount}
|
||||||
|
open={!!authCodeAccount}
|
||||||
|
onOpenChange={(open) => {
|
||||||
|
if (!open) setAuthCodeAccount(null);
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
102
src/app/(app)/telegram/_components/auth-code-dialog.tsx
Normal file
102
src/app/(app)/telegram/_components/auth-code-dialog.tsx
Normal file
@@ -0,0 +1,102 @@
|
|||||||
|
"use client";
|
||||||
|
|
||||||
|
import { useState, useTransition } from "react";
|
||||||
|
import { toast } from "sonner";
|
||||||
|
import { submitAuthCode } from "../actions";
|
||||||
|
import { Button } from "@/components/ui/button";
|
||||||
|
import { Input } from "@/components/ui/input";
|
||||||
|
import {
|
||||||
|
Dialog,
|
||||||
|
DialogContent,
|
||||||
|
DialogDescription,
|
||||||
|
DialogFooter,
|
||||||
|
DialogHeader,
|
||||||
|
DialogTitle,
|
||||||
|
} from "@/components/ui/dialog";
|
||||||
|
import { Label } from "@/components/ui/label";
|
||||||
|
import type { AccountRow } from "@/lib/telegram/admin-queries";
|
||||||
|
|
||||||
|
interface AuthCodeDialogProps {
|
||||||
|
account: AccountRow | null;
|
||||||
|
open: boolean;
|
||||||
|
onOpenChange: (open: boolean) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function AuthCodeDialog({
|
||||||
|
account,
|
||||||
|
open,
|
||||||
|
onOpenChange,
|
||||||
|
}: AuthCodeDialogProps) {
|
||||||
|
const [code, setCode] = useState("");
|
||||||
|
const [isPending, startTransition] = useTransition();
|
||||||
|
|
||||||
|
const isPassword = account?.authState === "AWAITING_PASSWORD";
|
||||||
|
const title = isPassword ? "Enter 2FA Password" : "Enter Auth Code";
|
||||||
|
const description = isPassword
|
||||||
|
? "Your Telegram account requires a two-factor authentication password."
|
||||||
|
: "Enter the code sent to your Telegram app or SMS.";
|
||||||
|
const placeholder = isPassword ? "Password" : "12345";
|
||||||
|
|
||||||
|
function handleSubmit() {
|
||||||
|
if (!account || !code.trim()) return;
|
||||||
|
|
||||||
|
startTransition(async () => {
|
||||||
|
const result = await submitAuthCode(account.id, { code: code.trim() });
|
||||||
|
if (result.success) {
|
||||||
|
toast.success(isPassword ? "Password submitted" : "Code submitted");
|
||||||
|
setCode("");
|
||||||
|
onOpenChange(false);
|
||||||
|
} else {
|
||||||
|
toast.error(result.error);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<Dialog
|
||||||
|
open={open}
|
||||||
|
onOpenChange={(v) => {
|
||||||
|
if (!v) setCode("");
|
||||||
|
onOpenChange(v);
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<DialogContent className="sm:max-w-md">
|
||||||
|
<DialogHeader>
|
||||||
|
<DialogTitle>{title}</DialogTitle>
|
||||||
|
<DialogDescription>{description}</DialogDescription>
|
||||||
|
</DialogHeader>
|
||||||
|
<div className="space-y-2">
|
||||||
|
<Label htmlFor="auth-code">
|
||||||
|
{isPassword ? "Password" : "Code"}
|
||||||
|
</Label>
|
||||||
|
<Input
|
||||||
|
id="auth-code"
|
||||||
|
type={isPassword ? "password" : "text"}
|
||||||
|
placeholder={placeholder}
|
||||||
|
value={code}
|
||||||
|
onChange={(e) => setCode(e.target.value)}
|
||||||
|
onKeyDown={(e) => {
|
||||||
|
if (e.key === "Enter") handleSubmit();
|
||||||
|
}}
|
||||||
|
autoFocus
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<DialogFooter>
|
||||||
|
<Button
|
||||||
|
variant="outline"
|
||||||
|
onClick={() => onOpenChange(false)}
|
||||||
|
disabled={isPending}
|
||||||
|
>
|
||||||
|
Cancel
|
||||||
|
</Button>
|
||||||
|
<Button
|
||||||
|
onClick={handleSubmit}
|
||||||
|
disabled={isPending || !code.trim()}
|
||||||
|
>
|
||||||
|
{isPending ? "Submitting..." : "Submit"}
|
||||||
|
</Button>
|
||||||
|
</DialogFooter>
|
||||||
|
</DialogContent>
|
||||||
|
</Dialog>
|
||||||
|
);
|
||||||
|
}
|
||||||
132
src/app/(app)/telegram/_components/channel-columns.tsx
Normal file
132
src/app/(app)/telegram/_components/channel-columns.tsx
Normal file
@@ -0,0 +1,132 @@
|
|||||||
|
"use client";
|
||||||
|
|
||||||
|
import { type ColumnDef } from "@tanstack/react-table";
|
||||||
|
import {
|
||||||
|
MoreHorizontal,
|
||||||
|
Pencil,
|
||||||
|
Trash2,
|
||||||
|
Power,
|
||||||
|
} from "lucide-react";
|
||||||
|
import { Badge } from "@/components/ui/badge";
|
||||||
|
import { Button } from "@/components/ui/button";
|
||||||
|
import {
|
||||||
|
DropdownMenu,
|
||||||
|
DropdownMenuContent,
|
||||||
|
DropdownMenuItem,
|
||||||
|
DropdownMenuSeparator,
|
||||||
|
DropdownMenuTrigger,
|
||||||
|
} from "@/components/ui/dropdown-menu";
|
||||||
|
import type { ChannelRow } from "@/lib/telegram/admin-queries";
|
||||||
|
|
||||||
|
interface ChannelColumnsProps {
|
||||||
|
onEdit: (channel: ChannelRow) => void;
|
||||||
|
onToggleActive: (id: string) => void;
|
||||||
|
onDelete: (id: string) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getChannelColumns({
|
||||||
|
onEdit,
|
||||||
|
onToggleActive,
|
||||||
|
onDelete,
|
||||||
|
}: ChannelColumnsProps): ColumnDef<ChannelRow, unknown>[] {
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
accessorKey: "title",
|
||||||
|
header: "Channel",
|
||||||
|
cell: ({ row }) => (
|
||||||
|
<div className="flex flex-col">
|
||||||
|
<span className="font-medium">{row.original.title}</span>
|
||||||
|
<span className="text-xs text-muted-foreground">
|
||||||
|
ID: {row.original.telegramId}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
),
|
||||||
|
enableHiding: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
accessorKey: "type",
|
||||||
|
header: "Type",
|
||||||
|
cell: ({ row }) => (
|
||||||
|
<Badge
|
||||||
|
variant="outline"
|
||||||
|
className={
|
||||||
|
row.original.type === "SOURCE"
|
||||||
|
? "bg-blue-500/10 text-blue-600 border-blue-500/20"
|
||||||
|
: "bg-purple-500/10 text-purple-600 border-purple-500/20"
|
||||||
|
}
|
||||||
|
>
|
||||||
|
{row.original.type}
|
||||||
|
</Badge>
|
||||||
|
),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
accessorKey: "isActive",
|
||||||
|
header: "Status",
|
||||||
|
cell: ({ row }) => (
|
||||||
|
<Badge variant={row.original.isActive ? "default" : "secondary"}>
|
||||||
|
{row.original.isActive ? "Active" : "Disabled"}
|
||||||
|
</Badge>
|
||||||
|
),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "accounts",
|
||||||
|
header: "Accounts",
|
||||||
|
cell: ({ row }) => (
|
||||||
|
<span className="text-sm text-muted-foreground">
|
||||||
|
{row.original.accountCount}
|
||||||
|
</span>
|
||||||
|
),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "packages",
|
||||||
|
header: "Packages",
|
||||||
|
cell: ({ row }) => (
|
||||||
|
<span className="text-sm text-muted-foreground">
|
||||||
|
{row.original.packageCount}
|
||||||
|
</span>
|
||||||
|
),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
accessorKey: "createdAt",
|
||||||
|
header: "Created",
|
||||||
|
cell: ({ row }) => (
|
||||||
|
<span className="text-sm text-muted-foreground">
|
||||||
|
{new Date(row.original.createdAt).toLocaleDateString()}
|
||||||
|
</span>
|
||||||
|
),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "actions",
|
||||||
|
cell: ({ row }) => (
|
||||||
|
<DropdownMenu>
|
||||||
|
<DropdownMenuTrigger asChild>
|
||||||
|
<Button variant="ghost" size="icon" className="h-8 w-8">
|
||||||
|
<MoreHorizontal className="h-4 w-4" />
|
||||||
|
</Button>
|
||||||
|
</DropdownMenuTrigger>
|
||||||
|
<DropdownMenuContent align="end">
|
||||||
|
<DropdownMenuItem onClick={() => onEdit(row.original)}>
|
||||||
|
<Pencil className="mr-2 h-3.5 w-3.5" />
|
||||||
|
Edit
|
||||||
|
</DropdownMenuItem>
|
||||||
|
<DropdownMenuItem
|
||||||
|
onClick={() => onToggleActive(row.original.id)}
|
||||||
|
>
|
||||||
|
<Power className="mr-2 h-3.5 w-3.5" />
|
||||||
|
{row.original.isActive ? "Disable" : "Enable"}
|
||||||
|
</DropdownMenuItem>
|
||||||
|
<DropdownMenuSeparator />
|
||||||
|
<DropdownMenuItem
|
||||||
|
onClick={() => onDelete(row.original.id)}
|
||||||
|
className="text-destructive focus:text-destructive"
|
||||||
|
>
|
||||||
|
<Trash2 className="mr-2 h-3.5 w-3.5" />
|
||||||
|
Delete
|
||||||
|
</DropdownMenuItem>
|
||||||
|
</DropdownMenuContent>
|
||||||
|
</DropdownMenu>
|
||||||
|
),
|
||||||
|
enableHiding: false,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
}
|
||||||
142
src/app/(app)/telegram/_components/channel-form.tsx
Normal file
142
src/app/(app)/telegram/_components/channel-form.tsx
Normal file
@@ -0,0 +1,142 @@
|
|||||||
|
"use client";
|
||||||
|
|
||||||
|
import { useTransition } from "react";
|
||||||
|
import { useForm } from "react-hook-form";
|
||||||
|
import { zodResolver } from "@hookform/resolvers/zod";
|
||||||
|
import { toast } from "sonner";
|
||||||
|
import {
|
||||||
|
telegramChannelSchema,
|
||||||
|
type TelegramChannelInput,
|
||||||
|
} from "@/schemas/telegram";
|
||||||
|
import { createChannel, updateChannel } from "../actions";
|
||||||
|
import { Button } from "@/components/ui/button";
|
||||||
|
import { Input } from "@/components/ui/input";
|
||||||
|
import {
|
||||||
|
Select,
|
||||||
|
SelectContent,
|
||||||
|
SelectItem,
|
||||||
|
SelectTrigger,
|
||||||
|
SelectValue,
|
||||||
|
} from "@/components/ui/select";
|
||||||
|
import {
|
||||||
|
Form,
|
||||||
|
FormControl,
|
||||||
|
FormDescription,
|
||||||
|
FormField,
|
||||||
|
FormItem,
|
||||||
|
FormLabel,
|
||||||
|
FormMessage,
|
||||||
|
} from "@/components/ui/form";
|
||||||
|
import type { ChannelRow } from "@/lib/telegram/admin-queries";
|
||||||
|
|
||||||
|
interface ChannelFormProps {
|
||||||
|
channel?: ChannelRow;
|
||||||
|
onSuccess: () => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function ChannelForm({ channel, onSuccess }: ChannelFormProps) {
|
||||||
|
const [isPending, startTransition] = useTransition();
|
||||||
|
const isEditing = !!channel;
|
||||||
|
|
||||||
|
const form = useForm<TelegramChannelInput>({
|
||||||
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||||
|
resolver: zodResolver(telegramChannelSchema) as any,
|
||||||
|
defaultValues: {
|
||||||
|
telegramId: channel ? Number(channel.telegramId) : (0 as unknown as number),
|
||||||
|
title: channel?.title ?? "",
|
||||||
|
type: channel?.type ?? "SOURCE",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
function onSubmit(values: TelegramChannelInput) {
|
||||||
|
startTransition(async () => {
|
||||||
|
const result = isEditing
|
||||||
|
? await updateChannel(channel!.id, values)
|
||||||
|
: await createChannel(values);
|
||||||
|
|
||||||
|
if (!result.success) {
|
||||||
|
toast.error(result.error);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
toast.success(isEditing ? "Channel updated" : "Channel created");
|
||||||
|
form.reset();
|
||||||
|
onSuccess();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<Form {...form}>
|
||||||
|
<form onSubmit={form.handleSubmit(onSubmit)} className="space-y-4">
|
||||||
|
<FormField
|
||||||
|
control={form.control}
|
||||||
|
name="title"
|
||||||
|
render={({ field }) => (
|
||||||
|
<FormItem>
|
||||||
|
<FormLabel>Title</FormLabel>
|
||||||
|
<FormControl>
|
||||||
|
<Input placeholder="Channel name" {...field} />
|
||||||
|
</FormControl>
|
||||||
|
<FormMessage />
|
||||||
|
</FormItem>
|
||||||
|
)}
|
||||||
|
/>
|
||||||
|
|
||||||
|
<FormField
|
||||||
|
control={form.control}
|
||||||
|
name="telegramId"
|
||||||
|
render={({ field }) => (
|
||||||
|
<FormItem>
|
||||||
|
<FormLabel>Telegram ID</FormLabel>
|
||||||
|
<FormControl>
|
||||||
|
<Input
|
||||||
|
type="number"
|
||||||
|
placeholder="1234567890"
|
||||||
|
{...field}
|
||||||
|
value={field.value || ""}
|
||||||
|
/>
|
||||||
|
</FormControl>
|
||||||
|
<FormDescription>
|
||||||
|
Numeric ID of the Telegram channel or group
|
||||||
|
</FormDescription>
|
||||||
|
<FormMessage />
|
||||||
|
</FormItem>
|
||||||
|
)}
|
||||||
|
/>
|
||||||
|
|
||||||
|
<FormField
|
||||||
|
control={form.control}
|
||||||
|
name="type"
|
||||||
|
render={({ field }) => (
|
||||||
|
<FormItem>
|
||||||
|
<FormLabel>Type</FormLabel>
|
||||||
|
<Select
|
||||||
|
onValueChange={field.onChange}
|
||||||
|
defaultValue={field.value}
|
||||||
|
>
|
||||||
|
<FormControl>
|
||||||
|
<SelectTrigger>
|
||||||
|
<SelectValue placeholder="Select type" />
|
||||||
|
</SelectTrigger>
|
||||||
|
</FormControl>
|
||||||
|
<SelectContent>
|
||||||
|
<SelectItem value="SOURCE">Source (read archives)</SelectItem>
|
||||||
|
<SelectItem value="DESTINATION">
|
||||||
|
Destination (forward indexed)
|
||||||
|
</SelectItem>
|
||||||
|
</SelectContent>
|
||||||
|
</Select>
|
||||||
|
<FormMessage />
|
||||||
|
</FormItem>
|
||||||
|
)}
|
||||||
|
/>
|
||||||
|
|
||||||
|
<div className="flex justify-end gap-2">
|
||||||
|
<Button type="submit" disabled={isPending}>
|
||||||
|
{isPending ? "Saving..." : isEditing ? "Update" : "Create"}
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
</form>
|
||||||
|
</Form>
|
||||||
|
);
|
||||||
|
}
|
||||||
44
src/app/(app)/telegram/_components/channel-modal.tsx
Normal file
44
src/app/(app)/telegram/_components/channel-modal.tsx
Normal file
@@ -0,0 +1,44 @@
|
|||||||
|
"use client";
|
||||||
|
|
||||||
|
import {
|
||||||
|
Dialog,
|
||||||
|
DialogContent,
|
||||||
|
DialogDescription,
|
||||||
|
DialogHeader,
|
||||||
|
DialogTitle,
|
||||||
|
} from "@/components/ui/dialog";
|
||||||
|
import { ChannelForm } from "./channel-form";
|
||||||
|
import type { ChannelRow } from "@/lib/telegram/admin-queries";
|
||||||
|
|
||||||
|
interface ChannelModalProps {
|
||||||
|
open: boolean;
|
||||||
|
onOpenChange: (open: boolean) => void;
|
||||||
|
channel?: ChannelRow;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function ChannelModal({
|
||||||
|
open,
|
||||||
|
onOpenChange,
|
||||||
|
channel,
|
||||||
|
}: ChannelModalProps) {
|
||||||
|
return (
|
||||||
|
<Dialog open={open} onOpenChange={onOpenChange}>
|
||||||
|
<DialogContent className="sm:max-w-md">
|
||||||
|
<DialogHeader>
|
||||||
|
<DialogTitle>
|
||||||
|
{channel ? "Edit Channel" : "Add Channel"}
|
||||||
|
</DialogTitle>
|
||||||
|
<DialogDescription>
|
||||||
|
{channel
|
||||||
|
? "Update the channel details below."
|
||||||
|
: "Add a Telegram channel. Source channels are scanned for archives, destination channels receive indexed files."}
|
||||||
|
</DialogDescription>
|
||||||
|
</DialogHeader>
|
||||||
|
<ChannelForm
|
||||||
|
channel={channel}
|
||||||
|
onSuccess={() => onOpenChange(false)}
|
||||||
|
/>
|
||||||
|
</DialogContent>
|
||||||
|
</Dialog>
|
||||||
|
);
|
||||||
|
}
|
||||||
97
src/app/(app)/telegram/_components/channels-tab.tsx
Normal file
97
src/app/(app)/telegram/_components/channels-tab.tsx
Normal file
@@ -0,0 +1,97 @@
|
|||||||
|
"use client";
|
||||||
|
|
||||||
|
import { useState, useTransition } from "react";
|
||||||
|
import { Plus } from "lucide-react";
|
||||||
|
import { toast } from "sonner";
|
||||||
|
import { getChannelColumns } from "./channel-columns";
|
||||||
|
import { ChannelModal } from "./channel-modal";
|
||||||
|
import { deleteChannel, toggleChannelActive } from "../actions";
|
||||||
|
import { DataTable } from "@/components/shared/data-table";
|
||||||
|
import { DeleteDialog } from "@/components/shared/delete-dialog";
|
||||||
|
import { Button } from "@/components/ui/button";
|
||||||
|
import type { ChannelRow } from "@/lib/telegram/admin-queries";
|
||||||
|
import { useDataTable } from "@/hooks/use-data-table";
|
||||||
|
|
||||||
|
interface ChannelsTabProps {
|
||||||
|
channels: ChannelRow[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export function ChannelsTab({ channels }: ChannelsTabProps) {
|
||||||
|
const [isPending, startTransition] = useTransition();
|
||||||
|
const [modalOpen, setModalOpen] = useState(false);
|
||||||
|
const [editChannel, setEditChannel] = useState<ChannelRow | undefined>();
|
||||||
|
const [deleteId, setDeleteId] = useState<string | null>(null);
|
||||||
|
|
||||||
|
const columns = getChannelColumns({
|
||||||
|
onEdit: (channel) => {
|
||||||
|
setEditChannel(channel);
|
||||||
|
setModalOpen(true);
|
||||||
|
},
|
||||||
|
onToggleActive: (id) => {
|
||||||
|
startTransition(async () => {
|
||||||
|
const result = await toggleChannelActive(id);
|
||||||
|
if (result.success) toast.success("Channel toggled");
|
||||||
|
else toast.error(result.error);
|
||||||
|
});
|
||||||
|
},
|
||||||
|
onDelete: (id) => setDeleteId(id),
|
||||||
|
});
|
||||||
|
|
||||||
|
const { table } = useDataTable({
|
||||||
|
data: channels,
|
||||||
|
columns,
|
||||||
|
pageCount: 1,
|
||||||
|
});
|
||||||
|
|
||||||
|
const handleDelete = () => {
|
||||||
|
if (!deleteId) return;
|
||||||
|
startTransition(async () => {
|
||||||
|
const result = await deleteChannel(deleteId);
|
||||||
|
if (result.success) {
|
||||||
|
toast.success("Channel deleted");
|
||||||
|
setDeleteId(null);
|
||||||
|
} else {
|
||||||
|
toast.error(result.error);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="space-y-4">
|
||||||
|
<div className="flex items-center gap-2">
|
||||||
|
<Button
|
||||||
|
onClick={() => {
|
||||||
|
setEditChannel(undefined);
|
||||||
|
setModalOpen(true);
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<Plus className="mr-2 h-4 w-4" />
|
||||||
|
Add Channel
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<DataTable
|
||||||
|
table={table}
|
||||||
|
emptyMessage="No channels configured. Add a Telegram channel to start ingesting."
|
||||||
|
/>
|
||||||
|
|
||||||
|
<ChannelModal
|
||||||
|
open={modalOpen}
|
||||||
|
onOpenChange={(open) => {
|
||||||
|
setModalOpen(open);
|
||||||
|
if (!open) setEditChannel(undefined);
|
||||||
|
}}
|
||||||
|
channel={editChannel}
|
||||||
|
/>
|
||||||
|
|
||||||
|
<DeleteDialog
|
||||||
|
open={!!deleteId}
|
||||||
|
onOpenChange={(open) => !open && setDeleteId(null)}
|
||||||
|
title="Delete Channel"
|
||||||
|
description="This will permanently delete this channel and unlink it from all accounts. Existing packages will NOT be deleted."
|
||||||
|
onConfirm={handleDelete}
|
||||||
|
isLoading={isPending}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
41
src/app/(app)/telegram/_components/telegram-admin.tsx
Normal file
41
src/app/(app)/telegram/_components/telegram-admin.tsx
Normal file
@@ -0,0 +1,41 @@
|
|||||||
|
"use client";
|
||||||
|
|
||||||
|
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
|
||||||
|
import { PageHeader } from "@/components/shared/page-header";
|
||||||
|
import { AccountsTab } from "./accounts-tab";
|
||||||
|
import { ChannelsTab } from "./channels-tab";
|
||||||
|
import type { AccountRow, ChannelRow } from "@/lib/telegram/admin-queries";
|
||||||
|
|
||||||
|
interface TelegramAdminProps {
|
||||||
|
accounts: AccountRow[];
|
||||||
|
channels: ChannelRow[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export function TelegramAdmin({ accounts, channels }: TelegramAdminProps) {
|
||||||
|
return (
|
||||||
|
<div className="space-y-4">
|
||||||
|
<PageHeader
|
||||||
|
title="Telegram"
|
||||||
|
description="Manage Telegram accounts, channels, and ingestion"
|
||||||
|
/>
|
||||||
|
|
||||||
|
<Tabs defaultValue="accounts" className="space-y-4">
|
||||||
|
<TabsList>
|
||||||
|
<TabsTrigger value="accounts">
|
||||||
|
Accounts ({accounts.length})
|
||||||
|
</TabsTrigger>
|
||||||
|
<TabsTrigger value="channels">
|
||||||
|
Channels ({channels.length})
|
||||||
|
</TabsTrigger>
|
||||||
|
</TabsList>
|
||||||
|
|
||||||
|
<TabsContent value="accounts">
|
||||||
|
<AccountsTab accounts={accounts} />
|
||||||
|
</TabsContent>
|
||||||
|
<TabsContent value="channels">
|
||||||
|
<ChannelsTab channels={channels} />
|
||||||
|
</TabsContent>
|
||||||
|
</Tabs>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
345
src/app/(app)/telegram/actions.ts
Normal file
345
src/app/(app)/telegram/actions.ts
Normal file
@@ -0,0 +1,345 @@
|
|||||||
|
"use server";
|
||||||
|
|
||||||
|
import { auth } from "@/lib/auth";
|
||||||
|
import { prisma } from "@/lib/prisma";
|
||||||
|
import { revalidatePath } from "next/cache";
|
||||||
|
import type { ActionResult } from "@/types/api.types";
|
||||||
|
import {
|
||||||
|
telegramAccountSchema,
|
||||||
|
telegramChannelSchema,
|
||||||
|
linkChannelSchema,
|
||||||
|
submitAuthCodeSchema,
|
||||||
|
} from "@/schemas/telegram";
|
||||||
|
|
||||||
|
const REVALIDATE_PATH = "/telegram";
|
||||||
|
|
||||||
|
async function requireAdmin(): Promise<
|
||||||
|
{ success: true; userId: string } | { success: false; error: string }
|
||||||
|
> {
|
||||||
|
const session = await auth();
|
||||||
|
if (!session?.user?.id) return { success: false, error: "Unauthorized" };
|
||||||
|
if (session.user.role !== "ADMIN")
|
||||||
|
return { success: false, error: "Admin access required" };
|
||||||
|
return { success: true, userId: session.user.id };
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Account actions ──
|
||||||
|
|
||||||
|
export async function createAccount(
|
||||||
|
input: unknown
|
||||||
|
): Promise<ActionResult<{ id: string }>> {
|
||||||
|
const admin = await requireAdmin();
|
||||||
|
if (!admin.success) return admin;
|
||||||
|
|
||||||
|
const parsed = telegramAccountSchema.safeParse(input);
|
||||||
|
if (!parsed.success) return { success: false, error: "Validation failed" };
|
||||||
|
|
||||||
|
try {
|
||||||
|
const account = await prisma.telegramAccount.create({
|
||||||
|
data: {
|
||||||
|
phone: parsed.data.phone.replace(/[\s\-]/g, ""),
|
||||||
|
displayName: parsed.data.displayName || null,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
revalidatePath(REVALIDATE_PATH);
|
||||||
|
return { success: true, data: { id: account.id } };
|
||||||
|
} catch (err: unknown) {
|
||||||
|
if (
|
||||||
|
err instanceof Error &&
|
||||||
|
err.message.includes("Unique constraint failed")
|
||||||
|
) {
|
||||||
|
return { success: false, error: "Phone number already registered" };
|
||||||
|
}
|
||||||
|
return { success: false, error: "Failed to create account" };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function updateAccount(
|
||||||
|
id: string,
|
||||||
|
input: unknown
|
||||||
|
): Promise<ActionResult> {
|
||||||
|
const admin = await requireAdmin();
|
||||||
|
if (!admin.success) return admin;
|
||||||
|
|
||||||
|
const parsed = telegramAccountSchema.safeParse(input);
|
||||||
|
if (!parsed.success) return { success: false, error: "Validation failed" };
|
||||||
|
|
||||||
|
const existing = await prisma.telegramAccount.findUnique({ where: { id } });
|
||||||
|
if (!existing) return { success: false, error: "Account not found" };
|
||||||
|
|
||||||
|
try {
|
||||||
|
await prisma.telegramAccount.update({
|
||||||
|
where: { id },
|
||||||
|
data: {
|
||||||
|
phone: parsed.data.phone.replace(/[\s\-]/g, ""),
|
||||||
|
displayName: parsed.data.displayName || null,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
revalidatePath(REVALIDATE_PATH);
|
||||||
|
return { success: true, data: undefined };
|
||||||
|
} catch (err: unknown) {
|
||||||
|
if (
|
||||||
|
err instanceof Error &&
|
||||||
|
err.message.includes("Unique constraint failed")
|
||||||
|
) {
|
||||||
|
return { success: false, error: "Phone number already registered" };
|
||||||
|
}
|
||||||
|
return { success: false, error: "Failed to update account" };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function toggleAccountActive(id: string): Promise<ActionResult> {
|
||||||
|
const admin = await requireAdmin();
|
||||||
|
if (!admin.success) return admin;
|
||||||
|
|
||||||
|
const existing = await prisma.telegramAccount.findUnique({ where: { id } });
|
||||||
|
if (!existing) return { success: false, error: "Account not found" };
|
||||||
|
|
||||||
|
try {
|
||||||
|
await prisma.telegramAccount.update({
|
||||||
|
where: { id },
|
||||||
|
data: { isActive: !existing.isActive },
|
||||||
|
});
|
||||||
|
revalidatePath(REVALIDATE_PATH);
|
||||||
|
return { success: true, data: undefined };
|
||||||
|
} catch {
|
||||||
|
return { success: false, error: "Failed to toggle account" };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function deleteAccount(id: string): Promise<ActionResult> {
|
||||||
|
const admin = await requireAdmin();
|
||||||
|
if (!admin.success) return admin;
|
||||||
|
|
||||||
|
const existing = await prisma.telegramAccount.findUnique({ where: { id } });
|
||||||
|
if (!existing) return { success: false, error: "Account not found" };
|
||||||
|
|
||||||
|
try {
|
||||||
|
await prisma.telegramAccount.delete({ where: { id } });
|
||||||
|
revalidatePath(REVALIDATE_PATH);
|
||||||
|
return { success: true, data: undefined };
|
||||||
|
} catch {
|
||||||
|
return { success: false, error: "Failed to delete account" };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function submitAuthCode(
|
||||||
|
accountId: string,
|
||||||
|
input: unknown
|
||||||
|
): Promise<ActionResult> {
|
||||||
|
const admin = await requireAdmin();
|
||||||
|
if (!admin.success) return admin;
|
||||||
|
|
||||||
|
const parsed = submitAuthCodeSchema.safeParse(input);
|
||||||
|
if (!parsed.success) return { success: false, error: "Validation failed" };
|
||||||
|
|
||||||
|
const existing = await prisma.telegramAccount.findUnique({
|
||||||
|
where: { id: accountId },
|
||||||
|
});
|
||||||
|
if (!existing) return { success: false, error: "Account not found" };
|
||||||
|
if (
|
||||||
|
existing.authState !== "AWAITING_CODE" &&
|
||||||
|
existing.authState !== "AWAITING_PASSWORD"
|
||||||
|
) {
|
||||||
|
return { success: false, error: "Account is not waiting for a code" };
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
await prisma.telegramAccount.update({
|
||||||
|
where: { id: accountId },
|
||||||
|
data: { authCode: parsed.data.code },
|
||||||
|
});
|
||||||
|
revalidatePath(REVALIDATE_PATH);
|
||||||
|
return { success: true, data: undefined };
|
||||||
|
} catch {
|
||||||
|
return { success: false, error: "Failed to submit code" };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Channel actions ──
|
||||||
|
|
||||||
|
export async function createChannel(
|
||||||
|
input: unknown
|
||||||
|
): Promise<ActionResult<{ id: string }>> {
|
||||||
|
const admin = await requireAdmin();
|
||||||
|
if (!admin.success) return admin;
|
||||||
|
|
||||||
|
const parsed = telegramChannelSchema.safeParse(input);
|
||||||
|
if (!parsed.success) return { success: false, error: "Validation failed" };
|
||||||
|
|
||||||
|
try {
|
||||||
|
const channel = await prisma.telegramChannel.create({
|
||||||
|
data: {
|
||||||
|
telegramId: BigInt(parsed.data.telegramId),
|
||||||
|
title: parsed.data.title,
|
||||||
|
type: parsed.data.type,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
revalidatePath(REVALIDATE_PATH);
|
||||||
|
return { success: true, data: { id: channel.id } };
|
||||||
|
} catch (err: unknown) {
|
||||||
|
if (
|
||||||
|
err instanceof Error &&
|
||||||
|
err.message.includes("Unique constraint failed")
|
||||||
|
) {
|
||||||
|
return { success: false, error: "Channel with this Telegram ID already exists" };
|
||||||
|
}
|
||||||
|
return { success: false, error: "Failed to create channel" };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function updateChannel(
|
||||||
|
id: string,
|
||||||
|
input: unknown
|
||||||
|
): Promise<ActionResult> {
|
||||||
|
const admin = await requireAdmin();
|
||||||
|
if (!admin.success) return admin;
|
||||||
|
|
||||||
|
const parsed = telegramChannelSchema.safeParse(input);
|
||||||
|
if (!parsed.success) return { success: false, error: "Validation failed" };
|
||||||
|
|
||||||
|
const existing = await prisma.telegramChannel.findUnique({ where: { id } });
|
||||||
|
if (!existing) return { success: false, error: "Channel not found" };
|
||||||
|
|
||||||
|
try {
|
||||||
|
await prisma.telegramChannel.update({
|
||||||
|
where: { id },
|
||||||
|
data: {
|
||||||
|
telegramId: BigInt(parsed.data.telegramId),
|
||||||
|
title: parsed.data.title,
|
||||||
|
type: parsed.data.type,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
revalidatePath(REVALIDATE_PATH);
|
||||||
|
return { success: true, data: undefined };
|
||||||
|
} catch (err: unknown) {
|
||||||
|
if (
|
||||||
|
err instanceof Error &&
|
||||||
|
err.message.includes("Unique constraint failed")
|
||||||
|
) {
|
||||||
|
return { success: false, error: "Channel with this Telegram ID already exists" };
|
||||||
|
}
|
||||||
|
return { success: false, error: "Failed to update channel" };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function toggleChannelActive(id: string): Promise<ActionResult> {
|
||||||
|
const admin = await requireAdmin();
|
||||||
|
if (!admin.success) return admin;
|
||||||
|
|
||||||
|
const existing = await prisma.telegramChannel.findUnique({ where: { id } });
|
||||||
|
if (!existing) return { success: false, error: "Channel not found" };
|
||||||
|
|
||||||
|
try {
|
||||||
|
await prisma.telegramChannel.update({
|
||||||
|
where: { id },
|
||||||
|
data: { isActive: !existing.isActive },
|
||||||
|
});
|
||||||
|
revalidatePath(REVALIDATE_PATH);
|
||||||
|
return { success: true, data: undefined };
|
||||||
|
} catch {
|
||||||
|
return { success: false, error: "Failed to toggle channel" };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function deleteChannel(id: string): Promise<ActionResult> {
|
||||||
|
const admin = await requireAdmin();
|
||||||
|
if (!admin.success) return admin;
|
||||||
|
|
||||||
|
const existing = await prisma.telegramChannel.findUnique({ where: { id } });
|
||||||
|
if (!existing) return { success: false, error: "Channel not found" };
|
||||||
|
|
||||||
|
try {
|
||||||
|
await prisma.telegramChannel.delete({ where: { id } });
|
||||||
|
revalidatePath(REVALIDATE_PATH);
|
||||||
|
return { success: true, data: undefined };
|
||||||
|
} catch {
|
||||||
|
return { success: false, error: "Failed to delete channel" };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Account-Channel link actions ──
|
||||||
|
|
||||||
|
export async function linkChannel(
|
||||||
|
input: unknown
|
||||||
|
): Promise<ActionResult<{ id: string }>> {
|
||||||
|
const admin = await requireAdmin();
|
||||||
|
if (!admin.success) return admin;
|
||||||
|
|
||||||
|
const parsed = linkChannelSchema.safeParse(input);
|
||||||
|
if (!parsed.success) return { success: false, error: "Validation failed" };
|
||||||
|
|
||||||
|
try {
|
||||||
|
const link = await prisma.accountChannelMap.create({
|
||||||
|
data: {
|
||||||
|
accountId: parsed.data.accountId,
|
||||||
|
channelId: parsed.data.channelId,
|
||||||
|
role: parsed.data.role,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
revalidatePath(REVALIDATE_PATH);
|
||||||
|
return { success: true, data: { id: link.id } };
|
||||||
|
} catch (err: unknown) {
|
||||||
|
if (
|
||||||
|
err instanceof Error &&
|
||||||
|
err.message.includes("Unique constraint failed")
|
||||||
|
) {
|
||||||
|
return { success: false, error: "This channel is already linked to this account" };
|
||||||
|
}
|
||||||
|
return { success: false, error: "Failed to link channel" };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function unlinkChannel(id: string): Promise<ActionResult> {
|
||||||
|
const admin = await requireAdmin();
|
||||||
|
if (!admin.success) return admin;
|
||||||
|
|
||||||
|
const existing = await prisma.accountChannelMap.findUnique({
|
||||||
|
where: { id },
|
||||||
|
});
|
||||||
|
if (!existing) return { success: false, error: "Link not found" };
|
||||||
|
|
||||||
|
try {
|
||||||
|
await prisma.accountChannelMap.delete({ where: { id } });
|
||||||
|
revalidatePath(REVALIDATE_PATH);
|
||||||
|
return { success: true, data: undefined };
|
||||||
|
} catch {
|
||||||
|
return { success: false, error: "Failed to unlink channel" };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Ingestion trigger ──
|
||||||
|
|
||||||
|
export async function triggerIngestion(
|
||||||
|
accountId?: string
|
||||||
|
): Promise<ActionResult> {
|
||||||
|
const admin = await requireAdmin();
|
||||||
|
if (!admin.success) return admin;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const res = await fetch(
|
||||||
|
`${process.env.NEXT_PUBLIC_APP_URL || "http://localhost:3000"}/api/ingestion/trigger`,
|
||||||
|
{
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
"X-API-Key": process.env.INGESTION_API_KEY || "",
|
||||||
|
},
|
||||||
|
body: JSON.stringify({ accountId }),
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!res.ok) {
|
||||||
|
const data = await res.json().catch(() => ({}));
|
||||||
|
return {
|
||||||
|
success: false,
|
||||||
|
error: (data as { error?: string }).error || "Failed to trigger ingestion",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
revalidatePath(REVALIDATE_PATH);
|
||||||
|
return { success: true, data: undefined };
|
||||||
|
} catch {
|
||||||
|
return { success: false, error: "Failed to trigger ingestion" };
|
||||||
|
}
|
||||||
|
}
|
||||||
17
src/app/(app)/telegram/page.tsx
Normal file
17
src/app/(app)/telegram/page.tsx
Normal file
@@ -0,0 +1,17 @@
|
|||||||
|
import { auth } from "@/lib/auth";
|
||||||
|
import { redirect } from "next/navigation";
|
||||||
|
import { listAccounts, listChannels } from "@/lib/telegram/admin-queries";
|
||||||
|
import { TelegramAdmin } from "./_components/telegram-admin";
|
||||||
|
|
||||||
|
export default async function TelegramPage() {
|
||||||
|
const session = await auth();
|
||||||
|
if (!session?.user?.id) redirect("/login");
|
||||||
|
if (session.user.role !== "ADMIN") redirect("/dashboard");
|
||||||
|
|
||||||
|
const [accounts, channels] = await Promise.all([
|
||||||
|
listAccounts(),
|
||||||
|
listChannels(),
|
||||||
|
]);
|
||||||
|
|
||||||
|
return <TelegramAdmin accounts={accounts} channels={channels} />;
|
||||||
|
}
|
||||||
13
src/app/api/ingestion/status/route.ts
Normal file
13
src/app/api/ingestion/status/route.ts
Normal file
@@ -0,0 +1,13 @@
|
|||||||
|
import { NextResponse } from "next/server";
|
||||||
|
import { authenticateApiRequest } from "@/lib/telegram/api-auth";
|
||||||
|
import { getIngestionStatus } from "@/lib/telegram/queries";
|
||||||
|
|
||||||
|
export const dynamic = "force-dynamic";
|
||||||
|
|
||||||
|
export async function GET(request: Request) {
|
||||||
|
const authResult = await authenticateApiRequest(request);
|
||||||
|
if ("error" in authResult) return authResult.error;
|
||||||
|
|
||||||
|
const accounts = await getIngestionStatus();
|
||||||
|
return NextResponse.json({ accounts });
|
||||||
|
}
|
||||||
77
src/app/api/ingestion/trigger/route.ts
Normal file
77
src/app/api/ingestion/trigger/route.ts
Normal file
@@ -0,0 +1,77 @@
|
|||||||
|
import { NextResponse } from "next/server";
|
||||||
|
import { authenticateApiRequest } from "@/lib/telegram/api-auth";
|
||||||
|
import { triggerIngestionSchema } from "@/schemas/telegram";
|
||||||
|
import { prisma } from "@/lib/prisma";
|
||||||
|
|
||||||
|
export const dynamic = "force-dynamic";
|
||||||
|
|
||||||
|
export async function POST(request: Request) {
|
||||||
|
const authResult = await authenticateApiRequest(request, true);
|
||||||
|
if ("error" in authResult) return authResult.error;
|
||||||
|
|
||||||
|
let body: unknown = {};
|
||||||
|
try {
|
||||||
|
body = await request.json();
|
||||||
|
} catch {
|
||||||
|
// Empty body is fine — triggers all accounts
|
||||||
|
}
|
||||||
|
|
||||||
|
const parsed = triggerIngestionSchema.safeParse(body);
|
||||||
|
if (!parsed.success) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{ error: "Invalid parameters", details: parsed.error.flatten() },
|
||||||
|
{ status: 400 }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Find accounts to trigger
|
||||||
|
const where: { isActive: boolean; authState: "AUTHENTICATED"; id?: string } = {
|
||||||
|
isActive: true,
|
||||||
|
authState: "AUTHENTICATED",
|
||||||
|
};
|
||||||
|
if (parsed.data.accountId) {
|
||||||
|
where.id = parsed.data.accountId;
|
||||||
|
}
|
||||||
|
|
||||||
|
const accounts = await prisma.telegramAccount.findMany({
|
||||||
|
where,
|
||||||
|
select: { id: true },
|
||||||
|
});
|
||||||
|
|
||||||
|
if (accounts.length === 0) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{ triggered: false, message: "No eligible accounts found" },
|
||||||
|
{ status: 404 }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create ingestion runs marked as RUNNING — the worker will pick these up
|
||||||
|
// when it next polls, or we use pg_notify for immediate pickup
|
||||||
|
for (const account of accounts) {
|
||||||
|
// Only create if no run is already RUNNING for this account
|
||||||
|
const existing = await prisma.ingestionRun.findFirst({
|
||||||
|
where: { accountId: account.id, status: "RUNNING" },
|
||||||
|
});
|
||||||
|
if (!existing) {
|
||||||
|
await prisma.ingestionRun.create({
|
||||||
|
data: { accountId: account.id, status: "RUNNING" },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Send pg_notify for immediate worker pickup
|
||||||
|
try {
|
||||||
|
await prisma.$queryRawUnsafe(
|
||||||
|
`SELECT pg_notify('ingestion_trigger', $1)`,
|
||||||
|
accounts.map((a) => a.id).join(",")
|
||||||
|
);
|
||||||
|
} catch {
|
||||||
|
// pg_notify is best-effort — worker will pick up on next cycle anyway
|
||||||
|
}
|
||||||
|
|
||||||
|
return NextResponse.json({
|
||||||
|
triggered: true,
|
||||||
|
accountIds: accounts.map((a) => a.id),
|
||||||
|
message: `Ingestion queued for ${accounts.length} account(s)`,
|
||||||
|
});
|
||||||
|
}
|
||||||
17
src/app/api/telegram/accounts/[accountId]/links/route.ts
Normal file
17
src/app/api/telegram/accounts/[accountId]/links/route.ts
Normal file
@@ -0,0 +1,17 @@
|
|||||||
|
import { NextResponse } from "next/server";
|
||||||
|
import { authenticateApiRequest } from "@/lib/telegram/api-auth";
|
||||||
|
import { listAccountChannelLinks } from "@/lib/telegram/admin-queries";
|
||||||
|
|
||||||
|
export const dynamic = "force-dynamic";
|
||||||
|
|
||||||
|
export async function GET(
|
||||||
|
request: Request,
|
||||||
|
{ params }: { params: Promise<{ accountId: string }> }
|
||||||
|
) {
|
||||||
|
const authResult = await authenticateApiRequest(request, true);
|
||||||
|
if ("error" in authResult) return authResult.error;
|
||||||
|
|
||||||
|
const { accountId } = await params;
|
||||||
|
const links = await listAccountChannelLinks(accountId);
|
||||||
|
return NextResponse.json(links);
|
||||||
|
}
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
import { NextResponse } from "next/server";
|
||||||
|
import { authenticateApiRequest } from "@/lib/telegram/api-auth";
|
||||||
|
import { getUnlinkedChannels } from "@/lib/telegram/admin-queries";
|
||||||
|
|
||||||
|
export const dynamic = "force-dynamic";
|
||||||
|
|
||||||
|
export async function GET(
|
||||||
|
request: Request,
|
||||||
|
{ params }: { params: Promise<{ accountId: string }> }
|
||||||
|
) {
|
||||||
|
const authResult = await authenticateApiRequest(request, true);
|
||||||
|
if ("error" in authResult) return authResult.error;
|
||||||
|
|
||||||
|
const { accountId } = await params;
|
||||||
|
const channels = await getUnlinkedChannels(accountId);
|
||||||
|
return NextResponse.json(channels);
|
||||||
|
}
|
||||||
32
src/app/api/zips/[id]/files/route.ts
Normal file
32
src/app/api/zips/[id]/files/route.ts
Normal file
@@ -0,0 +1,32 @@
|
|||||||
|
import { NextResponse } from "next/server";
|
||||||
|
import { authenticateApiRequest } from "@/lib/telegram/api-auth";
|
||||||
|
import { listPackageFiles } from "@/lib/telegram/queries";
|
||||||
|
import { listFilesSchema } from "@/schemas/telegram";
|
||||||
|
|
||||||
|
export const dynamic = "force-dynamic";
|
||||||
|
|
||||||
|
export async function GET(
|
||||||
|
request: Request,
|
||||||
|
{ params }: { params: Promise<{ id: string }> }
|
||||||
|
) {
|
||||||
|
const authResult = await authenticateApiRequest(request);
|
||||||
|
if ("error" in authResult) return authResult.error;
|
||||||
|
|
||||||
|
const { id } = await params;
|
||||||
|
const { searchParams } = new URL(request.url);
|
||||||
|
const parsed = listFilesSchema.safeParse(Object.fromEntries(searchParams));
|
||||||
|
|
||||||
|
if (!parsed.success) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{ error: "Invalid parameters", details: parsed.error.flatten() },
|
||||||
|
{ status: 400 }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = await listPackageFiles({
|
||||||
|
packageId: id,
|
||||||
|
...parsed.data,
|
||||||
|
});
|
||||||
|
|
||||||
|
return NextResponse.json(result);
|
||||||
|
}
|
||||||
42
src/app/api/zips/[id]/preview/route.ts
Normal file
42
src/app/api/zips/[id]/preview/route.ts
Normal file
@@ -0,0 +1,42 @@
|
|||||||
|
import { NextResponse } from "next/server";
|
||||||
|
import { prisma } from "@/lib/prisma";
|
||||||
|
import { authenticateApiRequest } from "@/lib/telegram/api-auth";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* GET /api/zips/:id/preview
|
||||||
|
* Returns the preview thumbnail image as JPEG binary.
|
||||||
|
* Cached for 1 hour (immutable once set).
|
||||||
|
*/
|
||||||
|
export async function GET(
|
||||||
|
request: Request,
|
||||||
|
{ params }: { params: Promise<{ id: string }> }
|
||||||
|
) {
|
||||||
|
const authResult = await authenticateApiRequest(request);
|
||||||
|
if ("error" in authResult) return authResult.error;
|
||||||
|
|
||||||
|
const { id } = await params;
|
||||||
|
|
||||||
|
const pkg = await prisma.package.findUnique({
|
||||||
|
where: { id },
|
||||||
|
select: { previewData: true },
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!pkg || !pkg.previewData) {
|
||||||
|
return new NextResponse(null, { status: 404 });
|
||||||
|
}
|
||||||
|
|
||||||
|
// previewData is stored as Bytes (Buffer) from Prisma
|
||||||
|
const buffer =
|
||||||
|
pkg.previewData instanceof Buffer
|
||||||
|
? pkg.previewData
|
||||||
|
: Buffer.from(pkg.previewData);
|
||||||
|
|
||||||
|
return new NextResponse(buffer, {
|
||||||
|
status: 200,
|
||||||
|
headers: {
|
||||||
|
"Content-Type": "image/jpeg",
|
||||||
|
"Content-Length": String(buffer.length),
|
||||||
|
"Cache-Control": "public, max-age=3600, immutable",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
22
src/app/api/zips/[id]/route.ts
Normal file
22
src/app/api/zips/[id]/route.ts
Normal file
@@ -0,0 +1,22 @@
|
|||||||
|
import { NextResponse } from "next/server";
|
||||||
|
import { authenticateApiRequest } from "@/lib/telegram/api-auth";
|
||||||
|
import { getPackageById } from "@/lib/telegram/queries";
|
||||||
|
|
||||||
|
export const dynamic = "force-dynamic";
|
||||||
|
|
||||||
|
export async function GET(
|
||||||
|
request: Request,
|
||||||
|
{ params }: { params: Promise<{ id: string }> }
|
||||||
|
) {
|
||||||
|
const authResult = await authenticateApiRequest(request);
|
||||||
|
if ("error" in authResult) return authResult.error;
|
||||||
|
|
||||||
|
const { id } = await params;
|
||||||
|
const pkg = await getPackageById(id);
|
||||||
|
|
||||||
|
if (!pkg) {
|
||||||
|
return NextResponse.json({ error: "Package not found" }, { status: 404 });
|
||||||
|
}
|
||||||
|
|
||||||
|
return NextResponse.json(pkg);
|
||||||
|
}
|
||||||
24
src/app/api/zips/route.ts
Normal file
24
src/app/api/zips/route.ts
Normal file
@@ -0,0 +1,24 @@
|
|||||||
|
import { NextResponse } from "next/server";
|
||||||
|
import { authenticateApiRequest } from "@/lib/telegram/api-auth";
|
||||||
|
import { listPackages } from "@/lib/telegram/queries";
|
||||||
|
import { listPackagesSchema } from "@/schemas/telegram";
|
||||||
|
|
||||||
|
export const dynamic = "force-dynamic";
|
||||||
|
|
||||||
|
export async function GET(request: Request) {
|
||||||
|
const authResult = await authenticateApiRequest(request);
|
||||||
|
if ("error" in authResult) return authResult.error;
|
||||||
|
|
||||||
|
const { searchParams } = new URL(request.url);
|
||||||
|
const parsed = listPackagesSchema.safeParse(Object.fromEntries(searchParams));
|
||||||
|
|
||||||
|
if (!parsed.success) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{ error: "Invalid parameters", details: parsed.error.flatten() },
|
||||||
|
{ status: 400 }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = await listPackages(parsed.data);
|
||||||
|
return NextResponse.json(result);
|
||||||
|
}
|
||||||
25
src/app/api/zips/search/route.ts
Normal file
25
src/app/api/zips/search/route.ts
Normal file
@@ -0,0 +1,25 @@
|
|||||||
|
import { NextResponse } from "next/server";
|
||||||
|
import { authenticateApiRequest } from "@/lib/telegram/api-auth";
|
||||||
|
import { searchPackages } from "@/lib/telegram/queries";
|
||||||
|
import { searchSchema } from "@/schemas/telegram";
|
||||||
|
|
||||||
|
export const dynamic = "force-dynamic";
|
||||||
|
|
||||||
|
export async function GET(request: Request) {
|
||||||
|
const authResult = await authenticateApiRequest(request);
|
||||||
|
if ("error" in authResult) return authResult.error;
|
||||||
|
|
||||||
|
const { searchParams } = new URL(request.url);
|
||||||
|
const parsed = searchSchema.safeParse(Object.fromEntries(searchParams));
|
||||||
|
|
||||||
|
if (!parsed.success) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{ error: "Invalid parameters", details: parsed.error.flatten() },
|
||||||
|
{ status: 400 }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const { q, ...rest } = parsed.data;
|
||||||
|
const result = await searchPackages({ query: q, ...rest });
|
||||||
|
return NextResponse.json(result);
|
||||||
|
}
|
||||||
@@ -8,6 +8,8 @@ import {
|
|||||||
Droplets,
|
Droplets,
|
||||||
Paintbrush,
|
Paintbrush,
|
||||||
Gem,
|
Gem,
|
||||||
|
FileBox,
|
||||||
|
Send,
|
||||||
ClipboardList,
|
ClipboardList,
|
||||||
Building2,
|
Building2,
|
||||||
MapPin,
|
MapPin,
|
||||||
@@ -18,7 +20,7 @@ import { cn } from "@/lib/utils";
|
|||||||
import { APP_NAME } from "@/lib/constants";
|
import { APP_NAME } from "@/lib/constants";
|
||||||
import { SheetHeader, SheetTitle } from "@/components/ui/sheet";
|
import { SheetHeader, SheetTitle } from "@/components/ui/sheet";
|
||||||
|
|
||||||
const icons = { LayoutDashboard, Cylinder, Droplets, Paintbrush, Gem, ClipboardList, Building2, MapPin, Settings };
|
const icons = { LayoutDashboard, Cylinder, Droplets, Paintbrush, Gem, FileBox, Send, ClipboardList, Building2, MapPin, Settings };
|
||||||
|
|
||||||
const navItems = [
|
const navItems = [
|
||||||
{ label: "Dashboard", href: "/dashboard", icon: "LayoutDashboard" as const },
|
{ label: "Dashboard", href: "/dashboard", icon: "LayoutDashboard" as const },
|
||||||
@@ -26,6 +28,8 @@ const navItems = [
|
|||||||
{ label: "Resins", href: "/resins", icon: "Droplets" as const },
|
{ label: "Resins", href: "/resins", icon: "Droplets" as const },
|
||||||
{ label: "Paints", href: "/paints", icon: "Paintbrush" as const },
|
{ label: "Paints", href: "/paints", icon: "Paintbrush" as const },
|
||||||
{ label: "Supplies", href: "/supplies", icon: "Gem" as const },
|
{ label: "Supplies", href: "/supplies", icon: "Gem" as const },
|
||||||
|
{ label: "STL Files", href: "/stls", icon: "FileBox" as const },
|
||||||
|
{ label: "Telegram", href: "/telegram", icon: "Send" as const },
|
||||||
{ label: "Usage", href: "/usage", icon: "ClipboardList" as const },
|
{ label: "Usage", href: "/usage", icon: "ClipboardList" as const },
|
||||||
{ label: "Vendors", href: "/vendors", icon: "Building2" as const },
|
{ label: "Vendors", href: "/vendors", icon: "Building2" as const },
|
||||||
{ label: "Locations", href: "/locations", icon: "MapPin" as const },
|
{ label: "Locations", href: "/locations", icon: "MapPin" as const },
|
||||||
|
|||||||
@@ -9,6 +9,8 @@ import {
|
|||||||
Droplets,
|
Droplets,
|
||||||
Paintbrush,
|
Paintbrush,
|
||||||
Gem,
|
Gem,
|
||||||
|
FileBox,
|
||||||
|
Send,
|
||||||
ClipboardList,
|
ClipboardList,
|
||||||
Building2,
|
Building2,
|
||||||
MapPin,
|
MapPin,
|
||||||
@@ -28,6 +30,8 @@ const icons = {
|
|||||||
Droplets,
|
Droplets,
|
||||||
Paintbrush,
|
Paintbrush,
|
||||||
Gem,
|
Gem,
|
||||||
|
FileBox,
|
||||||
|
Send,
|
||||||
ClipboardList,
|
ClipboardList,
|
||||||
Building2,
|
Building2,
|
||||||
MapPin,
|
MapPin,
|
||||||
@@ -40,6 +44,8 @@ const navItems = [
|
|||||||
{ label: "Resins", href: "/resins", icon: "Droplets" as const },
|
{ label: "Resins", href: "/resins", icon: "Droplets" as const },
|
||||||
{ label: "Paints", href: "/paints", icon: "Paintbrush" as const },
|
{ label: "Paints", href: "/paints", icon: "Paintbrush" as const },
|
||||||
{ label: "Supplies", href: "/supplies", icon: "Gem" as const },
|
{ label: "Supplies", href: "/supplies", icon: "Gem" as const },
|
||||||
|
{ label: "STL Files", href: "/stls", icon: "FileBox" as const },
|
||||||
|
{ label: "Telegram", href: "/telegram", icon: "Send" as const },
|
||||||
{ label: "Usage", href: "/usage", icon: "ClipboardList" as const },
|
{ label: "Usage", href: "/usage", icon: "ClipboardList" as const },
|
||||||
{ label: "Vendors", href: "/vendors", icon: "Building2" as const },
|
{ label: "Vendors", href: "/vendors", icon: "Building2" as const },
|
||||||
{ label: "Locations", href: "/locations", icon: "MapPin" as const },
|
{ label: "Locations", href: "/locations", icon: "MapPin" as const },
|
||||||
|
|||||||
@@ -6,6 +6,8 @@ export const NAV_ITEMS = [
|
|||||||
{ label: "Resins", href: "/resins", icon: "Droplets" },
|
{ label: "Resins", href: "/resins", icon: "Droplets" },
|
||||||
{ label: "Paints", href: "/paints", icon: "Paintbrush" },
|
{ label: "Paints", href: "/paints", icon: "Paintbrush" },
|
||||||
{ label: "Supplies", href: "/supplies", icon: "Gem" },
|
{ label: "Supplies", href: "/supplies", icon: "Gem" },
|
||||||
|
{ label: "STL Files", href: "/stls", icon: "FileBox" },
|
||||||
|
{ label: "Telegram", href: "/telegram", icon: "Send" },
|
||||||
{ label: "Usage", href: "/usage", icon: "ClipboardList" },
|
{ label: "Usage", href: "/usage", icon: "ClipboardList" },
|
||||||
{ label: "Vendors", href: "/vendors", icon: "Building2" },
|
{ label: "Vendors", href: "/vendors", icon: "Building2" },
|
||||||
{ label: "Locations", href: "/locations", icon: "MapPin" },
|
{ label: "Locations", href: "/locations", icon: "MapPin" },
|
||||||
|
|||||||
105
src/lib/telegram/admin-queries.ts
Normal file
105
src/lib/telegram/admin-queries.ts
Normal file
@@ -0,0 +1,105 @@
|
|||||||
|
import { prisma } from "@/lib/prisma";
|
||||||
|
|
||||||
|
// ── Account queries ──
|
||||||
|
|
||||||
|
export async function listAccounts() {
|
||||||
|
const accounts = await prisma.telegramAccount.findMany({
|
||||||
|
orderBy: { createdAt: "desc" },
|
||||||
|
include: {
|
||||||
|
_count: { select: { channelMaps: true, ingestionRuns: true } },
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
return accounts.map((a) => ({
|
||||||
|
id: a.id,
|
||||||
|
phone: a.phone,
|
||||||
|
displayName: a.displayName,
|
||||||
|
isActive: a.isActive,
|
||||||
|
authState: a.authState,
|
||||||
|
authCode: a.authCode,
|
||||||
|
lastSeenAt: a.lastSeenAt?.toISOString() ?? null,
|
||||||
|
createdAt: a.createdAt.toISOString(),
|
||||||
|
channelCount: a._count.channelMaps,
|
||||||
|
runCount: a._count.ingestionRuns,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
export type AccountRow = Awaited<ReturnType<typeof listAccounts>>[number];
|
||||||
|
|
||||||
|
// ── Channel queries ──
|
||||||
|
|
||||||
|
export async function listChannels() {
|
||||||
|
const channels = await prisma.telegramChannel.findMany({
|
||||||
|
orderBy: { createdAt: "desc" },
|
||||||
|
include: {
|
||||||
|
_count: { select: { accountMaps: true, packages: true } },
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
return channels.map((c) => ({
|
||||||
|
id: c.id,
|
||||||
|
telegramId: c.telegramId.toString(),
|
||||||
|
title: c.title,
|
||||||
|
type: c.type,
|
||||||
|
isActive: c.isActive,
|
||||||
|
createdAt: c.createdAt.toISOString(),
|
||||||
|
accountCount: c._count.accountMaps,
|
||||||
|
packageCount: c._count.packages,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
export type ChannelRow = Awaited<ReturnType<typeof listChannels>>[number];
|
||||||
|
|
||||||
|
// ── Account-Channel link queries ──
|
||||||
|
|
||||||
|
export async function listAccountChannelLinks(accountId: string) {
|
||||||
|
const links = await prisma.accountChannelMap.findMany({
|
||||||
|
where: { accountId },
|
||||||
|
include: {
|
||||||
|
channel: { select: { id: true, title: true, type: true, telegramId: true } },
|
||||||
|
},
|
||||||
|
orderBy: { createdAt: "desc" },
|
||||||
|
});
|
||||||
|
|
||||||
|
return links.map((l) => ({
|
||||||
|
id: l.id,
|
||||||
|
accountId: l.accountId,
|
||||||
|
channelId: l.channelId,
|
||||||
|
role: l.role,
|
||||||
|
lastProcessedMessageId: l.lastProcessedMessageId?.toString() ?? null,
|
||||||
|
channel: {
|
||||||
|
id: l.channel.id,
|
||||||
|
title: l.channel.title,
|
||||||
|
type: l.channel.type,
|
||||||
|
telegramId: l.channel.telegramId.toString(),
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
export type AccountChannelLinkRow = Awaited<
|
||||||
|
ReturnType<typeof listAccountChannelLinks>
|
||||||
|
>[number];
|
||||||
|
|
||||||
|
export async function getUnlinkedChannels(accountId: string) {
|
||||||
|
const linked = await prisma.accountChannelMap.findMany({
|
||||||
|
where: { accountId },
|
||||||
|
select: { channelId: true },
|
||||||
|
});
|
||||||
|
const linkedIds = linked.map((l) => l.channelId);
|
||||||
|
|
||||||
|
const unlinked = await prisma.telegramChannel.findMany({
|
||||||
|
where: {
|
||||||
|
id: { notIn: linkedIds },
|
||||||
|
isActive: true,
|
||||||
|
},
|
||||||
|
orderBy: { title: "asc" },
|
||||||
|
select: { id: true, title: true, type: true, telegramId: true },
|
||||||
|
});
|
||||||
|
|
||||||
|
return unlinked.map((c) => ({
|
||||||
|
id: c.id,
|
||||||
|
title: c.title,
|
||||||
|
type: c.type,
|
||||||
|
telegramId: c.telegramId.toString(),
|
||||||
|
}));
|
||||||
|
}
|
||||||
45
src/lib/telegram/api-auth.ts
Normal file
45
src/lib/telegram/api-auth.ts
Normal file
@@ -0,0 +1,45 @@
|
|||||||
|
import { auth } from "@/lib/auth";
|
||||||
|
import { NextResponse } from "next/server";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Authenticate an API request. Checks:
|
||||||
|
* 1. X-API-Key header against TELEGRAM_API_KEY env var
|
||||||
|
* 2. NextAuth session
|
||||||
|
*
|
||||||
|
* Returns null if authenticated, or a NextResponse error if not.
|
||||||
|
*/
|
||||||
|
export async function authenticateApiRequest(
|
||||||
|
request: Request,
|
||||||
|
requireAdmin = false
|
||||||
|
): Promise<{ error: NextResponse } | { userId: string; role: string }> {
|
||||||
|
// Check API key first
|
||||||
|
const apiKey = request.headers.get("X-API-Key");
|
||||||
|
const envKey = process.env.TELEGRAM_API_KEY;
|
||||||
|
|
||||||
|
if (apiKey && envKey && apiKey === envKey) {
|
||||||
|
// API key auth — treated as admin
|
||||||
|
return { userId: "api-key", role: "ADMIN" };
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fall back to session auth
|
||||||
|
const session = await auth();
|
||||||
|
if (!session?.user?.id) {
|
||||||
|
return {
|
||||||
|
error: NextResponse.json(
|
||||||
|
{ error: "Unauthorized" },
|
||||||
|
{ status: 401 }
|
||||||
|
),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (requireAdmin && session.user.role !== "ADMIN") {
|
||||||
|
return {
|
||||||
|
error: NextResponse.json(
|
||||||
|
{ error: "Forbidden: admin role required" },
|
||||||
|
{ status: 403 }
|
||||||
|
),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return { userId: session.user.id, role: session.user.role };
|
||||||
|
}
|
||||||
314
src/lib/telegram/queries.ts
Normal file
314
src/lib/telegram/queries.ts
Normal file
@@ -0,0 +1,314 @@
|
|||||||
|
import { prisma } from "@/lib/prisma";
|
||||||
|
import type {
|
||||||
|
PackageListItem,
|
||||||
|
PackageDetail,
|
||||||
|
PackageFileItem,
|
||||||
|
IngestionAccountStatus,
|
||||||
|
} from "./types";
|
||||||
|
|
||||||
|
export async function listPackages(options: {
|
||||||
|
page: number;
|
||||||
|
limit: number;
|
||||||
|
channelId?: string;
|
||||||
|
creator?: string;
|
||||||
|
sortBy: "indexedAt" | "fileName" | "fileSize";
|
||||||
|
order: "asc" | "desc";
|
||||||
|
}) {
|
||||||
|
const where: Record<string, unknown> = {};
|
||||||
|
if (options.channelId) where.sourceChannelId = options.channelId;
|
||||||
|
if (options.creator) where.creator = options.creator;
|
||||||
|
|
||||||
|
const [items, total] = await Promise.all([
|
||||||
|
prisma.package.findMany({
|
||||||
|
where,
|
||||||
|
orderBy: { [options.sortBy]: options.order },
|
||||||
|
skip: (options.page - 1) * options.limit,
|
||||||
|
take: options.limit,
|
||||||
|
select: {
|
||||||
|
id: true,
|
||||||
|
fileName: true,
|
||||||
|
fileSize: true,
|
||||||
|
contentHash: true,
|
||||||
|
archiveType: true,
|
||||||
|
fileCount: true,
|
||||||
|
isMultipart: true,
|
||||||
|
indexedAt: true,
|
||||||
|
creator: true,
|
||||||
|
previewMsgId: true, // cheap null check — avoids loading blob
|
||||||
|
sourceChannel: { select: { id: true, title: true } },
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
prisma.package.count({ where }),
|
||||||
|
]);
|
||||||
|
|
||||||
|
const mapped: PackageListItem[] = items.map((pkg) => ({
|
||||||
|
id: pkg.id,
|
||||||
|
fileName: pkg.fileName,
|
||||||
|
fileSize: pkg.fileSize.toString(),
|
||||||
|
contentHash: pkg.contentHash,
|
||||||
|
archiveType: pkg.archiveType,
|
||||||
|
fileCount: pkg.fileCount,
|
||||||
|
isMultipart: pkg.isMultipart,
|
||||||
|
hasPreview: pkg.previewMsgId !== null,
|
||||||
|
creator: pkg.creator,
|
||||||
|
indexedAt: pkg.indexedAt.toISOString(),
|
||||||
|
sourceChannel: pkg.sourceChannel,
|
||||||
|
}));
|
||||||
|
|
||||||
|
return {
|
||||||
|
items: mapped,
|
||||||
|
pagination: {
|
||||||
|
page: options.page,
|
||||||
|
limit: options.limit,
|
||||||
|
total,
|
||||||
|
totalPages: Math.ceil(total / options.limit),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function getPackageById(
|
||||||
|
id: string
|
||||||
|
): Promise<PackageDetail | null> {
|
||||||
|
const pkg = await prisma.package.findUnique({
|
||||||
|
where: { id },
|
||||||
|
include: {
|
||||||
|
sourceChannel: { select: { id: true, title: true } },
|
||||||
|
ingestionRun: { select: { id: true, startedAt: true } },
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!pkg) return null;
|
||||||
|
|
||||||
|
let destChannel: { id: string; title: string } | null = null;
|
||||||
|
if (pkg.destChannelId) {
|
||||||
|
const ch = await prisma.telegramChannel.findUnique({
|
||||||
|
where: { id: pkg.destChannelId },
|
||||||
|
select: { id: true, title: true },
|
||||||
|
});
|
||||||
|
destChannel = ch;
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
id: pkg.id,
|
||||||
|
fileName: pkg.fileName,
|
||||||
|
fileSize: pkg.fileSize.toString(),
|
||||||
|
contentHash: pkg.contentHash,
|
||||||
|
archiveType: pkg.archiveType,
|
||||||
|
fileCount: pkg.fileCount,
|
||||||
|
isMultipart: pkg.isMultipart,
|
||||||
|
hasPreview: pkg.previewMsgId !== null,
|
||||||
|
creator: pkg.creator,
|
||||||
|
partCount: pkg.partCount,
|
||||||
|
indexedAt: pkg.indexedAt.toISOString(),
|
||||||
|
sourceChannel: pkg.sourceChannel,
|
||||||
|
destChannel,
|
||||||
|
destMessageId: pkg.destMessageId?.toString() ?? null,
|
||||||
|
sourceMessageId: pkg.sourceMessageId.toString(),
|
||||||
|
ingestionRun: pkg.ingestionRun
|
||||||
|
? {
|
||||||
|
id: pkg.ingestionRun.id,
|
||||||
|
startedAt: pkg.ingestionRun.startedAt.toISOString(),
|
||||||
|
}
|
||||||
|
: null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function listPackageFiles(options: {
|
||||||
|
packageId: string;
|
||||||
|
page: number;
|
||||||
|
limit: number;
|
||||||
|
extension?: string;
|
||||||
|
}) {
|
||||||
|
const where: { packageId: string; extension?: string } = {
|
||||||
|
packageId: options.packageId,
|
||||||
|
};
|
||||||
|
if (options.extension) {
|
||||||
|
where.extension = options.extension;
|
||||||
|
}
|
||||||
|
|
||||||
|
const [items, total] = await Promise.all([
|
||||||
|
prisma.packageFile.findMany({
|
||||||
|
where,
|
||||||
|
orderBy: { path: "asc" },
|
||||||
|
skip: (options.page - 1) * options.limit,
|
||||||
|
take: options.limit,
|
||||||
|
}),
|
||||||
|
prisma.packageFile.count({ where }),
|
||||||
|
]);
|
||||||
|
|
||||||
|
const mapped: PackageFileItem[] = items.map((f) => ({
|
||||||
|
id: f.id,
|
||||||
|
path: f.path,
|
||||||
|
fileName: f.fileName,
|
||||||
|
extension: f.extension,
|
||||||
|
compressedSize: f.compressedSize.toString(),
|
||||||
|
uncompressedSize: f.uncompressedSize.toString(),
|
||||||
|
crc32: f.crc32,
|
||||||
|
}));
|
||||||
|
|
||||||
|
return {
|
||||||
|
items: mapped,
|
||||||
|
pagination: {
|
||||||
|
page: options.page,
|
||||||
|
limit: options.limit,
|
||||||
|
total,
|
||||||
|
totalPages: Math.ceil(total / options.limit),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function searchPackages(options: {
|
||||||
|
query: string;
|
||||||
|
page: number;
|
||||||
|
limit: number;
|
||||||
|
searchIn: "packages" | "files" | "both";
|
||||||
|
}) {
|
||||||
|
const q = options.query;
|
||||||
|
|
||||||
|
if (options.searchIn === "files" || options.searchIn === "both") {
|
||||||
|
// Search in package files, return parent packages
|
||||||
|
const fileMatches = await prisma.packageFile.findMany({
|
||||||
|
where: {
|
||||||
|
OR: [
|
||||||
|
{ fileName: { contains: q, mode: "insensitive" } },
|
||||||
|
{ path: { contains: q, mode: "insensitive" } },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
select: { packageId: true },
|
||||||
|
distinct: ["packageId"],
|
||||||
|
});
|
||||||
|
|
||||||
|
const packageIds = fileMatches.map((f) => f.packageId);
|
||||||
|
|
||||||
|
const packageNameIds =
|
||||||
|
options.searchIn === "both"
|
||||||
|
? (
|
||||||
|
await prisma.package.findMany({
|
||||||
|
where: { fileName: { contains: q, mode: "insensitive" } },
|
||||||
|
select: { id: true },
|
||||||
|
})
|
||||||
|
).map((p) => p.id)
|
||||||
|
: [];
|
||||||
|
|
||||||
|
const allIds = [...new Set([...packageIds, ...packageNameIds])];
|
||||||
|
|
||||||
|
const [items, total] = await Promise.all([
|
||||||
|
prisma.package.findMany({
|
||||||
|
where: { id: { in: allIds } },
|
||||||
|
orderBy: { indexedAt: "desc" },
|
||||||
|
skip: (options.page - 1) * options.limit,
|
||||||
|
take: options.limit,
|
||||||
|
select: {
|
||||||
|
id: true,
|
||||||
|
fileName: true,
|
||||||
|
fileSize: true,
|
||||||
|
contentHash: true,
|
||||||
|
archiveType: true,
|
||||||
|
fileCount: true,
|
||||||
|
isMultipart: true,
|
||||||
|
indexedAt: true,
|
||||||
|
creator: true,
|
||||||
|
previewMsgId: true,
|
||||||
|
sourceChannel: { select: { id: true, title: true } },
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
Promise.resolve(allIds.length),
|
||||||
|
]);
|
||||||
|
|
||||||
|
const mapped: PackageListItem[] = items.map((pkg) => ({
|
||||||
|
id: pkg.id,
|
||||||
|
fileName: pkg.fileName,
|
||||||
|
fileSize: pkg.fileSize.toString(),
|
||||||
|
contentHash: pkg.contentHash,
|
||||||
|
archiveType: pkg.archiveType,
|
||||||
|
fileCount: pkg.fileCount,
|
||||||
|
isMultipart: pkg.isMultipart,
|
||||||
|
hasPreview: pkg.previewMsgId !== null,
|
||||||
|
creator: pkg.creator,
|
||||||
|
indexedAt: pkg.indexedAt.toISOString(),
|
||||||
|
sourceChannel: pkg.sourceChannel,
|
||||||
|
}));
|
||||||
|
|
||||||
|
return {
|
||||||
|
items: mapped,
|
||||||
|
pagination: {
|
||||||
|
page: options.page,
|
||||||
|
limit: options.limit,
|
||||||
|
total,
|
||||||
|
totalPages: Math.ceil(total / options.limit),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// Search packages only
|
||||||
|
return listPackages({
|
||||||
|
page: options.page,
|
||||||
|
limit: options.limit,
|
||||||
|
sortBy: "indexedAt",
|
||||||
|
order: "desc",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function getIngestionStatus(): Promise<IngestionAccountStatus[]> {
|
||||||
|
const accounts = await prisma.telegramAccount.findMany({
|
||||||
|
orderBy: { createdAt: "asc" },
|
||||||
|
});
|
||||||
|
|
||||||
|
const statuses: IngestionAccountStatus[] = [];
|
||||||
|
|
||||||
|
for (const account of accounts) {
|
||||||
|
const lastRun = await prisma.ingestionRun.findFirst({
|
||||||
|
where: { accountId: account.id, status: { not: "RUNNING" } },
|
||||||
|
orderBy: { startedAt: "desc" },
|
||||||
|
});
|
||||||
|
|
||||||
|
const currentRun = await prisma.ingestionRun.findFirst({
|
||||||
|
where: { accountId: account.id, status: "RUNNING" },
|
||||||
|
orderBy: { startedAt: "desc" },
|
||||||
|
});
|
||||||
|
|
||||||
|
statuses.push({
|
||||||
|
id: account.id,
|
||||||
|
displayName: account.displayName,
|
||||||
|
phone: account.phone,
|
||||||
|
isActive: account.isActive,
|
||||||
|
authState: account.authState,
|
||||||
|
lastSeenAt: account.lastSeenAt?.toISOString() ?? null,
|
||||||
|
lastRun: lastRun
|
||||||
|
? {
|
||||||
|
id: lastRun.id,
|
||||||
|
status: lastRun.status,
|
||||||
|
startedAt: lastRun.startedAt.toISOString(),
|
||||||
|
finishedAt: lastRun.finishedAt?.toISOString() ?? null,
|
||||||
|
messagesScanned: lastRun.messagesScanned,
|
||||||
|
zipsFound: lastRun.zipsFound,
|
||||||
|
zipsDuplicate: lastRun.zipsDuplicate,
|
||||||
|
zipsIngested: lastRun.zipsIngested,
|
||||||
|
}
|
||||||
|
: null,
|
||||||
|
currentRun: currentRun
|
||||||
|
? {
|
||||||
|
id: currentRun.id,
|
||||||
|
startedAt: currentRun.startedAt.toISOString(),
|
||||||
|
messagesScanned: currentRun.messagesScanned,
|
||||||
|
zipsFound: currentRun.zipsFound,
|
||||||
|
zipsDuplicate: currentRun.zipsDuplicate,
|
||||||
|
zipsIngested: currentRun.zipsIngested,
|
||||||
|
// Live activity tracking
|
||||||
|
currentActivity: currentRun.currentActivity,
|
||||||
|
currentStep: currentRun.currentStep,
|
||||||
|
currentChannel: currentRun.currentChannel,
|
||||||
|
currentFile: currentRun.currentFile,
|
||||||
|
currentFileNum: currentRun.currentFileNum,
|
||||||
|
totalFiles: currentRun.totalFiles,
|
||||||
|
downloadedBytes: currentRun.downloadedBytes?.toString() ?? null,
|
||||||
|
totalBytes: currentRun.totalBytes?.toString() ?? null,
|
||||||
|
downloadPercent: currentRun.downloadPercent,
|
||||||
|
lastActivityAt: currentRun.lastActivityAt?.toISOString() ?? null,
|
||||||
|
}
|
||||||
|
: null,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
return statuses;
|
||||||
|
}
|
||||||
88
src/lib/telegram/types.ts
Normal file
88
src/lib/telegram/types.ts
Normal file
@@ -0,0 +1,88 @@
|
|||||||
|
export interface PackageListItem {
|
||||||
|
id: string;
|
||||||
|
fileName: string;
|
||||||
|
fileSize: string; // BigInt serialized as string
|
||||||
|
contentHash: string;
|
||||||
|
archiveType: "ZIP" | "RAR";
|
||||||
|
fileCount: number;
|
||||||
|
isMultipart: boolean;
|
||||||
|
hasPreview: boolean;
|
||||||
|
creator: string | null;
|
||||||
|
indexedAt: string;
|
||||||
|
sourceChannel: {
|
||||||
|
id: string;
|
||||||
|
title: string;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PackageDetail extends PackageListItem {
|
||||||
|
partCount: number;
|
||||||
|
destChannel: {
|
||||||
|
id: string;
|
||||||
|
title: string;
|
||||||
|
} | null;
|
||||||
|
destMessageId: string | null;
|
||||||
|
sourceMessageId: string;
|
||||||
|
ingestionRun: {
|
||||||
|
id: string;
|
||||||
|
startedAt: string;
|
||||||
|
} | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PackageFileItem {
|
||||||
|
id: string;
|
||||||
|
path: string;
|
||||||
|
fileName: string;
|
||||||
|
extension: string | null;
|
||||||
|
compressedSize: string;
|
||||||
|
uncompressedSize: string;
|
||||||
|
crc32: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PaginatedResponse<T> {
|
||||||
|
items: T[];
|
||||||
|
pagination: {
|
||||||
|
page: number;
|
||||||
|
limit: number;
|
||||||
|
total: number;
|
||||||
|
totalPages: number;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface IngestionAccountStatus {
|
||||||
|
id: string;
|
||||||
|
displayName: string | null;
|
||||||
|
phone: string;
|
||||||
|
isActive: boolean;
|
||||||
|
authState: string;
|
||||||
|
lastSeenAt: string | null;
|
||||||
|
lastRun: {
|
||||||
|
id: string;
|
||||||
|
status: string;
|
||||||
|
startedAt: string;
|
||||||
|
finishedAt: string | null;
|
||||||
|
messagesScanned: number;
|
||||||
|
zipsFound: number;
|
||||||
|
zipsDuplicate: number;
|
||||||
|
zipsIngested: number;
|
||||||
|
} | null;
|
||||||
|
currentRun: {
|
||||||
|
id: string;
|
||||||
|
startedAt: string;
|
||||||
|
messagesScanned: number;
|
||||||
|
zipsFound: number;
|
||||||
|
zipsDuplicate: number;
|
||||||
|
zipsIngested: number;
|
||||||
|
// Live activity tracking
|
||||||
|
currentActivity: string | null;
|
||||||
|
currentStep: string | null;
|
||||||
|
currentChannel: string | null;
|
||||||
|
currentFile: string | null;
|
||||||
|
currentFileNum: number | null;
|
||||||
|
totalFiles: number | null;
|
||||||
|
downloadedBytes: string | null; // BigInt serialized as string
|
||||||
|
totalBytes: string | null; // BigInt serialized as string
|
||||||
|
downloadPercent: number | null;
|
||||||
|
lastActivityAt: string | null;
|
||||||
|
} | null;
|
||||||
|
}
|
||||||
71
src/schemas/telegram.ts
Normal file
71
src/schemas/telegram.ts
Normal file
@@ -0,0 +1,71 @@
|
|||||||
|
import { z } from "zod/v4";
|
||||||
|
|
||||||
|
export const paginationSchema = z.object({
|
||||||
|
page: z.coerce.number().int().min(1).default(1),
|
||||||
|
limit: z.coerce.number().int().min(1).max(100).default(25),
|
||||||
|
});
|
||||||
|
|
||||||
|
export const listPackagesSchema = paginationSchema.extend({
|
||||||
|
channelId: z.string().optional(),
|
||||||
|
creator: z.string().optional(),
|
||||||
|
sortBy: z.enum(["indexedAt", "fileName", "fileSize"]).default("indexedAt"),
|
||||||
|
order: z.enum(["asc", "desc"]).default("desc"),
|
||||||
|
});
|
||||||
|
|
||||||
|
export const listFilesSchema = paginationSchema.extend({
|
||||||
|
limit: z.coerce.number().int().min(1).max(500).default(50),
|
||||||
|
extension: z.string().optional(),
|
||||||
|
});
|
||||||
|
|
||||||
|
export const searchSchema = paginationSchema.extend({
|
||||||
|
q: z.string().min(1),
|
||||||
|
searchIn: z.enum(["packages", "files", "both"]).default("both"),
|
||||||
|
});
|
||||||
|
|
||||||
|
export const triggerIngestionSchema = z.object({
|
||||||
|
accountId: z.string().optional(),
|
||||||
|
});
|
||||||
|
|
||||||
|
// ── Account CRUD ──
|
||||||
|
|
||||||
|
export const telegramAccountSchema = z.object({
|
||||||
|
phone: z
|
||||||
|
.string()
|
||||||
|
.min(1, "Phone number is required")
|
||||||
|
.regex(/^\+?\d[\d\s\-]{6,20}$/, "Invalid phone format (e.g. +31612345678)"),
|
||||||
|
displayName: z.string().max(64).optional().or(z.literal("")),
|
||||||
|
});
|
||||||
|
|
||||||
|
export type TelegramAccountInput = z.infer<typeof telegramAccountSchema>;
|
||||||
|
|
||||||
|
export const submitAuthCodeSchema = z.object({
|
||||||
|
code: z.string().min(3, "Auth code is required").max(10),
|
||||||
|
});
|
||||||
|
|
||||||
|
export type SubmitAuthCodeInput = z.infer<typeof submitAuthCodeSchema>;
|
||||||
|
|
||||||
|
export const submitPasswordSchema = z.object({
|
||||||
|
password: z.string().min(1, "Password is required"),
|
||||||
|
});
|
||||||
|
|
||||||
|
export type SubmitPasswordInput = z.infer<typeof submitPasswordSchema>;
|
||||||
|
|
||||||
|
// ── Channel CRUD ──
|
||||||
|
|
||||||
|
export const telegramChannelSchema = z.object({
|
||||||
|
telegramId: z.coerce.number().int().min(1, "Telegram ID is required"),
|
||||||
|
title: z.string().min(1, "Title is required").max(256),
|
||||||
|
type: z.enum(["SOURCE", "DESTINATION"]),
|
||||||
|
});
|
||||||
|
|
||||||
|
export type TelegramChannelInput = z.infer<typeof telegramChannelSchema>;
|
||||||
|
|
||||||
|
// ── Account-Channel linking ──
|
||||||
|
|
||||||
|
export const linkChannelSchema = z.object({
|
||||||
|
accountId: z.string().min(1),
|
||||||
|
channelId: z.string().min(1),
|
||||||
|
role: z.enum(["READER", "WRITER"]).default("READER"),
|
||||||
|
});
|
||||||
|
|
||||||
|
export type LinkChannelInput = z.infer<typeof linkChannelSchema>;
|
||||||
@@ -31,5 +31,5 @@
|
|||||||
".next/dev/types/**/*.ts",
|
".next/dev/types/**/*.ts",
|
||||||
"**/*.mts"
|
"**/*.mts"
|
||||||
],
|
],
|
||||||
"exclude": ["node_modules", "prisma/seed.ts", "scripts/**"]
|
"exclude": ["node_modules", "prisma/seed.ts", "scripts/**", "worker/**"]
|
||||||
}
|
}
|
||||||
|
|||||||
48
worker/Dockerfile
Normal file
48
worker/Dockerfile
Normal file
@@ -0,0 +1,48 @@
|
|||||||
|
# ── Stage 1: Install production deps ─────────────────────────
|
||||||
|
FROM node:20-bookworm-slim AS deps
|
||||||
|
|
||||||
|
RUN sed -i 's/^Components: main$/Components: main non-free/' /etc/apt/sources.list.d/debian.sources && \
|
||||||
|
apt-get update && apt-get install -y \
|
||||||
|
libssl-dev zlib1g-dev unrar \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
COPY worker/package.json worker/package-lock.json* ./
|
||||||
|
COPY prisma/ ./prisma/
|
||||||
|
|
||||||
|
# Install ALL deps (including devDependencies for tsc) and generate Prisma
|
||||||
|
RUN npm ci && npx prisma generate
|
||||||
|
|
||||||
|
# ── Stage 2: Build TypeScript ─────────────────────────────────
|
||||||
|
FROM deps AS builder
|
||||||
|
|
||||||
|
COPY worker/tsconfig.json ./
|
||||||
|
COPY worker/src/ ./src/
|
||||||
|
RUN npx tsc
|
||||||
|
|
||||||
|
# ── Stage 3: Production runner ────────────────────────────────
|
||||||
|
FROM node:20-bookworm-slim AS runner
|
||||||
|
|
||||||
|
RUN sed -i 's/^Components: main$/Components: main non-free/' /etc/apt/sources.list.d/debian.sources && \
|
||||||
|
apt-get update && apt-get install -y \
|
||||||
|
libssl3 zlib1g unrar \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
# Copy only production node_modules (prune devDeps)
|
||||||
|
COPY --from=builder /app/node_modules ./node_modules
|
||||||
|
COPY --from=builder /app/prisma ./prisma
|
||||||
|
COPY --from=builder /app/dist ./dist
|
||||||
|
|
||||||
|
# Re-generate Prisma client for production (after pruning isn't needed since we copy all)
|
||||||
|
RUN npx prisma generate
|
||||||
|
|
||||||
|
RUN addgroup --system worker && adduser --system --ingroup worker worker
|
||||||
|
RUN mkdir -p /data/tdlib /tmp/zips && chown -R worker:worker /data/tdlib /tmp/zips
|
||||||
|
USER worker
|
||||||
|
|
||||||
|
VOLUME ["/data/tdlib", "/tmp/zips"]
|
||||||
|
|
||||||
|
CMD ["node", "dist/index.js"]
|
||||||
2140
worker/package-lock.json
generated
Normal file
2140
worker/package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load Diff
28
worker/package.json
Normal file
28
worker/package.json
Normal file
@@ -0,0 +1,28 @@
|
|||||||
|
{
|
||||||
|
"name": "dragonsstash-worker",
|
||||||
|
"version": "0.1.0",
|
||||||
|
"private": true,
|
||||||
|
"type": "module",
|
||||||
|
"scripts": {
|
||||||
|
"build": "tsc",
|
||||||
|
"start": "node dist/index.js",
|
||||||
|
"dev": "tsx watch src/index.ts"
|
||||||
|
},
|
||||||
|
"dependencies": {
|
||||||
|
"@prisma/adapter-pg": "^7.4.0",
|
||||||
|
"@prisma/client": "^7.4.0",
|
||||||
|
"pg": "^8.18.0",
|
||||||
|
"pino": "^9.6.0",
|
||||||
|
"prebuilt-tdlib": "^0.1008050.0",
|
||||||
|
"tdl": "^8.0.0",
|
||||||
|
"yauzl": "^3.2.0"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@types/node": "^20",
|
||||||
|
"@types/pg": "^8.16.0",
|
||||||
|
"@types/yauzl": "^2.10.3",
|
||||||
|
"prisma": "^7.4.0",
|
||||||
|
"tsx": "^4.21.0",
|
||||||
|
"typescript": "^5"
|
||||||
|
}
|
||||||
|
}
|
||||||
21
worker/src/archive/creator.ts
Normal file
21
worker/src/archive/creator.ts
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
/**
|
||||||
|
* Extract a creator name from common archive file naming patterns.
|
||||||
|
*
|
||||||
|
* Priority in the worker: topic name > filename extraction.
|
||||||
|
* This is the fallback when no forum topic name is available.
|
||||||
|
*
|
||||||
|
* Patterns handled (split on ` - `):
|
||||||
|
* "Mammoth Factory - 2026-01.zip" → "Mammoth Factory"
|
||||||
|
* "Artist Name - Pack Title.part01.rar" → "Artist Name"
|
||||||
|
* "some_random_file.zip" → null
|
||||||
|
*/
|
||||||
|
export function extractCreatorFromFileName(fileName: string): string | null {
|
||||||
|
// Strip archive extensions (.zip, .rar, .part01.rar, .z01, etc.)
|
||||||
|
const bare = fileName.replace(/(\.(part\d+\.rar|z\d{2}|zip|rar))+$/i, "");
|
||||||
|
|
||||||
|
const idx = bare.indexOf(" - ");
|
||||||
|
if (idx <= 0) return null;
|
||||||
|
|
||||||
|
const creator = bare.slice(0, idx).trim();
|
||||||
|
return creator.length > 0 ? creator : null;
|
||||||
|
}
|
||||||
96
worker/src/archive/detect.ts
Normal file
96
worker/src/archive/detect.ts
Normal file
@@ -0,0 +1,96 @@
|
|||||||
|
export type ArchiveFormat = "ZIP" | "RAR";
|
||||||
|
|
||||||
|
export interface MultipartInfo {
|
||||||
|
baseName: string;
|
||||||
|
partNumber: number;
|
||||||
|
format: ArchiveFormat;
|
||||||
|
pattern: "ZIP_NUMBERED" | "ZIP_LEGACY" | "RAR_PART" | "RAR_LEGACY" | "SINGLE";
|
||||||
|
}
|
||||||
|
|
||||||
|
const patterns: {
|
||||||
|
regex: RegExp;
|
||||||
|
format: ArchiveFormat;
|
||||||
|
pattern: MultipartInfo["pattern"];
|
||||||
|
getBaseName: (match: RegExpMatchArray) => string;
|
||||||
|
getPartNumber: (match: RegExpMatchArray) => number;
|
||||||
|
}[] = [
|
||||||
|
// pack.zip.001, pack.zip.002
|
||||||
|
{
|
||||||
|
regex: /^(.+\.zip)\.(\d{3,})$/i,
|
||||||
|
format: "ZIP",
|
||||||
|
pattern: "ZIP_NUMBERED",
|
||||||
|
getBaseName: (m) => m[1],
|
||||||
|
getPartNumber: (m) => parseInt(m[2], 10),
|
||||||
|
},
|
||||||
|
// pack.z01, pack.z02 (legacy split — final part is pack.zip)
|
||||||
|
{
|
||||||
|
regex: /^(.+)\.z(\d{2,})$/i,
|
||||||
|
format: "ZIP",
|
||||||
|
pattern: "ZIP_LEGACY",
|
||||||
|
getBaseName: (m) => m[1],
|
||||||
|
getPartNumber: (m) => parseInt(m[2], 10),
|
||||||
|
},
|
||||||
|
// pack.part1.rar, pack.part2.rar
|
||||||
|
{
|
||||||
|
regex: /^(.+)\.part(\d+)\.rar$/i,
|
||||||
|
format: "RAR",
|
||||||
|
pattern: "RAR_PART",
|
||||||
|
getBaseName: (m) => m[1],
|
||||||
|
getPartNumber: (m) => parseInt(m[2], 10),
|
||||||
|
},
|
||||||
|
// pack.r00, pack.r01 (legacy split — final part is pack.rar)
|
||||||
|
{
|
||||||
|
regex: /^(.+)\.r(\d{2,})$/i,
|
||||||
|
format: "RAR",
|
||||||
|
pattern: "RAR_LEGACY",
|
||||||
|
getBaseName: (m) => m[1],
|
||||||
|
getPartNumber: (m) => parseInt(m[2], 10),
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Detect if a filename is an archive and extract multipart info.
|
||||||
|
*/
|
||||||
|
export function detectArchive(fileName: string): MultipartInfo | null {
|
||||||
|
// Check multipart patterns first
|
||||||
|
for (const p of patterns) {
|
||||||
|
const match = fileName.match(p.regex);
|
||||||
|
if (match) {
|
||||||
|
return {
|
||||||
|
baseName: p.getBaseName(match),
|
||||||
|
partNumber: p.getPartNumber(match),
|
||||||
|
format: p.format,
|
||||||
|
pattern: p.pattern,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Single .zip file — could be a standalone or the final part of a ZIP_LEGACY set
|
||||||
|
if (/\.zip$/i.test(fileName)) {
|
||||||
|
return {
|
||||||
|
baseName: fileName.replace(/\.zip$/i, ""),
|
||||||
|
partNumber: -1, // -1 signals "could be single or final legacy part"
|
||||||
|
format: "ZIP",
|
||||||
|
pattern: "SINGLE",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// Single .rar file — could be standalone or final part of RAR_LEGACY set
|
||||||
|
if (/\.rar$/i.test(fileName)) {
|
||||||
|
return {
|
||||||
|
baseName: fileName.replace(/\.rar$/i, ""),
|
||||||
|
partNumber: -1,
|
||||||
|
format: "RAR",
|
||||||
|
pattern: "SINGLE",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if a filename looks like any archive attachment we should process.
|
||||||
|
*/
|
||||||
|
export function isArchiveAttachment(fileName: string): boolean {
|
||||||
|
return detectArchive(fileName) !== null;
|
||||||
|
}
|
||||||
25
worker/src/archive/hash.ts
Normal file
25
worker/src/archive/hash.ts
Normal file
@@ -0,0 +1,25 @@
|
|||||||
|
import { createReadStream } from "fs";
|
||||||
|
import { createHash } from "crypto";
|
||||||
|
import { pipeline } from "stream/promises";
|
||||||
|
import { PassThrough } from "stream";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Compute SHA-256 hash of one or more files by streaming them in order.
|
||||||
|
* Memory usage: O(1) — reads in 64KB chunks regardless of total size.
|
||||||
|
* For multipart archives, pass all parts sorted by part number.
|
||||||
|
*/
|
||||||
|
export async function hashParts(filePaths: string[]): Promise<string> {
|
||||||
|
const hash = createHash("sha256");
|
||||||
|
for (const filePath of filePaths) {
|
||||||
|
await pipeline(
|
||||||
|
createReadStream(filePath),
|
||||||
|
new PassThrough({
|
||||||
|
transform(chunk, _encoding, callback) {
|
||||||
|
hash.update(chunk);
|
||||||
|
callback();
|
||||||
|
},
|
||||||
|
})
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return hash.digest("hex");
|
||||||
|
}
|
||||||
100
worker/src/archive/multipart.ts
Normal file
100
worker/src/archive/multipart.ts
Normal file
@@ -0,0 +1,100 @@
|
|||||||
|
import { detectArchive, type ArchiveFormat, type MultipartInfo } from "./detect.js";
|
||||||
|
import { config } from "../util/config.js";
|
||||||
|
import { childLogger } from "../util/logger.js";
|
||||||
|
|
||||||
|
const log = childLogger("multipart");
|
||||||
|
|
||||||
|
export interface TelegramMessage {
|
||||||
|
id: bigint;
|
||||||
|
fileName: string;
|
||||||
|
fileId: string;
|
||||||
|
fileSize: bigint;
|
||||||
|
date: Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ArchiveSet {
|
||||||
|
type: ArchiveFormat;
|
||||||
|
baseName: string;
|
||||||
|
parts: TelegramMessage[];
|
||||||
|
isMultipart: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Group messages into archive sets (single files + multipart groups).
|
||||||
|
* Messages should be pre-filtered to only include archive attachments.
|
||||||
|
*/
|
||||||
|
export function groupArchiveSets(messages: TelegramMessage[]): ArchiveSet[] {
|
||||||
|
// Detect and annotate each message
|
||||||
|
const annotated: { msg: TelegramMessage; info: MultipartInfo }[] = [];
|
||||||
|
for (const msg of messages) {
|
||||||
|
const info = detectArchive(msg.fileName);
|
||||||
|
if (info) {
|
||||||
|
annotated.push({ msg, info });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Group by baseName + format
|
||||||
|
const groups = new Map<string, { msg: TelegramMessage; info: MultipartInfo }[]>();
|
||||||
|
for (const item of annotated) {
|
||||||
|
const key = `${item.info.format}:${item.info.baseName.toLowerCase()}`;
|
||||||
|
const group = groups.get(key) ?? [];
|
||||||
|
group.push(item);
|
||||||
|
groups.set(key, group);
|
||||||
|
}
|
||||||
|
|
||||||
|
const results: ArchiveSet[] = [];
|
||||||
|
|
||||||
|
for (const [, group] of groups) {
|
||||||
|
const format = group[0].info.format;
|
||||||
|
const baseName = group[0].info.baseName;
|
||||||
|
|
||||||
|
// Separate explicit multipart entries from potential singles
|
||||||
|
const multipartEntries = group.filter((g) => g.info.pattern !== "SINGLE");
|
||||||
|
const singleEntries = group.filter((g) => g.info.pattern === "SINGLE");
|
||||||
|
|
||||||
|
if (multipartEntries.length > 0) {
|
||||||
|
// This is a multipart set
|
||||||
|
// Check if any single entry is the "final part" of a legacy split
|
||||||
|
const allEntries = [...multipartEntries, ...singleEntries];
|
||||||
|
|
||||||
|
// Check time span — skip if parts span too long
|
||||||
|
const dates = allEntries.map((e) => e.msg.date.getTime());
|
||||||
|
const span = Math.max(...dates) - Math.min(...dates);
|
||||||
|
const maxSpanMs = config.multipartTimeoutHours * 60 * 60 * 1000;
|
||||||
|
|
||||||
|
if (span > maxSpanMs) {
|
||||||
|
log.warn(
|
||||||
|
{ baseName, format, span: span / 3600000 },
|
||||||
|
"Multipart set spans too long, skipping"
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sort by part number (singles get a very high number so they come last — they're the final part)
|
||||||
|
allEntries.sort((a, b) => {
|
||||||
|
const aNum = a.info.partNumber === -1 ? 999999 : a.info.partNumber;
|
||||||
|
const bNum = b.info.partNumber === -1 ? 999999 : b.info.partNumber;
|
||||||
|
return aNum - bNum;
|
||||||
|
});
|
||||||
|
|
||||||
|
results.push({
|
||||||
|
type: format,
|
||||||
|
baseName,
|
||||||
|
parts: allEntries.map((e) => e.msg),
|
||||||
|
isMultipart: true,
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
// All entries are singles — each is its own archive set
|
||||||
|
for (const entry of singleEntries) {
|
||||||
|
results.push({
|
||||||
|
type: format,
|
||||||
|
baseName: entry.info.baseName,
|
||||||
|
parts: [entry.msg],
|
||||||
|
isMultipart: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return results;
|
||||||
|
}
|
||||||
90
worker/src/archive/rar-reader.ts
Normal file
90
worker/src/archive/rar-reader.ts
Normal file
@@ -0,0 +1,90 @@
|
|||||||
|
import { execFile } from "child_process";
|
||||||
|
import { promisify } from "util";
|
||||||
|
import path from "path";
|
||||||
|
import { childLogger } from "../util/logger.js";
|
||||||
|
import type { FileEntry } from "./zip-reader.js";
|
||||||
|
|
||||||
|
const execFileAsync = promisify(execFile);
|
||||||
|
const log = childLogger("rar-reader");
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Parse output of `unrar l -v <file>` to extract file metadata.
|
||||||
|
* unrar automatically discovers sibling parts when they're co-located.
|
||||||
|
*/
|
||||||
|
export async function readRarContents(
|
||||||
|
firstPartPath: string
|
||||||
|
): Promise<FileEntry[]> {
|
||||||
|
try {
|
||||||
|
const { stdout } = await execFileAsync("unrar", ["l", "-v", firstPartPath], {
|
||||||
|
timeout: 30000,
|
||||||
|
maxBuffer: 10 * 1024 * 1024, // 10MB for very large archives
|
||||||
|
});
|
||||||
|
|
||||||
|
return parseUnrarOutput(stdout);
|
||||||
|
} catch (err) {
|
||||||
|
log.warn({ err, file: firstPartPath }, "Failed to read RAR contents");
|
||||||
|
return []; // Fallback: return empty on error
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Parse the tabular output of `unrar l -v`.
|
||||||
|
*
|
||||||
|
* Example output format:
|
||||||
|
* Archive: test.rar
|
||||||
|
* Details: RAR 5
|
||||||
|
*
|
||||||
|
* Attributes Size Packed Ratio Date Time CRC-32 Name
|
||||||
|
* ----------- --------- --------- ----- -------- ----- -------- ----
|
||||||
|
* ...A.... 12345 10234 83% 2024-01-15 10:30 DEADBEEF folder/file.stl
|
||||||
|
* ----------- --------- --------- ----- -------- ----- -------- ----
|
||||||
|
*/
|
||||||
|
function parseUnrarOutput(output: string): FileEntry[] {
|
||||||
|
const entries: FileEntry[] = [];
|
||||||
|
const lines = output.split("\n");
|
||||||
|
|
||||||
|
let inFileList = false;
|
||||||
|
let separatorCount = 0;
|
||||||
|
|
||||||
|
for (const line of lines) {
|
||||||
|
const trimmed = line.trim();
|
||||||
|
|
||||||
|
// Detect separator lines (------- pattern)
|
||||||
|
if (/^-{5,}/.test(trimmed)) {
|
||||||
|
separatorCount++;
|
||||||
|
if (separatorCount === 1) {
|
||||||
|
inFileList = true;
|
||||||
|
} else if (separatorCount >= 2) {
|
||||||
|
inFileList = false;
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!inFileList) continue;
|
||||||
|
|
||||||
|
// Parse file entry line
|
||||||
|
// Format: Attributes Size Packed Ratio Date Time CRC Name
|
||||||
|
const match = trimmed.match(
|
||||||
|
/^\S+\s+(\d+)\s+(\d+)\s+\d+%\s+\S+\s+\S+\s+([0-9A-Fa-f]+)\s+(.+)$/
|
||||||
|
);
|
||||||
|
|
||||||
|
if (match) {
|
||||||
|
const [, uncompressedStr, compressedStr, crc32, filePath] = match;
|
||||||
|
|
||||||
|
// Skip directory entries (typically end with / or have size 0 with dir attributes)
|
||||||
|
if (filePath.endsWith("/") || filePath.endsWith("\\")) continue;
|
||||||
|
|
||||||
|
const ext = path.extname(filePath).toLowerCase();
|
||||||
|
entries.push({
|
||||||
|
path: filePath,
|
||||||
|
fileName: path.basename(filePath),
|
||||||
|
extension: ext ? ext.slice(1) : null,
|
||||||
|
compressedSize: BigInt(compressedStr),
|
||||||
|
uncompressedSize: BigInt(uncompressedStr),
|
||||||
|
crc32: crc32.toLowerCase(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return entries;
|
||||||
|
}
|
||||||
48
worker/src/archive/split.ts
Normal file
48
worker/src/archive/split.ts
Normal file
@@ -0,0 +1,48 @@
|
|||||||
|
import { createReadStream, createWriteStream } from "fs";
|
||||||
|
import { stat } from "fs/promises";
|
||||||
|
import path from "path";
|
||||||
|
import { pipeline } from "stream/promises";
|
||||||
|
import { childLogger } from "../util/logger.js";
|
||||||
|
|
||||||
|
const log = childLogger("split");
|
||||||
|
|
||||||
|
/** 2GB in bytes — Telegram's file size limit */
|
||||||
|
const MAX_PART_SIZE = 2n * 1024n * 1024n * 1024n;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Split a file into ≤2GB parts using byte-level splitting.
|
||||||
|
* Returns paths to the split parts. If the file is already ≤2GB, returns the original path.
|
||||||
|
*/
|
||||||
|
export async function byteLevelSplit(filePath: string): Promise<string[]> {
|
||||||
|
const stats = await stat(filePath);
|
||||||
|
const fileSize = BigInt(stats.size);
|
||||||
|
|
||||||
|
if (fileSize <= MAX_PART_SIZE) {
|
||||||
|
return [filePath];
|
||||||
|
}
|
||||||
|
|
||||||
|
const dir = path.dirname(filePath);
|
||||||
|
const baseName = path.basename(filePath);
|
||||||
|
const partSize = Number(MAX_PART_SIZE);
|
||||||
|
const totalParts = Math.ceil(Number(fileSize) / partSize);
|
||||||
|
const parts: string[] = [];
|
||||||
|
|
||||||
|
log.info({ filePath, fileSize: Number(fileSize), totalParts }, "Splitting file");
|
||||||
|
|
||||||
|
for (let i = 0; i < totalParts; i++) {
|
||||||
|
const partNum = String(i + 1).padStart(3, "0");
|
||||||
|
const partPath = path.join(dir, `${baseName}.${partNum}`);
|
||||||
|
const start = i * partSize;
|
||||||
|
const end = Math.min(start + partSize - 1, Number(fileSize) - 1);
|
||||||
|
|
||||||
|
await pipeline(
|
||||||
|
createReadStream(filePath, { start, end }),
|
||||||
|
createWriteStream(partPath)
|
||||||
|
);
|
||||||
|
|
||||||
|
parts.push(partPath);
|
||||||
|
}
|
||||||
|
|
||||||
|
log.info({ filePath, parts: parts.length }, "File split complete");
|
||||||
|
return parts;
|
||||||
|
}
|
||||||
61
worker/src/archive/zip-reader.ts
Normal file
61
worker/src/archive/zip-reader.ts
Normal file
@@ -0,0 +1,61 @@
|
|||||||
|
import yauzl from "yauzl";
|
||||||
|
import path from "path";
|
||||||
|
import { childLogger } from "../util/logger.js";
|
||||||
|
|
||||||
|
const log = childLogger("zip-reader");
|
||||||
|
|
||||||
|
export interface FileEntry {
|
||||||
|
path: string;
|
||||||
|
fileName: string;
|
||||||
|
extension: string | null;
|
||||||
|
compressedSize: bigint;
|
||||||
|
uncompressedSize: bigint;
|
||||||
|
crc32: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Read the central directory of a ZIP file without extracting any contents.
|
||||||
|
* For multipart ZIPs, pass the paths sorted by part order.
|
||||||
|
* We attempt to read from the last part first (central directory is at the end).
|
||||||
|
*/
|
||||||
|
export async function readZipCentralDirectory(
|
||||||
|
filePaths: string[]
|
||||||
|
): Promise<FileEntry[]> {
|
||||||
|
// The central directory lives at the end of the last file
|
||||||
|
const targetFile = filePaths[filePaths.length - 1];
|
||||||
|
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
yauzl.open(targetFile, { lazyEntries: true, autoClose: true }, (err, zipFile) => {
|
||||||
|
if (err) {
|
||||||
|
log.warn({ err, file: targetFile }, "Failed to open ZIP for reading");
|
||||||
|
resolve([]); // Fallback: return empty on error
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const entries: FileEntry[] = [];
|
||||||
|
|
||||||
|
zipFile.readEntry();
|
||||||
|
zipFile.on("entry", (entry: yauzl.Entry) => {
|
||||||
|
// Skip directories
|
||||||
|
if (!entry.fileName.endsWith("/")) {
|
||||||
|
const ext = path.extname(entry.fileName).toLowerCase();
|
||||||
|
entries.push({
|
||||||
|
path: entry.fileName,
|
||||||
|
fileName: path.basename(entry.fileName),
|
||||||
|
extension: ext ? ext.slice(1) : null, // Remove leading dot
|
||||||
|
compressedSize: BigInt(entry.compressedSize),
|
||||||
|
uncompressedSize: BigInt(entry.uncompressedSize),
|
||||||
|
crc32: entry.crc32 !== 0 ? entry.crc32.toString(16).padStart(8, "0") : null,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
zipFile.readEntry();
|
||||||
|
});
|
||||||
|
|
||||||
|
zipFile.on("end", () => resolve(entries));
|
||||||
|
zipFile.on("error", (error) => {
|
||||||
|
log.warn({ error, file: targetFile }, "Error reading ZIP entries");
|
||||||
|
resolve(entries); // Return whatever we got
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
14
worker/src/db/client.ts
Normal file
14
worker/src/db/client.ts
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
import { PrismaClient } from "@prisma/client";
|
||||||
|
import { PrismaPg } from "@prisma/adapter-pg";
|
||||||
|
import pg from "pg";
|
||||||
|
import { config } from "../util/config.js";
|
||||||
|
|
||||||
|
const pool = new pg.Pool({
|
||||||
|
connectionString: config.databaseUrl,
|
||||||
|
max: 5,
|
||||||
|
});
|
||||||
|
|
||||||
|
const adapter = new PrismaPg(pool);
|
||||||
|
export const db = new PrismaClient({ adapter });
|
||||||
|
|
||||||
|
export { pool };
|
||||||
56
worker/src/db/locks.ts
Normal file
56
worker/src/db/locks.ts
Normal file
@@ -0,0 +1,56 @@
|
|||||||
|
import { pool } from "./client.js";
|
||||||
|
import { childLogger } from "../util/logger.js";
|
||||||
|
|
||||||
|
const log = childLogger("locks");
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Derive a stable 32-bit integer lock ID from an account ID string.
|
||||||
|
* PostgreSQL advisory locks use bigint, but we use 32-bit for safety.
|
||||||
|
*/
|
||||||
|
function hashToLockId(accountId: string): number {
|
||||||
|
let hash = 0;
|
||||||
|
for (let i = 0; i < accountId.length; i++) {
|
||||||
|
const char = accountId.charCodeAt(i);
|
||||||
|
hash = (hash << 5) - hash + char;
|
||||||
|
hash |= 0; // Convert to 32-bit integer
|
||||||
|
}
|
||||||
|
return Math.abs(hash);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Try to acquire a PostgreSQL advisory lock for an account.
|
||||||
|
* Returns true if acquired, false if already held by another session.
|
||||||
|
*/
|
||||||
|
export async function tryAcquireLock(accountId: string): Promise<boolean> {
|
||||||
|
const lockId = hashToLockId(accountId);
|
||||||
|
const client = await pool.connect();
|
||||||
|
try {
|
||||||
|
const result = await client.query<{ pg_try_advisory_lock: boolean }>(
|
||||||
|
"SELECT pg_try_advisory_lock($1)",
|
||||||
|
[lockId]
|
||||||
|
);
|
||||||
|
const acquired = result.rows[0]?.pg_try_advisory_lock ?? false;
|
||||||
|
if (acquired) {
|
||||||
|
log.debug({ accountId, lockId }, "Advisory lock acquired");
|
||||||
|
} else {
|
||||||
|
log.debug({ accountId, lockId }, "Advisory lock already held");
|
||||||
|
}
|
||||||
|
return acquired;
|
||||||
|
} finally {
|
||||||
|
client.release();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Release the advisory lock for an account.
|
||||||
|
*/
|
||||||
|
export async function releaseLock(accountId: string): Promise<void> {
|
||||||
|
const lockId = hashToLockId(accountId);
|
||||||
|
const client = await pool.connect();
|
||||||
|
try {
|
||||||
|
await client.query("SELECT pg_advisory_unlock($1)", [lockId]);
|
||||||
|
log.debug({ accountId, lockId }, "Advisory lock released");
|
||||||
|
} finally {
|
||||||
|
client.release();
|
||||||
|
}
|
||||||
|
}
|
||||||
270
worker/src/db/queries.ts
Normal file
270
worker/src/db/queries.ts
Normal file
@@ -0,0 +1,270 @@
|
|||||||
|
import { db } from "./client.js";
|
||||||
|
import type { ArchiveType } from "@prisma/client";
|
||||||
|
|
||||||
|
export async function getActiveAccounts() {
|
||||||
|
return db.telegramAccount.findMany({
|
||||||
|
where: { isActive: true, authState: "AUTHENTICATED" },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function getSourceChannelMappings(accountId: string) {
|
||||||
|
return db.accountChannelMap.findMany({
|
||||||
|
where: {
|
||||||
|
accountId,
|
||||||
|
role: "READER",
|
||||||
|
channel: { type: "SOURCE", isActive: true },
|
||||||
|
},
|
||||||
|
include: { channel: true },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function getDestinationChannel(accountId: string) {
|
||||||
|
const mapping = await db.accountChannelMap.findFirst({
|
||||||
|
where: {
|
||||||
|
accountId,
|
||||||
|
role: "WRITER",
|
||||||
|
channel: { type: "DESTINATION", isActive: true },
|
||||||
|
},
|
||||||
|
include: { channel: true },
|
||||||
|
});
|
||||||
|
return mapping?.channel ?? null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function packageExistsByHash(contentHash: string) {
|
||||||
|
const pkg = await db.package.findUnique({
|
||||||
|
where: { contentHash },
|
||||||
|
select: { id: true },
|
||||||
|
});
|
||||||
|
return pkg !== null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CreatePackageInput {
|
||||||
|
contentHash: string;
|
||||||
|
fileName: string;
|
||||||
|
fileSize: bigint;
|
||||||
|
archiveType: ArchiveType;
|
||||||
|
sourceChannelId: string;
|
||||||
|
sourceMessageId: bigint;
|
||||||
|
sourceTopicId?: bigint | null;
|
||||||
|
destChannelId?: string;
|
||||||
|
destMessageId?: bigint;
|
||||||
|
isMultipart: boolean;
|
||||||
|
partCount: number;
|
||||||
|
ingestionRunId: string;
|
||||||
|
creator?: string | null;
|
||||||
|
previewData?: Buffer | null;
|
||||||
|
previewMsgId?: bigint | null;
|
||||||
|
files: {
|
||||||
|
path: string;
|
||||||
|
fileName: string;
|
||||||
|
extension: string | null;
|
||||||
|
compressedSize: bigint;
|
||||||
|
uncompressedSize: bigint;
|
||||||
|
crc32: string | null;
|
||||||
|
}[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function createPackageWithFiles(input: CreatePackageInput) {
|
||||||
|
return db.package.create({
|
||||||
|
data: {
|
||||||
|
contentHash: input.contentHash,
|
||||||
|
fileName: input.fileName,
|
||||||
|
fileSize: input.fileSize,
|
||||||
|
archiveType: input.archiveType,
|
||||||
|
sourceChannelId: input.sourceChannelId,
|
||||||
|
sourceMessageId: input.sourceMessageId,
|
||||||
|
sourceTopicId: input.sourceTopicId ?? undefined,
|
||||||
|
destChannelId: input.destChannelId,
|
||||||
|
destMessageId: input.destMessageId,
|
||||||
|
isMultipart: input.isMultipart,
|
||||||
|
partCount: input.partCount,
|
||||||
|
fileCount: input.files.length,
|
||||||
|
ingestionRunId: input.ingestionRunId,
|
||||||
|
creator: input.creator ?? undefined,
|
||||||
|
previewData: input.previewData ? new Uint8Array(input.previewData) : undefined,
|
||||||
|
previewMsgId: input.previewMsgId ?? undefined,
|
||||||
|
files: {
|
||||||
|
create: input.files,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function createIngestionRun(accountId: string) {
|
||||||
|
return db.ingestionRun.create({
|
||||||
|
data: {
|
||||||
|
accountId,
|
||||||
|
status: "RUNNING",
|
||||||
|
currentActivity: "Starting ingestion run",
|
||||||
|
currentStep: "initializing",
|
||||||
|
lastActivityAt: new Date(),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ActivityUpdate {
|
||||||
|
currentActivity: string;
|
||||||
|
currentStep: string;
|
||||||
|
currentChannel?: string | null;
|
||||||
|
currentFile?: string | null;
|
||||||
|
currentFileNum?: number | null;
|
||||||
|
totalFiles?: number | null;
|
||||||
|
downloadedBytes?: bigint | null;
|
||||||
|
totalBytes?: bigint | null;
|
||||||
|
downloadPercent?: number | null;
|
||||||
|
messagesScanned?: number;
|
||||||
|
zipsFound?: number;
|
||||||
|
zipsDuplicate?: number;
|
||||||
|
zipsIngested?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function updateRunActivity(
|
||||||
|
runId: string,
|
||||||
|
activity: ActivityUpdate
|
||||||
|
) {
|
||||||
|
return db.ingestionRun.update({
|
||||||
|
where: { id: runId },
|
||||||
|
data: {
|
||||||
|
currentActivity: activity.currentActivity,
|
||||||
|
currentStep: activity.currentStep,
|
||||||
|
currentChannel: activity.currentChannel ?? undefined,
|
||||||
|
currentFile: activity.currentFile ?? undefined,
|
||||||
|
currentFileNum: activity.currentFileNum ?? undefined,
|
||||||
|
totalFiles: activity.totalFiles ?? undefined,
|
||||||
|
downloadedBytes: activity.downloadedBytes ?? undefined,
|
||||||
|
totalBytes: activity.totalBytes ?? undefined,
|
||||||
|
downloadPercent: activity.downloadPercent ?? undefined,
|
||||||
|
lastActivityAt: new Date(),
|
||||||
|
...(activity.messagesScanned !== undefined && { messagesScanned: activity.messagesScanned }),
|
||||||
|
...(activity.zipsFound !== undefined && { zipsFound: activity.zipsFound }),
|
||||||
|
...(activity.zipsDuplicate !== undefined && { zipsDuplicate: activity.zipsDuplicate }),
|
||||||
|
...(activity.zipsIngested !== undefined && { zipsIngested: activity.zipsIngested }),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const CLEAR_ACTIVITY = {
|
||||||
|
currentActivity: null,
|
||||||
|
currentStep: null,
|
||||||
|
currentChannel: null,
|
||||||
|
currentFile: null,
|
||||||
|
currentFileNum: null,
|
||||||
|
totalFiles: null,
|
||||||
|
downloadedBytes: null,
|
||||||
|
totalBytes: null,
|
||||||
|
downloadPercent: null,
|
||||||
|
lastActivityAt: new Date(),
|
||||||
|
};
|
||||||
|
|
||||||
|
export async function completeIngestionRun(
|
||||||
|
runId: string,
|
||||||
|
counters: {
|
||||||
|
messagesScanned: number;
|
||||||
|
zipsFound: number;
|
||||||
|
zipsDuplicate: number;
|
||||||
|
zipsIngested: number;
|
||||||
|
}
|
||||||
|
) {
|
||||||
|
return db.ingestionRun.update({
|
||||||
|
where: { id: runId },
|
||||||
|
data: {
|
||||||
|
status: "COMPLETED",
|
||||||
|
finishedAt: new Date(),
|
||||||
|
...counters,
|
||||||
|
...CLEAR_ACTIVITY,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function failIngestionRun(runId: string, errorMessage: string) {
|
||||||
|
return db.ingestionRun.update({
|
||||||
|
where: { id: runId },
|
||||||
|
data: {
|
||||||
|
status: "FAILED",
|
||||||
|
finishedAt: new Date(),
|
||||||
|
errorMessage,
|
||||||
|
...CLEAR_ACTIVITY,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function updateLastProcessedMessage(
|
||||||
|
mappingId: string,
|
||||||
|
messageId: bigint
|
||||||
|
) {
|
||||||
|
return db.accountChannelMap.update({
|
||||||
|
where: { id: mappingId },
|
||||||
|
data: { lastProcessedMessageId: messageId },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function markStaleRunsAsFailed() {
|
||||||
|
return db.ingestionRun.updateMany({
|
||||||
|
where: { status: "RUNNING" },
|
||||||
|
data: {
|
||||||
|
status: "FAILED",
|
||||||
|
finishedAt: new Date(),
|
||||||
|
errorMessage: "Worker restarted — run was still marked as RUNNING",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function updateAccountAuthState(
|
||||||
|
accountId: string,
|
||||||
|
authState: "PENDING" | "AWAITING_CODE" | "AWAITING_PASSWORD" | "AUTHENTICATED" | "EXPIRED",
|
||||||
|
authCode?: string | null
|
||||||
|
) {
|
||||||
|
return db.telegramAccount.update({
|
||||||
|
where: { id: accountId },
|
||||||
|
data: { authState, authCode, lastSeenAt: authState === "AUTHENTICATED" ? new Date() : undefined },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function getAccountAuthCode(accountId: string) {
|
||||||
|
const account = await db.telegramAccount.findUnique({
|
||||||
|
where: { id: accountId },
|
||||||
|
select: { authCode: true, authState: true },
|
||||||
|
});
|
||||||
|
return account;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Forum / Topic progress ──
|
||||||
|
|
||||||
|
export async function setChannelForum(channelId: string, isForum: boolean) {
|
||||||
|
return db.telegramChannel.update({
|
||||||
|
where: { id: channelId },
|
||||||
|
data: { isForum },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function getTopicProgress(mappingId: string) {
|
||||||
|
return db.topicProgress.findMany({
|
||||||
|
where: { accountChannelMapId: mappingId },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function upsertTopicProgress(
|
||||||
|
mappingId: string,
|
||||||
|
topicId: bigint,
|
||||||
|
topicName: string | null,
|
||||||
|
lastProcessedMessageId: bigint
|
||||||
|
) {
|
||||||
|
return db.topicProgress.upsert({
|
||||||
|
where: {
|
||||||
|
accountChannelMapId_topicId: {
|
||||||
|
accountChannelMapId: mappingId,
|
||||||
|
topicId,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
create: {
|
||||||
|
accountChannelMapId: mappingId,
|
||||||
|
topicId,
|
||||||
|
topicName,
|
||||||
|
lastProcessedMessageId,
|
||||||
|
},
|
||||||
|
update: {
|
||||||
|
topicName,
|
||||||
|
lastProcessedMessageId,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
50
worker/src/index.ts
Normal file
50
worker/src/index.ts
Normal file
@@ -0,0 +1,50 @@
|
|||||||
|
import { mkdir } from "fs/promises";
|
||||||
|
import { config } from "./util/config.js";
|
||||||
|
import { logger } from "./util/logger.js";
|
||||||
|
import { markStaleRunsAsFailed } from "./db/queries.js";
|
||||||
|
import { cleanupTempDir } from "./worker.js";
|
||||||
|
import { startScheduler, stopScheduler } from "./scheduler.js";
|
||||||
|
import { db, pool } from "./db/client.js";
|
||||||
|
|
||||||
|
const log = logger.child({ module: "main" });
|
||||||
|
|
||||||
|
async function main(): Promise<void> {
|
||||||
|
log.info("DragonsStash Telegram Worker starting");
|
||||||
|
log.info({ config: { ...config, databaseUrl: "***" } }, "Configuration loaded");
|
||||||
|
|
||||||
|
// Ensure temp directory exists
|
||||||
|
await mkdir(config.tempDir, { recursive: true });
|
||||||
|
await mkdir(config.tdlibStateDir, { recursive: true });
|
||||||
|
|
||||||
|
// Clean up stale state
|
||||||
|
await cleanupTempDir();
|
||||||
|
await markStaleRunsAsFailed();
|
||||||
|
|
||||||
|
// Start the scheduler
|
||||||
|
await startScheduler();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Graceful shutdown
|
||||||
|
function shutdown(signal: string): void {
|
||||||
|
log.info({ signal }, "Shutdown signal received");
|
||||||
|
stopScheduler();
|
||||||
|
|
||||||
|
// Close DB connections
|
||||||
|
Promise.all([db.$disconnect(), pool.end()])
|
||||||
|
.then(() => {
|
||||||
|
log.info("Shutdown complete");
|
||||||
|
process.exit(0);
|
||||||
|
})
|
||||||
|
.catch((err) => {
|
||||||
|
log.error({ err }, "Error during shutdown");
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
process.on("SIGTERM", () => shutdown("SIGTERM"));
|
||||||
|
process.on("SIGINT", () => shutdown("SIGINT"));
|
||||||
|
|
||||||
|
main().catch((err) => {
|
||||||
|
log.fatal({ err }, "Worker failed to start");
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
86
worker/src/preview/match.ts
Normal file
86
worker/src/preview/match.ts
Normal file
@@ -0,0 +1,86 @@
|
|||||||
|
import { childLogger } from "../util/logger.js";
|
||||||
|
|
||||||
|
const log = childLogger("preview-match");
|
||||||
|
|
||||||
|
export interface TelegramPhoto {
|
||||||
|
id: bigint;
|
||||||
|
date: Date;
|
||||||
|
/** Caption text on the photo message (if any). */
|
||||||
|
caption: string;
|
||||||
|
/** The smallest photo size available — used as thumbnail. */
|
||||||
|
fileId: string;
|
||||||
|
fileSize: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ArchiveRef {
|
||||||
|
baseName: string;
|
||||||
|
firstMessageId: bigint;
|
||||||
|
firstMessageDate: Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Try to match a photo message to an archive by:
|
||||||
|
* 1. Caption contains the archive baseName (without extension)
|
||||||
|
* 2. Photo was posted within ±10 messages (time-window: ±6 hours)
|
||||||
|
*
|
||||||
|
* Returns the best match (closest in time), or null.
|
||||||
|
*/
|
||||||
|
export function matchPreviewToArchive(
|
||||||
|
photos: TelegramPhoto[],
|
||||||
|
archives: ArchiveRef[]
|
||||||
|
): Map<string, TelegramPhoto> {
|
||||||
|
const results = new Map<string, TelegramPhoto>();
|
||||||
|
const TIME_WINDOW_MS = 6 * 60 * 60 * 1000; // 6 hours
|
||||||
|
|
||||||
|
for (const archive of archives) {
|
||||||
|
// Normalize the archive base name for matching
|
||||||
|
const normalizedBase = normalizeForMatch(archive.baseName);
|
||||||
|
if (!normalizedBase) continue;
|
||||||
|
|
||||||
|
let bestMatch: TelegramPhoto | null = null;
|
||||||
|
let bestTimeDiff = Infinity;
|
||||||
|
|
||||||
|
for (const photo of photos) {
|
||||||
|
const timeDiff = Math.abs(
|
||||||
|
photo.date.getTime() - archive.firstMessageDate.getTime()
|
||||||
|
);
|
||||||
|
|
||||||
|
// Must be within time window
|
||||||
|
if (timeDiff > TIME_WINDOW_MS) continue;
|
||||||
|
|
||||||
|
// Check if the photo caption contains the archive base name
|
||||||
|
const normalizedCaption = normalizeForMatch(photo.caption);
|
||||||
|
if (!normalizedCaption) continue;
|
||||||
|
|
||||||
|
const matches =
|
||||||
|
normalizedCaption.includes(normalizedBase) ||
|
||||||
|
normalizedBase.includes(normalizedCaption);
|
||||||
|
|
||||||
|
if (matches && timeDiff < bestTimeDiff) {
|
||||||
|
bestMatch = photo;
|
||||||
|
bestTimeDiff = timeDiff;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (bestMatch) {
|
||||||
|
log.debug(
|
||||||
|
{ baseName: archive.baseName, photoId: bestMatch.id.toString() },
|
||||||
|
"Matched preview photo to archive"
|
||||||
|
);
|
||||||
|
results.set(archive.baseName, bestMatch);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Strip extension, punctuation, and normalize for fuzzy matching.
|
||||||
|
*/
|
||||||
|
function normalizeForMatch(input: string): string {
|
||||||
|
return input
|
||||||
|
.toLowerCase()
|
||||||
|
.replace(/\.[a-z0-9]{1,5}$/i, "") // strip extension
|
||||||
|
.replace(/[_\-.\s]+/g, " ") // normalize separators
|
||||||
|
.trim();
|
||||||
|
}
|
||||||
92
worker/src/scheduler.ts
Normal file
92
worker/src/scheduler.ts
Normal file
@@ -0,0 +1,92 @@
|
|||||||
|
import { config } from "./util/config.js";
|
||||||
|
import { childLogger } from "./util/logger.js";
|
||||||
|
import { getActiveAccounts } from "./db/queries.js";
|
||||||
|
import { runWorkerForAccount } from "./worker.js";
|
||||||
|
|
||||||
|
const log = childLogger("scheduler");
|
||||||
|
|
||||||
|
let running = false;
|
||||||
|
let timer: ReturnType<typeof setTimeout> | null = null;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Run one ingestion cycle: process all active, authenticated accounts sequentially.
|
||||||
|
*/
|
||||||
|
async function runCycle(): Promise<void> {
|
||||||
|
if (running) {
|
||||||
|
log.warn("Previous cycle still running, skipping");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
running = true;
|
||||||
|
log.info("Starting ingestion cycle");
|
||||||
|
|
||||||
|
try {
|
||||||
|
const accounts = await getActiveAccounts();
|
||||||
|
|
||||||
|
if (accounts.length === 0) {
|
||||||
|
log.info("No active authenticated accounts, nothing to do");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
log.info({ accountCount: accounts.length }, "Processing accounts");
|
||||||
|
|
||||||
|
for (const account of accounts) {
|
||||||
|
await runWorkerForAccount(account);
|
||||||
|
}
|
||||||
|
|
||||||
|
log.info("Ingestion cycle complete");
|
||||||
|
} catch (err) {
|
||||||
|
log.error({ err }, "Ingestion cycle failed");
|
||||||
|
} finally {
|
||||||
|
running = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Schedule the next cycle with jitter.
|
||||||
|
*/
|
||||||
|
function scheduleNext(): void {
|
||||||
|
const intervalMs = config.workerIntervalMinutes * 60 * 1000;
|
||||||
|
const jitterMs = Math.random() * config.jitterMinutes * 60 * 1000;
|
||||||
|
const delay = intervalMs + jitterMs;
|
||||||
|
|
||||||
|
log.info(
|
||||||
|
{ nextRunInMinutes: Math.round(delay / 60000) },
|
||||||
|
"Next cycle scheduled"
|
||||||
|
);
|
||||||
|
|
||||||
|
timer = setTimeout(async () => {
|
||||||
|
await runCycle();
|
||||||
|
scheduleNext();
|
||||||
|
}, delay);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Start the scheduler. Runs an immediate first cycle, then schedules subsequent ones.
|
||||||
|
*/
|
||||||
|
export async function startScheduler(): Promise<void> {
|
||||||
|
log.info(
|
||||||
|
{
|
||||||
|
intervalMinutes: config.workerIntervalMinutes,
|
||||||
|
jitterMinutes: config.jitterMinutes,
|
||||||
|
},
|
||||||
|
"Scheduler starting"
|
||||||
|
);
|
||||||
|
|
||||||
|
// Run immediately on start
|
||||||
|
await runCycle();
|
||||||
|
|
||||||
|
// Then schedule recurring cycles
|
||||||
|
scheduleNext();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Stop the scheduler gracefully.
|
||||||
|
*/
|
||||||
|
export function stopScheduler(): void {
|
||||||
|
if (timer) {
|
||||||
|
clearTimeout(timer);
|
||||||
|
timer = null;
|
||||||
|
}
|
||||||
|
log.info("Scheduler stopped");
|
||||||
|
}
|
||||||
120
worker/src/tdlib/client.ts
Normal file
120
worker/src/tdlib/client.ts
Normal file
@@ -0,0 +1,120 @@
|
|||||||
|
import tdl, { createClient, type Client } from "tdl";
|
||||||
|
import { getTdjson } from "prebuilt-tdlib";
|
||||||
|
import path from "path";
|
||||||
|
import { config } from "../util/config.js";
|
||||||
|
import { childLogger } from "../util/logger.js";
|
||||||
|
import {
|
||||||
|
updateAccountAuthState,
|
||||||
|
getAccountAuthCode,
|
||||||
|
} from "../db/queries.js";
|
||||||
|
|
||||||
|
const log = childLogger("tdlib-client");
|
||||||
|
|
||||||
|
// Configure tdl to use the prebuilt tdjson shared library
|
||||||
|
tdl.configure({ tdjson: getTdjson() });
|
||||||
|
|
||||||
|
interface AccountConfig {
|
||||||
|
id: string;
|
||||||
|
phone: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create and authenticate a TDLib client for a Telegram account.
|
||||||
|
* Authentication flow communicates with the admin UI via the database:
|
||||||
|
* - Worker sets authState to AWAITING_CODE when TDLib asks for phone code
|
||||||
|
* - Admin enters the code via UI, which writes it to authCode field
|
||||||
|
* - Worker polls DB for the code and feeds it to TDLib
|
||||||
|
*/
|
||||||
|
export async function createTdlibClient(
|
||||||
|
account: AccountConfig
|
||||||
|
): Promise<Client> {
|
||||||
|
const dbPath = path.join(config.tdlibStateDir, account.id);
|
||||||
|
|
||||||
|
const client = createClient({
|
||||||
|
apiId: config.telegramApiId,
|
||||||
|
apiHash: config.telegramApiHash,
|
||||||
|
databaseDirectory: dbPath,
|
||||||
|
filesDirectory: path.join(dbPath, "files"),
|
||||||
|
});
|
||||||
|
|
||||||
|
client.on("error", (err) => {
|
||||||
|
log.error({ err, accountId: account.id }, "TDLib client error");
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
await client.login(() => ({
|
||||||
|
getPhoneNumber: async () => {
|
||||||
|
log.info({ accountId: account.id }, "TDLib requesting phone number");
|
||||||
|
return account.phone;
|
||||||
|
},
|
||||||
|
getAuthCode: async () => {
|
||||||
|
log.info({ accountId: account.id }, "TDLib requesting auth code");
|
||||||
|
await updateAccountAuthState(account.id, "AWAITING_CODE");
|
||||||
|
|
||||||
|
// Poll database for the code entered via admin UI
|
||||||
|
const code = await pollForAuthCode(account.id);
|
||||||
|
if (!code) {
|
||||||
|
throw new Error("Auth code not provided within timeout");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clear the code after reading
|
||||||
|
await updateAccountAuthState(account.id, "AUTHENTICATED", null);
|
||||||
|
return code;
|
||||||
|
},
|
||||||
|
getPassword: async () => {
|
||||||
|
log.info({ accountId: account.id }, "TDLib requesting 2FA password");
|
||||||
|
await updateAccountAuthState(account.id, "AWAITING_PASSWORD");
|
||||||
|
|
||||||
|
// Poll database for the password entered via admin UI
|
||||||
|
const code = await pollForAuthCode(account.id);
|
||||||
|
if (!code) {
|
||||||
|
throw new Error("2FA password not provided within timeout");
|
||||||
|
}
|
||||||
|
|
||||||
|
await updateAccountAuthState(account.id, "AUTHENTICATED", null);
|
||||||
|
return code;
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
|
||||||
|
await updateAccountAuthState(account.id, "AUTHENTICATED");
|
||||||
|
log.info({ accountId: account.id }, "TDLib client authenticated");
|
||||||
|
return client;
|
||||||
|
} catch (err) {
|
||||||
|
log.error({ err, accountId: account.id }, "TDLib authentication failed");
|
||||||
|
await updateAccountAuthState(account.id, "EXPIRED");
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Poll the database every 5 seconds for an auth code, up to 5 minutes.
|
||||||
|
*/
|
||||||
|
async function pollForAuthCode(
|
||||||
|
accountId: string,
|
||||||
|
timeoutMs = 300_000
|
||||||
|
): Promise<string | null> {
|
||||||
|
const start = Date.now();
|
||||||
|
while (Date.now() - start < timeoutMs) {
|
||||||
|
const result = await getAccountAuthCode(accountId);
|
||||||
|
if (result?.authCode) {
|
||||||
|
return result.authCode;
|
||||||
|
}
|
||||||
|
await sleep(5000);
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function sleep(ms: number): Promise<void> {
|
||||||
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Close a TDLib client gracefully.
|
||||||
|
*/
|
||||||
|
export async function closeTdlibClient(client: Client): Promise<void> {
|
||||||
|
try {
|
||||||
|
await client.close();
|
||||||
|
} catch (err) {
|
||||||
|
log.warn({ err }, "Error closing TDLib client");
|
||||||
|
}
|
||||||
|
}
|
||||||
389
worker/src/tdlib/download.ts
Normal file
389
worker/src/tdlib/download.ts
Normal file
@@ -0,0 +1,389 @@
|
|||||||
|
import type { Client } from "tdl";
|
||||||
|
import { readFile, rename, stat } from "fs/promises";
|
||||||
|
import { config } from "../util/config.js";
|
||||||
|
import { childLogger } from "../util/logger.js";
|
||||||
|
import { isArchiveAttachment } from "../archive/detect.js";
|
||||||
|
import type { TelegramMessage } from "../archive/multipart.js";
|
||||||
|
import type { TelegramPhoto } from "../preview/match.js";
|
||||||
|
|
||||||
|
const log = childLogger("download");
|
||||||
|
|
||||||
|
interface TdPhotoSize {
|
||||||
|
type: string;
|
||||||
|
photo: {
|
||||||
|
id: number;
|
||||||
|
size: number;
|
||||||
|
expected_size: number;
|
||||||
|
local?: {
|
||||||
|
path?: string;
|
||||||
|
is_downloading_active?: boolean;
|
||||||
|
is_downloading_completed?: boolean;
|
||||||
|
downloaded_size?: number;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
width: number;
|
||||||
|
height: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface TdMessage {
|
||||||
|
id: number;
|
||||||
|
date: number;
|
||||||
|
content: {
|
||||||
|
_: string;
|
||||||
|
document?: {
|
||||||
|
file_name?: string;
|
||||||
|
document?: {
|
||||||
|
id: number;
|
||||||
|
size: number;
|
||||||
|
local?: {
|
||||||
|
path?: string;
|
||||||
|
is_downloading_completed?: boolean;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
photo?: {
|
||||||
|
sizes?: TdPhotoSize[];
|
||||||
|
};
|
||||||
|
caption?: {
|
||||||
|
text?: string;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
interface TdFile {
|
||||||
|
id: number;
|
||||||
|
size: number;
|
||||||
|
expected_size: number;
|
||||||
|
local: {
|
||||||
|
path: string;
|
||||||
|
is_downloading_active: boolean;
|
||||||
|
is_downloading_completed: boolean;
|
||||||
|
downloaded_size: number;
|
||||||
|
download_offset: number;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ChannelScanResult {
|
||||||
|
archives: TelegramMessage[];
|
||||||
|
photos: TelegramPhoto[];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Fetch messages from a channel since a given message ID.
|
||||||
|
* Collects both archive attachments AND photo messages (for preview matching).
|
||||||
|
* Returns messages in chronological order (oldest first).
|
||||||
|
*/
|
||||||
|
export async function getChannelMessages(
|
||||||
|
client: Client,
|
||||||
|
chatId: bigint,
|
||||||
|
fromMessageId?: bigint | null,
|
||||||
|
limit = 100
|
||||||
|
): Promise<ChannelScanResult> {
|
||||||
|
const archives: TelegramMessage[] = [];
|
||||||
|
const photos: TelegramPhoto[] = [];
|
||||||
|
let currentFromId = fromMessageId ? Number(fromMessageId) : 0;
|
||||||
|
|
||||||
|
// eslint-disable-next-line no-constant-condition
|
||||||
|
while (true) {
|
||||||
|
const result = (await client.invoke({
|
||||||
|
_: "getChatHistory",
|
||||||
|
chat_id: Number(chatId),
|
||||||
|
from_message_id: currentFromId,
|
||||||
|
offset: 0,
|
||||||
|
limit: Math.min(limit, 100),
|
||||||
|
only_local: false,
|
||||||
|
})) as { messages: TdMessage[] };
|
||||||
|
|
||||||
|
if (!result.messages || result.messages.length === 0) break;
|
||||||
|
|
||||||
|
for (const msg of result.messages) {
|
||||||
|
// Check for archive documents
|
||||||
|
const doc = msg.content?.document;
|
||||||
|
if (doc?.file_name && doc.document && isArchiveAttachment(doc.file_name)) {
|
||||||
|
archives.push({
|
||||||
|
id: BigInt(msg.id),
|
||||||
|
fileName: doc.file_name,
|
||||||
|
fileId: String(doc.document.id),
|
||||||
|
fileSize: BigInt(doc.document.size),
|
||||||
|
date: new Date(msg.date * 1000),
|
||||||
|
});
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check for photo messages (potential previews)
|
||||||
|
const photo = msg.content?.photo;
|
||||||
|
const caption = msg.content?.caption?.text ?? "";
|
||||||
|
if (photo?.sizes && photo.sizes.length > 0) {
|
||||||
|
// Pick the smallest size for thumbnail (type "s" or "m")
|
||||||
|
// TDLib photo sizes are ordered from smallest to largest
|
||||||
|
const smallest = photo.sizes[0];
|
||||||
|
photos.push({
|
||||||
|
id: BigInt(msg.id),
|
||||||
|
date: new Date(msg.date * 1000),
|
||||||
|
caption,
|
||||||
|
fileId: String(smallest.photo.id),
|
||||||
|
fileSize: smallest.photo.size || smallest.photo.expected_size,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
currentFromId = result.messages[result.messages.length - 1].id;
|
||||||
|
if (result.messages.length < 100) break;
|
||||||
|
|
||||||
|
// Rate limit delay
|
||||||
|
await sleep(config.apiDelayMs);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Return in chronological order (oldest first)
|
||||||
|
return {
|
||||||
|
archives: archives.reverse(),
|
||||||
|
photos: photos.reverse(),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Download a photo thumbnail from Telegram and return its raw bytes.
|
||||||
|
* Uses synchronous download (photos are small, typically < 100KB).
|
||||||
|
* Returns null if download fails (non-critical).
|
||||||
|
*/
|
||||||
|
export async function downloadPhotoThumbnail(
|
||||||
|
client: Client,
|
||||||
|
fileId: string
|
||||||
|
): Promise<Buffer | null> {
|
||||||
|
const numericId = parseInt(fileId, 10);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const result = (await client.invoke({
|
||||||
|
_: "downloadFile",
|
||||||
|
file_id: numericId,
|
||||||
|
priority: 1, // Low priority — thumbnails are nice-to-have
|
||||||
|
offset: 0,
|
||||||
|
limit: 0,
|
||||||
|
synchronous: true, // Small file — wait for it
|
||||||
|
})) as TdFile;
|
||||||
|
|
||||||
|
if (result?.local?.is_downloading_completed && result.local.path) {
|
||||||
|
const data = await readFile(result.local.path);
|
||||||
|
log.debug(
|
||||||
|
{ fileId, bytes: data.length },
|
||||||
|
"Downloaded photo thumbnail"
|
||||||
|
);
|
||||||
|
return data;
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
log.warn({ fileId, err }, "Failed to download photo thumbnail");
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DownloadProgress {
|
||||||
|
fileId: string;
|
||||||
|
fileName: string;
|
||||||
|
downloadedBytes: number;
|
||||||
|
totalBytes: number;
|
||||||
|
percent: number;
|
||||||
|
isComplete: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type ProgressCallback = (progress: DownloadProgress) => void;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Download a file from Telegram to a local path with progress tracking
|
||||||
|
* and integrity verification.
|
||||||
|
*
|
||||||
|
* Progress flow:
|
||||||
|
* 1. Starts async download via TDLib
|
||||||
|
* 2. Listens for `updateFile` events to track download progress
|
||||||
|
* 3. Logs progress at every 10% increment
|
||||||
|
* 4. Once complete, verifies the local file size matches the expected size
|
||||||
|
* 5. Moves the file from TDLib's cache to the destination path
|
||||||
|
*
|
||||||
|
* Verification:
|
||||||
|
* - Compares actual file size on disk to the expected size from Telegram
|
||||||
|
* - Throws on mismatch (partial/corrupt download)
|
||||||
|
* - Throws on timeout (configurable, scales with file size)
|
||||||
|
* - Throws if download stops without completing (network error, etc.)
|
||||||
|
*/
|
||||||
|
export async function downloadFile(
|
||||||
|
client: Client,
|
||||||
|
fileId: string,
|
||||||
|
destPath: string,
|
||||||
|
expectedSize: bigint,
|
||||||
|
fileName: string,
|
||||||
|
onProgress?: ProgressCallback
|
||||||
|
): Promise<void> {
|
||||||
|
const numericId = parseInt(fileId, 10);
|
||||||
|
const totalBytes = Number(expectedSize);
|
||||||
|
|
||||||
|
log.info(
|
||||||
|
{ fileId, fileName, destPath, totalBytes },
|
||||||
|
"Starting file download"
|
||||||
|
);
|
||||||
|
|
||||||
|
// Report initial progress
|
||||||
|
onProgress?.({
|
||||||
|
fileId,
|
||||||
|
fileName,
|
||||||
|
downloadedBytes: 0,
|
||||||
|
totalBytes,
|
||||||
|
percent: 0,
|
||||||
|
isComplete: false,
|
||||||
|
});
|
||||||
|
|
||||||
|
return new Promise<void>((resolve, reject) => {
|
||||||
|
let lastLoggedPercent = 0;
|
||||||
|
let settled = false;
|
||||||
|
|
||||||
|
// Timeout: 10 minutes per GB, minimum 5 minutes
|
||||||
|
const timeoutMs = Math.max(
|
||||||
|
5 * 60_000,
|
||||||
|
(totalBytes / (1024 * 1024 * 1024)) * 10 * 60_000
|
||||||
|
);
|
||||||
|
const timer = setTimeout(() => {
|
||||||
|
if (!settled) {
|
||||||
|
settled = true;
|
||||||
|
cleanup();
|
||||||
|
reject(
|
||||||
|
new Error(
|
||||||
|
`Download timed out after ${Math.round(timeoutMs / 60_000)}min for ${fileName}`
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}, timeoutMs);
|
||||||
|
|
||||||
|
// Listen for file update events to track progress
|
||||||
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||||
|
const handleUpdate = (update: any) => {
|
||||||
|
if (update?._ !== "updateFile") return;
|
||||||
|
const file = update.file as TdFile | undefined;
|
||||||
|
if (!file || file.id !== numericId) return;
|
||||||
|
|
||||||
|
const downloaded = file.local.downloaded_size;
|
||||||
|
const percent =
|
||||||
|
totalBytes > 0 ? Math.round((downloaded / totalBytes) * 100) : 0;
|
||||||
|
|
||||||
|
// Log at every 10% increment
|
||||||
|
if (percent >= lastLoggedPercent + 10) {
|
||||||
|
lastLoggedPercent = percent - (percent % 10);
|
||||||
|
log.info(
|
||||||
|
{ fileId, fileName, downloaded, totalBytes, percent: `${percent}%` },
|
||||||
|
"Download progress"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Report to callback
|
||||||
|
onProgress?.({
|
||||||
|
fileId,
|
||||||
|
fileName,
|
||||||
|
downloadedBytes: downloaded,
|
||||||
|
totalBytes,
|
||||||
|
percent,
|
||||||
|
isComplete: file.local.is_downloading_completed,
|
||||||
|
});
|
||||||
|
|
||||||
|
// Download finished
|
||||||
|
if (file.local.is_downloading_completed) {
|
||||||
|
if (!settled) {
|
||||||
|
settled = true;
|
||||||
|
cleanup();
|
||||||
|
verifyAndMove(file.local.path, destPath, totalBytes, fileName, fileId)
|
||||||
|
.then(resolve)
|
||||||
|
.catch(reject);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Download stopped without completing (network error, cancelled, etc.)
|
||||||
|
if (
|
||||||
|
!file.local.is_downloading_active &&
|
||||||
|
!file.local.is_downloading_completed
|
||||||
|
) {
|
||||||
|
if (!settled) {
|
||||||
|
settled = true;
|
||||||
|
cleanup();
|
||||||
|
reject(
|
||||||
|
new Error(
|
||||||
|
`Download stopped unexpectedly for ${fileName} ` +
|
||||||
|
`(${downloaded}/${totalBytes} bytes, ${percent}%)`
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const cleanup = () => {
|
||||||
|
clearTimeout(timer);
|
||||||
|
client.off("update", handleUpdate);
|
||||||
|
};
|
||||||
|
|
||||||
|
// Subscribe to updates BEFORE starting download
|
||||||
|
client.on("update", handleUpdate);
|
||||||
|
|
||||||
|
// Start async download (non-blocking — progress via updateFile events)
|
||||||
|
client
|
||||||
|
.invoke({
|
||||||
|
_: "downloadFile",
|
||||||
|
file_id: numericId,
|
||||||
|
priority: 32,
|
||||||
|
offset: 0,
|
||||||
|
limit: 0,
|
||||||
|
synchronous: false,
|
||||||
|
})
|
||||||
|
.then((result: unknown) => {
|
||||||
|
// If the file was already cached locally, invoke returns immediately
|
||||||
|
const file = result as TdFile | undefined;
|
||||||
|
if (file?.local?.is_downloading_completed && !settled) {
|
||||||
|
settled = true;
|
||||||
|
cleanup();
|
||||||
|
verifyAndMove(file.local.path, destPath, totalBytes, fileName, fileId)
|
||||||
|
.then(resolve)
|
||||||
|
.catch(reject);
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.catch((err: unknown) => {
|
||||||
|
if (!settled) {
|
||||||
|
settled = true;
|
||||||
|
cleanup();
|
||||||
|
reject(err);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verify the downloaded file's size matches the expected size,
|
||||||
|
* then move it to the destination path.
|
||||||
|
*/
|
||||||
|
async function verifyAndMove(
|
||||||
|
localPath: string,
|
||||||
|
destPath: string,
|
||||||
|
expectedBytes: number,
|
||||||
|
fileName: string,
|
||||||
|
fileId: string
|
||||||
|
): Promise<void> {
|
||||||
|
const stats = await stat(localPath);
|
||||||
|
const actualBytes = stats.size;
|
||||||
|
|
||||||
|
if (expectedBytes > 0 && actualBytes !== expectedBytes) {
|
||||||
|
log.error(
|
||||||
|
{ fileId, fileName, expectedBytes, actualBytes },
|
||||||
|
"Download size mismatch — file is incomplete or corrupted"
|
||||||
|
);
|
||||||
|
throw new Error(
|
||||||
|
`Download verification failed for ${fileName}: ` +
|
||||||
|
`expected ${expectedBytes} bytes, got ${actualBytes} bytes`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
log.info(
|
||||||
|
{ fileId, fileName, bytes: actualBytes, destPath },
|
||||||
|
"File verified and complete"
|
||||||
|
);
|
||||||
|
|
||||||
|
// Move from TDLib's cache to our temp directory
|
||||||
|
await rename(localPath, destPath);
|
||||||
|
}
|
||||||
|
|
||||||
|
function sleep(ms: number): Promise<void> {
|
||||||
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
|
}
|
||||||
222
worker/src/tdlib/topics.ts
Normal file
222
worker/src/tdlib/topics.ts
Normal file
@@ -0,0 +1,222 @@
|
|||||||
|
import type { Client } from "tdl";
|
||||||
|
import { config } from "../util/config.js";
|
||||||
|
import { childLogger } from "../util/logger.js";
|
||||||
|
import { isArchiveAttachment } from "../archive/detect.js";
|
||||||
|
import type { TelegramMessage } from "../archive/multipart.js";
|
||||||
|
import type { TelegramPhoto } from "../preview/match.js";
|
||||||
|
import type { ChannelScanResult } from "./download.js";
|
||||||
|
|
||||||
|
const log = childLogger("topics");
|
||||||
|
|
||||||
|
export interface ForumTopic {
|
||||||
|
topicId: bigint;
|
||||||
|
name: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if a chat is a forum supergroup (topics enabled).
|
||||||
|
*/
|
||||||
|
export async function isChatForum(
|
||||||
|
client: Client,
|
||||||
|
chatId: bigint
|
||||||
|
): Promise<boolean> {
|
||||||
|
try {
|
||||||
|
const chat = (await client.invoke({
|
||||||
|
_: "getChat",
|
||||||
|
chat_id: Number(chatId),
|
||||||
|
})) as {
|
||||||
|
type?: {
|
||||||
|
_: string;
|
||||||
|
supergroup_id?: number;
|
||||||
|
is_forum?: boolean;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
if (chat.type?._ === "chatTypeSupergroup" && chat.type.is_forum) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Also check via getSupergroup for older TDLib versions
|
||||||
|
if (chat.type?._ === "chatTypeSupergroup" && chat.type.supergroup_id) {
|
||||||
|
const sg = (await client.invoke({
|
||||||
|
_: "getSupergroup",
|
||||||
|
supergroup_id: chat.type.supergroup_id,
|
||||||
|
})) as { is_forum?: boolean };
|
||||||
|
return sg.is_forum === true;
|
||||||
|
}
|
||||||
|
|
||||||
|
return false;
|
||||||
|
} catch (err) {
|
||||||
|
log.warn({ err, chatId: chatId.toString() }, "Failed to check if chat is forum");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get all forum topics in a supergroup.
|
||||||
|
*/
|
||||||
|
export async function getForumTopicList(
|
||||||
|
client: Client,
|
||||||
|
chatId: bigint
|
||||||
|
): Promise<ForumTopic[]> {
|
||||||
|
const topics: ForumTopic[] = [];
|
||||||
|
let offsetDate = 0;
|
||||||
|
let offsetMessageId = 0;
|
||||||
|
let offsetMessageThreadId = 0;
|
||||||
|
|
||||||
|
// eslint-disable-next-line no-constant-condition
|
||||||
|
while (true) {
|
||||||
|
const result = (await client.invoke({
|
||||||
|
_: "getForumTopics",
|
||||||
|
chat_id: Number(chatId),
|
||||||
|
query: "",
|
||||||
|
offset_date: offsetDate,
|
||||||
|
offset_message_id: offsetMessageId,
|
||||||
|
offset_message_thread_id: offsetMessageThreadId,
|
||||||
|
limit: 100,
|
||||||
|
})) as {
|
||||||
|
topics?: {
|
||||||
|
info?: {
|
||||||
|
message_thread_id?: number;
|
||||||
|
name?: string;
|
||||||
|
is_general?: boolean;
|
||||||
|
};
|
||||||
|
}[];
|
||||||
|
next_offset_date?: number;
|
||||||
|
next_offset_message_id?: number;
|
||||||
|
next_offset_message_thread_id?: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
if (!result.topics || result.topics.length === 0) break;
|
||||||
|
|
||||||
|
for (const t of result.topics) {
|
||||||
|
if (!t.info?.message_thread_id) continue;
|
||||||
|
// Skip the "General" topic — it's not creator-specific
|
||||||
|
if (t.info.is_general) continue;
|
||||||
|
|
||||||
|
topics.push({
|
||||||
|
topicId: BigInt(t.info.message_thread_id),
|
||||||
|
name: t.info.name ?? "Unnamed",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if there are more pages
|
||||||
|
if (
|
||||||
|
!result.next_offset_date &&
|
||||||
|
!result.next_offset_message_id &&
|
||||||
|
!result.next_offset_message_thread_id
|
||||||
|
) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
offsetDate = result.next_offset_date ?? 0;
|
||||||
|
offsetMessageId = result.next_offset_message_id ?? 0;
|
||||||
|
offsetMessageThreadId = result.next_offset_message_thread_id ?? 0;
|
||||||
|
|
||||||
|
await sleep(config.apiDelayMs);
|
||||||
|
}
|
||||||
|
|
||||||
|
log.info(
|
||||||
|
{ chatId: chatId.toString(), topicCount: topics.length },
|
||||||
|
"Enumerated forum topics"
|
||||||
|
);
|
||||||
|
|
||||||
|
return topics;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Fetch messages from a specific forum topic (thread).
|
||||||
|
* Uses getMessageThreadHistory to scan within a topic.
|
||||||
|
*/
|
||||||
|
export async function getTopicMessages(
|
||||||
|
client: Client,
|
||||||
|
chatId: bigint,
|
||||||
|
topicId: bigint,
|
||||||
|
fromMessageId?: bigint | null,
|
||||||
|
limit = 100
|
||||||
|
): Promise<ChannelScanResult> {
|
||||||
|
const archives: TelegramMessage[] = [];
|
||||||
|
const photos: TelegramPhoto[] = [];
|
||||||
|
let currentFromId = fromMessageId ? Number(fromMessageId) : 0;
|
||||||
|
|
||||||
|
// eslint-disable-next-line no-constant-condition
|
||||||
|
while (true) {
|
||||||
|
const result = (await client.invoke({
|
||||||
|
_: "getMessageThreadHistory",
|
||||||
|
chat_id: Number(chatId),
|
||||||
|
message_id: Number(topicId),
|
||||||
|
from_message_id: currentFromId,
|
||||||
|
offset: 0,
|
||||||
|
limit: Math.min(limit, 100),
|
||||||
|
})) as {
|
||||||
|
messages?: {
|
||||||
|
id: number;
|
||||||
|
date: number;
|
||||||
|
content: {
|
||||||
|
_: string;
|
||||||
|
document?: {
|
||||||
|
file_name?: string;
|
||||||
|
document?: {
|
||||||
|
id: number;
|
||||||
|
size: number;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
photo?: {
|
||||||
|
sizes?: {
|
||||||
|
type: string;
|
||||||
|
photo: { id: number; size: number; expected_size: number };
|
||||||
|
width: number;
|
||||||
|
height: number;
|
||||||
|
}[];
|
||||||
|
};
|
||||||
|
caption?: { text?: string };
|
||||||
|
};
|
||||||
|
}[];
|
||||||
|
};
|
||||||
|
|
||||||
|
if (!result.messages || result.messages.length === 0) break;
|
||||||
|
|
||||||
|
for (const msg of result.messages) {
|
||||||
|
// Check for archive documents
|
||||||
|
const doc = msg.content?.document;
|
||||||
|
if (doc?.file_name && doc.document && isArchiveAttachment(doc.file_name)) {
|
||||||
|
archives.push({
|
||||||
|
id: BigInt(msg.id),
|
||||||
|
fileName: doc.file_name,
|
||||||
|
fileId: String(doc.document.id),
|
||||||
|
fileSize: BigInt(doc.document.size),
|
||||||
|
date: new Date(msg.date * 1000),
|
||||||
|
});
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check for photo messages (potential previews)
|
||||||
|
const photo = msg.content?.photo;
|
||||||
|
const caption = msg.content?.caption?.text ?? "";
|
||||||
|
if (photo?.sizes && photo.sizes.length > 0) {
|
||||||
|
const smallest = photo.sizes[0];
|
||||||
|
photos.push({
|
||||||
|
id: BigInt(msg.id),
|
||||||
|
date: new Date(msg.date * 1000),
|
||||||
|
caption,
|
||||||
|
fileId: String(smallest.photo.id),
|
||||||
|
fileSize: smallest.photo.size || smallest.photo.expected_size,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
currentFromId = result.messages[result.messages.length - 1].id;
|
||||||
|
if (result.messages.length < 100) break;
|
||||||
|
|
||||||
|
await sleep(config.apiDelayMs);
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
archives: archives.reverse(),
|
||||||
|
photos: photos.reverse(),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function sleep(ms: number): Promise<void> {
|
||||||
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
|
}
|
||||||
76
worker/src/upload/channel.ts
Normal file
76
worker/src/upload/channel.ts
Normal file
@@ -0,0 +1,76 @@
|
|||||||
|
import type { Client } from "tdl";
|
||||||
|
import { config } from "../util/config.js";
|
||||||
|
import { childLogger } from "../util/logger.js";
|
||||||
|
|
||||||
|
const log = childLogger("upload");
|
||||||
|
|
||||||
|
export interface UploadResult {
|
||||||
|
messageId: bigint;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Upload one or more files to a destination Telegram channel.
|
||||||
|
* For multipart archives, each file is sent as a separate message.
|
||||||
|
* Returns the message ID of the first uploaded message.
|
||||||
|
*/
|
||||||
|
export async function uploadToChannel(
|
||||||
|
client: Client,
|
||||||
|
chatId: bigint,
|
||||||
|
filePaths: string[],
|
||||||
|
caption?: string
|
||||||
|
): Promise<UploadResult> {
|
||||||
|
let firstMessageId: bigint | null = null;
|
||||||
|
|
||||||
|
for (let i = 0; i < filePaths.length; i++) {
|
||||||
|
const filePath = filePaths[i];
|
||||||
|
const fileCaption =
|
||||||
|
i === 0 && caption ? caption : undefined;
|
||||||
|
|
||||||
|
log.debug(
|
||||||
|
{ chatId: Number(chatId), filePath, part: i + 1, total: filePaths.length },
|
||||||
|
"Uploading file to channel"
|
||||||
|
);
|
||||||
|
|
||||||
|
const result = (await client.invoke({
|
||||||
|
_: "sendMessage",
|
||||||
|
chat_id: Number(chatId),
|
||||||
|
input_message_content: {
|
||||||
|
_: "inputMessageDocument",
|
||||||
|
document: {
|
||||||
|
_: "inputFileLocal",
|
||||||
|
path: filePath,
|
||||||
|
},
|
||||||
|
caption: fileCaption
|
||||||
|
? {
|
||||||
|
_: "formattedText",
|
||||||
|
text: fileCaption,
|
||||||
|
}
|
||||||
|
: undefined,
|
||||||
|
},
|
||||||
|
})) as { id: number };
|
||||||
|
|
||||||
|
if (i === 0) {
|
||||||
|
firstMessageId = BigInt(result.id);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Rate limit delay between uploads
|
||||||
|
if (i < filePaths.length - 1) {
|
||||||
|
await sleep(config.apiDelayMs);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (firstMessageId === null) {
|
||||||
|
throw new Error("Upload failed: no messages sent");
|
||||||
|
}
|
||||||
|
|
||||||
|
log.info(
|
||||||
|
{ chatId: Number(chatId), messageId: Number(firstMessageId), files: filePaths.length },
|
||||||
|
"Upload complete"
|
||||||
|
);
|
||||||
|
|
||||||
|
return { messageId: firstMessageId };
|
||||||
|
}
|
||||||
|
|
||||||
|
function sleep(ms: number): Promise<void> {
|
||||||
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
|
}
|
||||||
18
worker/src/util/config.ts
Normal file
18
worker/src/util/config.ts
Normal file
@@ -0,0 +1,18 @@
|
|||||||
|
export const config = {
|
||||||
|
databaseUrl: process.env.DATABASE_URL ?? "",
|
||||||
|
workerIntervalMinutes: parseInt(process.env.WORKER_INTERVAL_MINUTES ?? "60", 10),
|
||||||
|
tempDir: process.env.WORKER_TEMP_DIR ?? "/tmp/zips",
|
||||||
|
tdlibStateDir: process.env.TDLIB_STATE_DIR ?? "/data/tdlib",
|
||||||
|
maxZipSizeMB: parseInt(process.env.WORKER_MAX_ZIP_SIZE_MB ?? "4096", 10),
|
||||||
|
logLevel: (process.env.LOG_LEVEL ?? "info") as "debug" | "info" | "warn" | "error",
|
||||||
|
telegramApiId: parseInt(process.env.TELEGRAM_API_ID ?? "0", 10),
|
||||||
|
telegramApiHash: process.env.TELEGRAM_API_HASH ?? "",
|
||||||
|
/** Maximum jitter added to scheduler interval (in minutes) */
|
||||||
|
jitterMinutes: 5,
|
||||||
|
/** Maximum time between multipart archive parts (in hours) */
|
||||||
|
multipartTimeoutHours: 24,
|
||||||
|
/** Delay between Telegram API calls (in ms) to avoid rate limits */
|
||||||
|
apiDelayMs: 1000,
|
||||||
|
/** Max retries for rate-limited requests */
|
||||||
|
maxRetries: 5,
|
||||||
|
} as const;
|
||||||
14
worker/src/util/logger.ts
Normal file
14
worker/src/util/logger.ts
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
import pino from "pino";
|
||||||
|
import { config } from "./config.js";
|
||||||
|
|
||||||
|
export const logger = pino({
|
||||||
|
level: config.logLevel,
|
||||||
|
transport:
|
||||||
|
config.logLevel === "debug"
|
||||||
|
? { target: "pino/file", options: { destination: 1 } }
|
||||||
|
: undefined,
|
||||||
|
});
|
||||||
|
|
||||||
|
export function childLogger(name: string, extra?: Record<string, unknown>) {
|
||||||
|
return logger.child({ module: name, ...extra });
|
||||||
|
}
|
||||||
665
worker/src/worker.ts
Normal file
665
worker/src/worker.ts
Normal file
@@ -0,0 +1,665 @@
|
|||||||
|
import path from "path";
|
||||||
|
import { unlink, readdir } from "fs/promises";
|
||||||
|
import { config } from "./util/config.js";
|
||||||
|
import { childLogger } from "./util/logger.js";
|
||||||
|
import { tryAcquireLock, releaseLock } from "./db/locks.js";
|
||||||
|
import {
|
||||||
|
getSourceChannelMappings,
|
||||||
|
getDestinationChannel,
|
||||||
|
packageExistsByHash,
|
||||||
|
createPackageWithFiles,
|
||||||
|
createIngestionRun,
|
||||||
|
completeIngestionRun,
|
||||||
|
failIngestionRun,
|
||||||
|
updateLastProcessedMessage,
|
||||||
|
updateRunActivity,
|
||||||
|
setChannelForum,
|
||||||
|
getTopicProgress,
|
||||||
|
upsertTopicProgress,
|
||||||
|
} from "./db/queries.js";
|
||||||
|
import type { ActivityUpdate } from "./db/queries.js";
|
||||||
|
import { createTdlibClient, closeTdlibClient } from "./tdlib/client.js";
|
||||||
|
import { getChannelMessages, downloadFile, downloadPhotoThumbnail } from "./tdlib/download.js";
|
||||||
|
import type { DownloadProgress, ChannelScanResult } from "./tdlib/download.js";
|
||||||
|
import { isChatForum, getForumTopicList, getTopicMessages } from "./tdlib/topics.js";
|
||||||
|
import { matchPreviewToArchive } from "./preview/match.js";
|
||||||
|
import { groupArchiveSets } from "./archive/multipart.js";
|
||||||
|
import type { ArchiveSet } from "./archive/multipart.js";
|
||||||
|
import { extractCreatorFromFileName } from "./archive/creator.js";
|
||||||
|
import { hashParts } from "./archive/hash.js";
|
||||||
|
import { readZipCentralDirectory } from "./archive/zip-reader.js";
|
||||||
|
import { readRarContents } from "./archive/rar-reader.js";
|
||||||
|
import { byteLevelSplit } from "./archive/split.js";
|
||||||
|
import { uploadToChannel } from "./upload/channel.js";
|
||||||
|
import type { TelegramAccount, TelegramChannel } from "@prisma/client";
|
||||||
|
import type { Client } from "tdl";
|
||||||
|
|
||||||
|
const log = childLogger("worker");
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Throttle DB writes for download progress to avoid hammering the DB.
|
||||||
|
* Only writes if at least 2 seconds have passed since the last write.
|
||||||
|
*/
|
||||||
|
function createThrottledActivityUpdater(runId: string, minIntervalMs = 2000) {
|
||||||
|
let lastWriteTime = 0;
|
||||||
|
let pendingUpdate: ActivityUpdate | null = null;
|
||||||
|
let flushTimer: ReturnType<typeof setTimeout> | null = null;
|
||||||
|
|
||||||
|
const flush = async () => {
|
||||||
|
if (pendingUpdate) {
|
||||||
|
const update = pendingUpdate;
|
||||||
|
pendingUpdate = null;
|
||||||
|
lastWriteTime = Date.now();
|
||||||
|
await updateRunActivity(runId, update).catch(() => {});
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
return {
|
||||||
|
update: (activity: ActivityUpdate) => {
|
||||||
|
pendingUpdate = activity;
|
||||||
|
const elapsed = Date.now() - lastWriteTime;
|
||||||
|
if (elapsed >= minIntervalMs) {
|
||||||
|
if (flushTimer) clearTimeout(flushTimer);
|
||||||
|
flush();
|
||||||
|
} else if (!flushTimer) {
|
||||||
|
flushTimer = setTimeout(() => {
|
||||||
|
flushTimer = null;
|
||||||
|
flush();
|
||||||
|
}, minIntervalMs - elapsed);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
flush,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Shared context passed to the archive processing pipeline. */
|
||||||
|
interface PipelineContext {
|
||||||
|
client: Client;
|
||||||
|
runId: string;
|
||||||
|
channelTitle: string;
|
||||||
|
channel: TelegramChannel;
|
||||||
|
destChannelTelegramId: bigint;
|
||||||
|
destChannelId: string;
|
||||||
|
throttled: ReturnType<typeof createThrottledActivityUpdater>;
|
||||||
|
counters: {
|
||||||
|
messagesScanned: number;
|
||||||
|
zipsFound: number;
|
||||||
|
zipsDuplicate: number;
|
||||||
|
zipsIngested: number;
|
||||||
|
};
|
||||||
|
/** Creator from forum topic name (null for non-forum). */
|
||||||
|
topicCreator: string | null;
|
||||||
|
/** Forum topic ID (null for non-forum). */
|
||||||
|
sourceTopicId: bigint | null;
|
||||||
|
accountLog: ReturnType<typeof childLogger>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Run a full ingestion cycle for a single Telegram account.
|
||||||
|
* Every step writes live activity to the DB so the admin UI can display it.
|
||||||
|
*/
|
||||||
|
export async function runWorkerForAccount(
|
||||||
|
account: TelegramAccount
|
||||||
|
): Promise<void> {
|
||||||
|
const accountLog = childLogger("worker", { accountId: account.id, phone: account.phone });
|
||||||
|
|
||||||
|
// 1. Acquire advisory lock
|
||||||
|
const acquired = await tryAcquireLock(account.id);
|
||||||
|
if (!acquired) {
|
||||||
|
accountLog.info("Account already locked, skipping");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let runId: string | undefined;
|
||||||
|
|
||||||
|
try {
|
||||||
|
// 2. Create ingestion run
|
||||||
|
const run = await createIngestionRun(account.id);
|
||||||
|
runId = run.id;
|
||||||
|
const activeRunId = runId;
|
||||||
|
accountLog.info({ runId }, "Ingestion run started");
|
||||||
|
|
||||||
|
const throttled = createThrottledActivityUpdater(activeRunId);
|
||||||
|
|
||||||
|
// 3. Initialize TDLib client
|
||||||
|
await updateRunActivity(activeRunId, {
|
||||||
|
currentActivity: "Connecting to Telegram",
|
||||||
|
currentStep: "connecting",
|
||||||
|
});
|
||||||
|
|
||||||
|
const client = await createTdlibClient({
|
||||||
|
id: account.id,
|
||||||
|
phone: account.phone,
|
||||||
|
});
|
||||||
|
|
||||||
|
const counters = {
|
||||||
|
messagesScanned: 0,
|
||||||
|
zipsFound: 0,
|
||||||
|
zipsDuplicate: 0,
|
||||||
|
zipsIngested: 0,
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
// 4. Get assigned source channels and destination
|
||||||
|
const channelMappings = await getSourceChannelMappings(account.id);
|
||||||
|
const destChannel = await getDestinationChannel(account.id);
|
||||||
|
|
||||||
|
if (!destChannel) {
|
||||||
|
throw new Error("No active destination channel configured");
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const mapping of channelMappings) {
|
||||||
|
const channel = mapping.channel;
|
||||||
|
|
||||||
|
// ── Check if channel is a forum ──
|
||||||
|
const forum = await isChatForum(client, channel.telegramId);
|
||||||
|
if (forum !== channel.isForum) {
|
||||||
|
await setChannelForum(channel.id, forum);
|
||||||
|
accountLog.info(
|
||||||
|
{ channelId: channel.id, title: channel.title, isForum: forum },
|
||||||
|
"Updated channel forum status"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const pipelineCtx: PipelineContext = {
|
||||||
|
client,
|
||||||
|
runId: activeRunId,
|
||||||
|
channelTitle: channel.title,
|
||||||
|
channel,
|
||||||
|
destChannelTelegramId: destChannel.telegramId,
|
||||||
|
destChannelId: destChannel.id,
|
||||||
|
throttled,
|
||||||
|
counters,
|
||||||
|
topicCreator: null,
|
||||||
|
sourceTopicId: null,
|
||||||
|
accountLog,
|
||||||
|
};
|
||||||
|
|
||||||
|
if (forum) {
|
||||||
|
// ── Forum channel: scan per-topic ──
|
||||||
|
await updateRunActivity(activeRunId, {
|
||||||
|
currentActivity: `Enumerating topics in "${channel.title}"`,
|
||||||
|
currentStep: "scanning",
|
||||||
|
currentChannel: channel.title,
|
||||||
|
currentFile: null,
|
||||||
|
currentFileNum: null,
|
||||||
|
totalFiles: null,
|
||||||
|
downloadedBytes: null,
|
||||||
|
totalBytes: null,
|
||||||
|
downloadPercent: null,
|
||||||
|
});
|
||||||
|
|
||||||
|
const topics = await getForumTopicList(client, channel.telegramId);
|
||||||
|
const topicProgressList = await getTopicProgress(mapping.id);
|
||||||
|
|
||||||
|
accountLog.info(
|
||||||
|
{ channelId: channel.id, title: channel.title, topicCount: topics.length },
|
||||||
|
"Scanning forum channel by topic"
|
||||||
|
);
|
||||||
|
|
||||||
|
for (const topic of topics) {
|
||||||
|
const progress = topicProgressList.find(
|
||||||
|
(tp) => tp.topicId === topic.topicId
|
||||||
|
);
|
||||||
|
|
||||||
|
await updateRunActivity(activeRunId, {
|
||||||
|
currentActivity: `Scanning topic "${topic.name}" in "${channel.title}"`,
|
||||||
|
currentStep: "scanning",
|
||||||
|
currentChannel: `${channel.title} › ${topic.name}`,
|
||||||
|
currentFile: null,
|
||||||
|
currentFileNum: null,
|
||||||
|
totalFiles: null,
|
||||||
|
downloadedBytes: null,
|
||||||
|
totalBytes: null,
|
||||||
|
downloadPercent: null,
|
||||||
|
});
|
||||||
|
|
||||||
|
const scanResult = await getTopicMessages(
|
||||||
|
client,
|
||||||
|
channel.telegramId,
|
||||||
|
topic.topicId,
|
||||||
|
progress?.lastProcessedMessageId
|
||||||
|
);
|
||||||
|
|
||||||
|
if (scanResult.archives.length === 0) {
|
||||||
|
accountLog.debug(
|
||||||
|
{ channelId: channel.id, topic: topic.name },
|
||||||
|
"No new archives in topic"
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
accountLog.info(
|
||||||
|
{ topic: topic.name, archives: scanResult.archives.length, photos: scanResult.photos.length },
|
||||||
|
"Found messages in topic"
|
||||||
|
);
|
||||||
|
|
||||||
|
// Process archives with topic creator
|
||||||
|
pipelineCtx.topicCreator = topic.name;
|
||||||
|
pipelineCtx.sourceTopicId = topic.topicId;
|
||||||
|
pipelineCtx.channelTitle = `${channel.title} › ${topic.name}`;
|
||||||
|
|
||||||
|
await processArchiveSets(pipelineCtx, scanResult, run.id);
|
||||||
|
|
||||||
|
// Update topic progress
|
||||||
|
const allMsgIds = [
|
||||||
|
...scanResult.archives.map((m) => m.id),
|
||||||
|
...scanResult.photos.map((p) => p.id),
|
||||||
|
];
|
||||||
|
if (allMsgIds.length > 0) {
|
||||||
|
const maxId = allMsgIds.reduce((a, b) => (a > b ? a : b));
|
||||||
|
await upsertTopicProgress(
|
||||||
|
mapping.id,
|
||||||
|
topic.topicId,
|
||||||
|
topic.name,
|
||||||
|
maxId
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// ── Non-forum channel: flat scan (existing behavior) ──
|
||||||
|
await updateRunActivity(activeRunId, {
|
||||||
|
currentActivity: `Scanning "${channel.title}" for new archives`,
|
||||||
|
currentStep: "scanning",
|
||||||
|
currentChannel: channel.title,
|
||||||
|
currentFile: null,
|
||||||
|
currentFileNum: null,
|
||||||
|
totalFiles: null,
|
||||||
|
downloadedBytes: null,
|
||||||
|
totalBytes: null,
|
||||||
|
downloadPercent: null,
|
||||||
|
});
|
||||||
|
|
||||||
|
accountLog.info(
|
||||||
|
{ channelId: channel.id, title: channel.title },
|
||||||
|
"Processing source channel"
|
||||||
|
);
|
||||||
|
|
||||||
|
const scanResult = await getChannelMessages(
|
||||||
|
client,
|
||||||
|
channel.telegramId,
|
||||||
|
mapping.lastProcessedMessageId
|
||||||
|
);
|
||||||
|
|
||||||
|
if (scanResult.archives.length === 0) {
|
||||||
|
accountLog.debug({ channelId: channel.id }, "No new archives");
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
accountLog.info(
|
||||||
|
{ archives: scanResult.archives.length, photos: scanResult.photos.length },
|
||||||
|
"Found messages in channel"
|
||||||
|
);
|
||||||
|
|
||||||
|
// For non-forum, creator comes from filename (set to null, resolved per-archive)
|
||||||
|
pipelineCtx.topicCreator = null;
|
||||||
|
pipelineCtx.sourceTopicId = null;
|
||||||
|
pipelineCtx.channelTitle = channel.title;
|
||||||
|
|
||||||
|
await processArchiveSets(pipelineCtx, scanResult, run.id);
|
||||||
|
|
||||||
|
// Update last processed message
|
||||||
|
const allMsgIds = [
|
||||||
|
...scanResult.archives.map((m) => m.id),
|
||||||
|
...scanResult.photos.map((p) => p.id),
|
||||||
|
];
|
||||||
|
if (allMsgIds.length > 0) {
|
||||||
|
const maxId = allMsgIds.reduce((a, b) => (a > b ? a : b));
|
||||||
|
await updateLastProcessedMessage(mapping.id, maxId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Done ──
|
||||||
|
await completeIngestionRun(activeRunId, counters);
|
||||||
|
accountLog.info({ counters }, "Ingestion run completed");
|
||||||
|
} finally {
|
||||||
|
await closeTdlibClient(client);
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
const message = err instanceof Error ? err.message : String(err);
|
||||||
|
accountLog.error({ err }, "Ingestion run failed");
|
||||||
|
if (runId) {
|
||||||
|
await failIngestionRun(runId, message).catch((e) =>
|
||||||
|
accountLog.error({ e }, "Failed to mark run as failed")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
await releaseLock(account.id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Process a scan result through the archive pipeline:
|
||||||
|
* group → download → hash → dedup → metadata → split → upload → preview → index.
|
||||||
|
*/
|
||||||
|
async function processArchiveSets(
|
||||||
|
ctx: PipelineContext,
|
||||||
|
scanResult: ChannelScanResult,
|
||||||
|
ingestionRunId: string
|
||||||
|
): Promise<void> {
|
||||||
|
const { client, runId, channelTitle, channel, throttled, counters, accountLog } = ctx;
|
||||||
|
|
||||||
|
// Group into archive sets
|
||||||
|
const archiveSets = groupArchiveSets(scanResult.archives);
|
||||||
|
counters.zipsFound += archiveSets.length;
|
||||||
|
|
||||||
|
// Match preview photos to archive sets
|
||||||
|
const previewMatches = matchPreviewToArchive(
|
||||||
|
scanResult.photos,
|
||||||
|
archiveSets.map((s) => ({
|
||||||
|
baseName: s.baseName,
|
||||||
|
firstMessageId: s.parts[0].id,
|
||||||
|
firstMessageDate: s.parts[0].date,
|
||||||
|
}))
|
||||||
|
);
|
||||||
|
|
||||||
|
if (previewMatches.size > 0) {
|
||||||
|
accountLog.info(
|
||||||
|
{ matched: previewMatches.size, total: archiveSets.length },
|
||||||
|
"Matched preview photos to archives"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
await updateRunActivity(runId, {
|
||||||
|
currentActivity: `Found ${archiveSets.length} archive(s) in "${channelTitle}"`,
|
||||||
|
currentStep: "scanning",
|
||||||
|
currentChannel: channelTitle,
|
||||||
|
totalFiles: archiveSets.length,
|
||||||
|
zipsFound: counters.zipsFound,
|
||||||
|
});
|
||||||
|
|
||||||
|
for (let setIdx = 0; setIdx < archiveSets.length; setIdx++) {
|
||||||
|
await processOneArchiveSet(
|
||||||
|
ctx,
|
||||||
|
archiveSets[setIdx],
|
||||||
|
setIdx,
|
||||||
|
archiveSets.length,
|
||||||
|
previewMatches,
|
||||||
|
ingestionRunId
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Process a single archive set through the full pipeline.
|
||||||
|
*/
|
||||||
|
async function processOneArchiveSet(
|
||||||
|
ctx: PipelineContext,
|
||||||
|
archiveSet: ArchiveSet,
|
||||||
|
setIdx: number,
|
||||||
|
totalSets: number,
|
||||||
|
previewMatches: Map<string, { id: bigint; fileId: string }>,
|
||||||
|
ingestionRunId: string
|
||||||
|
): Promise<void> {
|
||||||
|
const {
|
||||||
|
client, runId, channelTitle, channel,
|
||||||
|
destChannelTelegramId, destChannelId,
|
||||||
|
throttled, counters, topicCreator, sourceTopicId, accountLog,
|
||||||
|
} = ctx;
|
||||||
|
|
||||||
|
counters.messagesScanned += archiveSet.parts.length;
|
||||||
|
const archiveName = archiveSet.parts[0].fileName;
|
||||||
|
const tempPaths: string[] = [];
|
||||||
|
let splitPaths: string[] = [];
|
||||||
|
|
||||||
|
try {
|
||||||
|
// ── Downloading ──
|
||||||
|
for (let partIdx = 0; partIdx < archiveSet.parts.length; partIdx++) {
|
||||||
|
const part = archiveSet.parts[partIdx];
|
||||||
|
const tempPath = path.join(
|
||||||
|
config.tempDir,
|
||||||
|
`${ingestionRunId}_${part.id}_${part.fileName}`
|
||||||
|
);
|
||||||
|
|
||||||
|
const partLabel = archiveSet.parts.length > 1
|
||||||
|
? ` (part ${partIdx + 1}/${archiveSet.parts.length})`
|
||||||
|
: "";
|
||||||
|
|
||||||
|
await updateRunActivity(runId, {
|
||||||
|
currentActivity: `Downloading ${part.fileName}${partLabel}`,
|
||||||
|
currentStep: "downloading",
|
||||||
|
currentChannel: channelTitle,
|
||||||
|
currentFile: part.fileName,
|
||||||
|
currentFileNum: setIdx + 1,
|
||||||
|
totalFiles: totalSets,
|
||||||
|
downloadedBytes: 0n,
|
||||||
|
totalBytes: part.fileSize,
|
||||||
|
downloadPercent: 0,
|
||||||
|
messagesScanned: counters.messagesScanned,
|
||||||
|
});
|
||||||
|
|
||||||
|
accountLog.info(
|
||||||
|
{
|
||||||
|
fileName: part.fileName,
|
||||||
|
fileSize: Number(part.fileSize),
|
||||||
|
part: partIdx + 1,
|
||||||
|
totalParts: archiveSet.parts.length,
|
||||||
|
},
|
||||||
|
"Downloading archive part"
|
||||||
|
);
|
||||||
|
|
||||||
|
await downloadFile(
|
||||||
|
client,
|
||||||
|
part.fileId,
|
||||||
|
tempPath,
|
||||||
|
part.fileSize,
|
||||||
|
part.fileName,
|
||||||
|
(progress: DownloadProgress) => {
|
||||||
|
throttled.update({
|
||||||
|
currentActivity: `Downloading ${part.fileName}${partLabel} — ${progress.percent}%`,
|
||||||
|
currentStep: "downloading",
|
||||||
|
currentChannel: channelTitle,
|
||||||
|
currentFile: part.fileName,
|
||||||
|
currentFileNum: setIdx + 1,
|
||||||
|
totalFiles: totalSets,
|
||||||
|
downloadedBytes: BigInt(progress.downloadedBytes),
|
||||||
|
totalBytes: BigInt(progress.totalBytes),
|
||||||
|
downloadPercent: progress.percent,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
);
|
||||||
|
await throttled.flush();
|
||||||
|
tempPaths.push(tempPath);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Hashing ──
|
||||||
|
await updateRunActivity(runId, {
|
||||||
|
currentActivity: `Computing hash for ${archiveName}`,
|
||||||
|
currentStep: "hashing",
|
||||||
|
currentChannel: channelTitle,
|
||||||
|
currentFile: archiveName,
|
||||||
|
currentFileNum: setIdx + 1,
|
||||||
|
totalFiles: totalSets,
|
||||||
|
downloadedBytes: null,
|
||||||
|
totalBytes: null,
|
||||||
|
downloadPercent: null,
|
||||||
|
});
|
||||||
|
|
||||||
|
const contentHash = await hashParts(tempPaths);
|
||||||
|
|
||||||
|
// ── Deduplicating ──
|
||||||
|
await updateRunActivity(runId, {
|
||||||
|
currentActivity: `Checking if ${archiveName} is a duplicate`,
|
||||||
|
currentStep: "deduplicating",
|
||||||
|
currentChannel: channelTitle,
|
||||||
|
currentFile: archiveName,
|
||||||
|
currentFileNum: setIdx + 1,
|
||||||
|
totalFiles: totalSets,
|
||||||
|
});
|
||||||
|
|
||||||
|
const exists = await packageExistsByHash(contentHash);
|
||||||
|
if (exists) {
|
||||||
|
counters.zipsDuplicate++;
|
||||||
|
accountLog.debug({ contentHash }, "Duplicate archive, skipping");
|
||||||
|
|
||||||
|
await updateRunActivity(runId, {
|
||||||
|
currentActivity: `Skipped ${archiveName} (duplicate)`,
|
||||||
|
currentStep: "deduplicating",
|
||||||
|
currentChannel: channelTitle,
|
||||||
|
currentFile: archiveName,
|
||||||
|
currentFileNum: setIdx + 1,
|
||||||
|
totalFiles: totalSets,
|
||||||
|
zipsDuplicate: counters.zipsDuplicate,
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Reading metadata ──
|
||||||
|
await updateRunActivity(runId, {
|
||||||
|
currentActivity: `Reading file list from ${archiveName}`,
|
||||||
|
currentStep: "reading_metadata",
|
||||||
|
currentChannel: channelTitle,
|
||||||
|
currentFile: archiveName,
|
||||||
|
currentFileNum: setIdx + 1,
|
||||||
|
totalFiles: totalSets,
|
||||||
|
});
|
||||||
|
|
||||||
|
let entries: { path: string; fileName: string; extension: string | null; compressedSize: bigint; uncompressedSize: bigint; crc32: string | null }[] = [];
|
||||||
|
try {
|
||||||
|
if (archiveSet.type === "ZIP") {
|
||||||
|
entries = await readZipCentralDirectory(tempPaths);
|
||||||
|
} else {
|
||||||
|
entries = await readRarContents(tempPaths[0]);
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
accountLog.warn({ err, baseName: archiveSet.baseName }, "Failed to read archive metadata, ingesting without file list");
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Splitting (if needed) ──
|
||||||
|
let uploadPaths = tempPaths;
|
||||||
|
const totalSize = archiveSet.parts.reduce(
|
||||||
|
(sum, p) => sum + p.fileSize,
|
||||||
|
0n
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!archiveSet.isMultipart && totalSize > 2n * 1024n * 1024n * 1024n) {
|
||||||
|
await updateRunActivity(runId, {
|
||||||
|
currentActivity: `Splitting ${archiveName} for upload (>2GB)`,
|
||||||
|
currentStep: "splitting",
|
||||||
|
currentChannel: channelTitle,
|
||||||
|
currentFile: archiveName,
|
||||||
|
currentFileNum: setIdx + 1,
|
||||||
|
totalFiles: totalSets,
|
||||||
|
});
|
||||||
|
splitPaths = await byteLevelSplit(tempPaths[0]);
|
||||||
|
uploadPaths = splitPaths;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Uploading ──
|
||||||
|
const uploadLabel = uploadPaths.length > 1
|
||||||
|
? ` (${uploadPaths.length} parts)`
|
||||||
|
: "";
|
||||||
|
await updateRunActivity(runId, {
|
||||||
|
currentActivity: `Uploading ${archiveName} to archive channel${uploadLabel}`,
|
||||||
|
currentStep: "uploading",
|
||||||
|
currentChannel: channelTitle,
|
||||||
|
currentFile: archiveName,
|
||||||
|
currentFileNum: setIdx + 1,
|
||||||
|
totalFiles: totalSets,
|
||||||
|
});
|
||||||
|
|
||||||
|
const destResult = await uploadToChannel(
|
||||||
|
client,
|
||||||
|
destChannelTelegramId,
|
||||||
|
uploadPaths
|
||||||
|
);
|
||||||
|
|
||||||
|
// ── Preview thumbnail ──
|
||||||
|
let previewData: Buffer | null = null;
|
||||||
|
let previewMsgId: bigint | null = null;
|
||||||
|
const matchedPhoto = previewMatches.get(archiveSet.baseName);
|
||||||
|
if (matchedPhoto) {
|
||||||
|
await updateRunActivity(runId, {
|
||||||
|
currentActivity: `Downloading preview image for ${archiveName}`,
|
||||||
|
currentStep: "preview",
|
||||||
|
currentChannel: channelTitle,
|
||||||
|
currentFile: archiveName,
|
||||||
|
currentFileNum: setIdx + 1,
|
||||||
|
totalFiles: totalSets,
|
||||||
|
});
|
||||||
|
previewData = await downloadPhotoThumbnail(client, matchedPhoto.fileId);
|
||||||
|
previewMsgId = matchedPhoto.id;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Resolve creator: topic name > filename extraction > null ──
|
||||||
|
const creator = topicCreator ?? extractCreatorFromFileName(archiveName) ?? null;
|
||||||
|
|
||||||
|
// ── Indexing ──
|
||||||
|
await updateRunActivity(runId, {
|
||||||
|
currentActivity: `Saving metadata for ${archiveName} (${entries.length} files)`,
|
||||||
|
currentStep: "indexing",
|
||||||
|
currentChannel: channelTitle,
|
||||||
|
currentFile: archiveName,
|
||||||
|
currentFileNum: setIdx + 1,
|
||||||
|
totalFiles: totalSets,
|
||||||
|
});
|
||||||
|
|
||||||
|
await createPackageWithFiles({
|
||||||
|
contentHash,
|
||||||
|
fileName: archiveName,
|
||||||
|
fileSize: totalSize,
|
||||||
|
archiveType: archiveSet.type,
|
||||||
|
sourceChannelId: channel.id,
|
||||||
|
sourceMessageId: archiveSet.parts[0].id,
|
||||||
|
sourceTopicId,
|
||||||
|
destChannelId,
|
||||||
|
destMessageId: destResult.messageId,
|
||||||
|
isMultipart:
|
||||||
|
archiveSet.parts.length > 1 || uploadPaths.length > 1,
|
||||||
|
partCount: uploadPaths.length,
|
||||||
|
ingestionRunId,
|
||||||
|
creator,
|
||||||
|
previewData,
|
||||||
|
previewMsgId,
|
||||||
|
files: entries,
|
||||||
|
});
|
||||||
|
|
||||||
|
counters.zipsIngested++;
|
||||||
|
|
||||||
|
await updateRunActivity(runId, {
|
||||||
|
currentActivity: `Ingested ${archiveName} (${entries.length} files indexed)`,
|
||||||
|
currentStep: "complete",
|
||||||
|
currentChannel: channelTitle,
|
||||||
|
currentFile: archiveName,
|
||||||
|
currentFileNum: setIdx + 1,
|
||||||
|
totalFiles: totalSets,
|
||||||
|
zipsIngested: counters.zipsIngested,
|
||||||
|
});
|
||||||
|
|
||||||
|
accountLog.info(
|
||||||
|
{ fileName: archiveName, contentHash, fileCount: entries.length, creator },
|
||||||
|
"Archive ingested"
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
// ALWAYS delete temp files
|
||||||
|
await deleteFiles([...tempPaths, ...splitPaths]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function deleteFiles(paths: string[]): Promise<void> {
|
||||||
|
for (const p of paths) {
|
||||||
|
try {
|
||||||
|
await unlink(p);
|
||||||
|
} catch {
|
||||||
|
// File may already be deleted or never created
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Clean up any leftover temp files from previous runs.
|
||||||
|
*/
|
||||||
|
export async function cleanupTempDir(): Promise<void> {
|
||||||
|
try {
|
||||||
|
const files = await readdir(config.tempDir);
|
||||||
|
for (const file of files) {
|
||||||
|
await unlink(path.join(config.tempDir, file)).catch(() => {});
|
||||||
|
}
|
||||||
|
if (files.length > 0) {
|
||||||
|
log.info({ count: files.length }, "Cleaned up stale temp files");
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Directory might not exist yet
|
||||||
|
}
|
||||||
|
}
|
||||||
19
worker/tsconfig.json
Normal file
19
worker/tsconfig.json
Normal file
@@ -0,0 +1,19 @@
|
|||||||
|
{
|
||||||
|
"compilerOptions": {
|
||||||
|
"target": "ES2022",
|
||||||
|
"module": "ESNext",
|
||||||
|
"moduleResolution": "bundler",
|
||||||
|
"lib": ["ES2022"],
|
||||||
|
"outDir": "dist",
|
||||||
|
"rootDir": "src",
|
||||||
|
"strict": true,
|
||||||
|
"esModuleInterop": true,
|
||||||
|
"skipLibCheck": true,
|
||||||
|
"resolveJsonModule": true,
|
||||||
|
"isolatedModules": true,
|
||||||
|
"declaration": true,
|
||||||
|
"sourceMap": true
|
||||||
|
},
|
||||||
|
"include": ["src/**/*.ts"],
|
||||||
|
"exclude": ["node_modules", "dist"]
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user