mirror of
https://github.com/xCyanGrizzly/DragonsStash.git
synced 2026-05-11 06:11:15 +00:00
feat: add Telegram integration with forum topic support and creator tracking
Adds full Telegram ZIP ingestion pipeline: TDLib worker service scans source channels for archive files, deduplicates by content hash, extracts metadata, uploads to archive channel, and indexes in Postgres. Forum supergroups are scanned per-topic with topic names used as creator. Filename-based creator extraction (e.g. "Mammoth Factory - 2026-01.zip") serves as fallback. Includes admin UI for managing accounts/channels, simplified account setup (API credentials via env vars), auth code/password submission dialog, package browser with creator column, and live ingestion activity tracking. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
xCyanGrizzly
@@ -6,6 +6,8 @@ export const NAV_ITEMS = [
|
||||
{ label: "Resins", href: "/resins", icon: "Droplets" },
|
||||
{ label: "Paints", href: "/paints", icon: "Paintbrush" },
|
||||
{ label: "Supplies", href: "/supplies", icon: "Gem" },
|
||||
{ label: "STL Files", href: "/stls", icon: "FileBox" },
|
||||
{ label: "Telegram", href: "/telegram", icon: "Send" },
|
||||
{ label: "Usage", href: "/usage", icon: "ClipboardList" },
|
||||
{ label: "Vendors", href: "/vendors", icon: "Building2" },
|
||||
{ label: "Locations", href: "/locations", icon: "MapPin" },
|
||||
|
||||
105
src/lib/telegram/admin-queries.ts
Normal file
105
src/lib/telegram/admin-queries.ts
Normal file
@@ -0,0 +1,105 @@
|
||||
import { prisma } from "@/lib/prisma";
|
||||
|
||||
// ── Account queries ──
|
||||
|
||||
export async function listAccounts() {
|
||||
const accounts = await prisma.telegramAccount.findMany({
|
||||
orderBy: { createdAt: "desc" },
|
||||
include: {
|
||||
_count: { select: { channelMaps: true, ingestionRuns: true } },
|
||||
},
|
||||
});
|
||||
|
||||
return accounts.map((a) => ({
|
||||
id: a.id,
|
||||
phone: a.phone,
|
||||
displayName: a.displayName,
|
||||
isActive: a.isActive,
|
||||
authState: a.authState,
|
||||
authCode: a.authCode,
|
||||
lastSeenAt: a.lastSeenAt?.toISOString() ?? null,
|
||||
createdAt: a.createdAt.toISOString(),
|
||||
channelCount: a._count.channelMaps,
|
||||
runCount: a._count.ingestionRuns,
|
||||
}));
|
||||
}
|
||||
|
||||
export type AccountRow = Awaited<ReturnType<typeof listAccounts>>[number];
|
||||
|
||||
// ── Channel queries ──
|
||||
|
||||
export async function listChannels() {
|
||||
const channels = await prisma.telegramChannel.findMany({
|
||||
orderBy: { createdAt: "desc" },
|
||||
include: {
|
||||
_count: { select: { accountMaps: true, packages: true } },
|
||||
},
|
||||
});
|
||||
|
||||
return channels.map((c) => ({
|
||||
id: c.id,
|
||||
telegramId: c.telegramId.toString(),
|
||||
title: c.title,
|
||||
type: c.type,
|
||||
isActive: c.isActive,
|
||||
createdAt: c.createdAt.toISOString(),
|
||||
accountCount: c._count.accountMaps,
|
||||
packageCount: c._count.packages,
|
||||
}));
|
||||
}
|
||||
|
||||
export type ChannelRow = Awaited<ReturnType<typeof listChannels>>[number];
|
||||
|
||||
// ── Account-Channel link queries ──
|
||||
|
||||
export async function listAccountChannelLinks(accountId: string) {
|
||||
const links = await prisma.accountChannelMap.findMany({
|
||||
where: { accountId },
|
||||
include: {
|
||||
channel: { select: { id: true, title: true, type: true, telegramId: true } },
|
||||
},
|
||||
orderBy: { createdAt: "desc" },
|
||||
});
|
||||
|
||||
return links.map((l) => ({
|
||||
id: l.id,
|
||||
accountId: l.accountId,
|
||||
channelId: l.channelId,
|
||||
role: l.role,
|
||||
lastProcessedMessageId: l.lastProcessedMessageId?.toString() ?? null,
|
||||
channel: {
|
||||
id: l.channel.id,
|
||||
title: l.channel.title,
|
||||
type: l.channel.type,
|
||||
telegramId: l.channel.telegramId.toString(),
|
||||
},
|
||||
}));
|
||||
}
|
||||
|
||||
export type AccountChannelLinkRow = Awaited<
|
||||
ReturnType<typeof listAccountChannelLinks>
|
||||
>[number];
|
||||
|
||||
export async function getUnlinkedChannels(accountId: string) {
|
||||
const linked = await prisma.accountChannelMap.findMany({
|
||||
where: { accountId },
|
||||
select: { channelId: true },
|
||||
});
|
||||
const linkedIds = linked.map((l) => l.channelId);
|
||||
|
||||
const unlinked = await prisma.telegramChannel.findMany({
|
||||
where: {
|
||||
id: { notIn: linkedIds },
|
||||
isActive: true,
|
||||
},
|
||||
orderBy: { title: "asc" },
|
||||
select: { id: true, title: true, type: true, telegramId: true },
|
||||
});
|
||||
|
||||
return unlinked.map((c) => ({
|
||||
id: c.id,
|
||||
title: c.title,
|
||||
type: c.type,
|
||||
telegramId: c.telegramId.toString(),
|
||||
}));
|
||||
}
|
||||
45
src/lib/telegram/api-auth.ts
Normal file
45
src/lib/telegram/api-auth.ts
Normal file
@@ -0,0 +1,45 @@
|
||||
import { auth } from "@/lib/auth";
|
||||
import { NextResponse } from "next/server";
|
||||
|
||||
/**
|
||||
* Authenticate an API request. Checks:
|
||||
* 1. X-API-Key header against TELEGRAM_API_KEY env var
|
||||
* 2. NextAuth session
|
||||
*
|
||||
* Returns null if authenticated, or a NextResponse error if not.
|
||||
*/
|
||||
export async function authenticateApiRequest(
|
||||
request: Request,
|
||||
requireAdmin = false
|
||||
): Promise<{ error: NextResponse } | { userId: string; role: string }> {
|
||||
// Check API key first
|
||||
const apiKey = request.headers.get("X-API-Key");
|
||||
const envKey = process.env.TELEGRAM_API_KEY;
|
||||
|
||||
if (apiKey && envKey && apiKey === envKey) {
|
||||
// API key auth — treated as admin
|
||||
return { userId: "api-key", role: "ADMIN" };
|
||||
}
|
||||
|
||||
// Fall back to session auth
|
||||
const session = await auth();
|
||||
if (!session?.user?.id) {
|
||||
return {
|
||||
error: NextResponse.json(
|
||||
{ error: "Unauthorized" },
|
||||
{ status: 401 }
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
if (requireAdmin && session.user.role !== "ADMIN") {
|
||||
return {
|
||||
error: NextResponse.json(
|
||||
{ error: "Forbidden: admin role required" },
|
||||
{ status: 403 }
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
return { userId: session.user.id, role: session.user.role };
|
||||
}
|
||||
314
src/lib/telegram/queries.ts
Normal file
314
src/lib/telegram/queries.ts
Normal file
@@ -0,0 +1,314 @@
|
||||
import { prisma } from "@/lib/prisma";
|
||||
import type {
|
||||
PackageListItem,
|
||||
PackageDetail,
|
||||
PackageFileItem,
|
||||
IngestionAccountStatus,
|
||||
} from "./types";
|
||||
|
||||
export async function listPackages(options: {
|
||||
page: number;
|
||||
limit: number;
|
||||
channelId?: string;
|
||||
creator?: string;
|
||||
sortBy: "indexedAt" | "fileName" | "fileSize";
|
||||
order: "asc" | "desc";
|
||||
}) {
|
||||
const where: Record<string, unknown> = {};
|
||||
if (options.channelId) where.sourceChannelId = options.channelId;
|
||||
if (options.creator) where.creator = options.creator;
|
||||
|
||||
const [items, total] = await Promise.all([
|
||||
prisma.package.findMany({
|
||||
where,
|
||||
orderBy: { [options.sortBy]: options.order },
|
||||
skip: (options.page - 1) * options.limit,
|
||||
take: options.limit,
|
||||
select: {
|
||||
id: true,
|
||||
fileName: true,
|
||||
fileSize: true,
|
||||
contentHash: true,
|
||||
archiveType: true,
|
||||
fileCount: true,
|
||||
isMultipart: true,
|
||||
indexedAt: true,
|
||||
creator: true,
|
||||
previewMsgId: true, // cheap null check — avoids loading blob
|
||||
sourceChannel: { select: { id: true, title: true } },
|
||||
},
|
||||
}),
|
||||
prisma.package.count({ where }),
|
||||
]);
|
||||
|
||||
const mapped: PackageListItem[] = items.map((pkg) => ({
|
||||
id: pkg.id,
|
||||
fileName: pkg.fileName,
|
||||
fileSize: pkg.fileSize.toString(),
|
||||
contentHash: pkg.contentHash,
|
||||
archiveType: pkg.archiveType,
|
||||
fileCount: pkg.fileCount,
|
||||
isMultipart: pkg.isMultipart,
|
||||
hasPreview: pkg.previewMsgId !== null,
|
||||
creator: pkg.creator,
|
||||
indexedAt: pkg.indexedAt.toISOString(),
|
||||
sourceChannel: pkg.sourceChannel,
|
||||
}));
|
||||
|
||||
return {
|
||||
items: mapped,
|
||||
pagination: {
|
||||
page: options.page,
|
||||
limit: options.limit,
|
||||
total,
|
||||
totalPages: Math.ceil(total / options.limit),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export async function getPackageById(
|
||||
id: string
|
||||
): Promise<PackageDetail | null> {
|
||||
const pkg = await prisma.package.findUnique({
|
||||
where: { id },
|
||||
include: {
|
||||
sourceChannel: { select: { id: true, title: true } },
|
||||
ingestionRun: { select: { id: true, startedAt: true } },
|
||||
},
|
||||
});
|
||||
|
||||
if (!pkg) return null;
|
||||
|
||||
let destChannel: { id: string; title: string } | null = null;
|
||||
if (pkg.destChannelId) {
|
||||
const ch = await prisma.telegramChannel.findUnique({
|
||||
where: { id: pkg.destChannelId },
|
||||
select: { id: true, title: true },
|
||||
});
|
||||
destChannel = ch;
|
||||
}
|
||||
|
||||
return {
|
||||
id: pkg.id,
|
||||
fileName: pkg.fileName,
|
||||
fileSize: pkg.fileSize.toString(),
|
||||
contentHash: pkg.contentHash,
|
||||
archiveType: pkg.archiveType,
|
||||
fileCount: pkg.fileCount,
|
||||
isMultipart: pkg.isMultipart,
|
||||
hasPreview: pkg.previewMsgId !== null,
|
||||
creator: pkg.creator,
|
||||
partCount: pkg.partCount,
|
||||
indexedAt: pkg.indexedAt.toISOString(),
|
||||
sourceChannel: pkg.sourceChannel,
|
||||
destChannel,
|
||||
destMessageId: pkg.destMessageId?.toString() ?? null,
|
||||
sourceMessageId: pkg.sourceMessageId.toString(),
|
||||
ingestionRun: pkg.ingestionRun
|
||||
? {
|
||||
id: pkg.ingestionRun.id,
|
||||
startedAt: pkg.ingestionRun.startedAt.toISOString(),
|
||||
}
|
||||
: null,
|
||||
};
|
||||
}
|
||||
|
||||
export async function listPackageFiles(options: {
|
||||
packageId: string;
|
||||
page: number;
|
||||
limit: number;
|
||||
extension?: string;
|
||||
}) {
|
||||
const where: { packageId: string; extension?: string } = {
|
||||
packageId: options.packageId,
|
||||
};
|
||||
if (options.extension) {
|
||||
where.extension = options.extension;
|
||||
}
|
||||
|
||||
const [items, total] = await Promise.all([
|
||||
prisma.packageFile.findMany({
|
||||
where,
|
||||
orderBy: { path: "asc" },
|
||||
skip: (options.page - 1) * options.limit,
|
||||
take: options.limit,
|
||||
}),
|
||||
prisma.packageFile.count({ where }),
|
||||
]);
|
||||
|
||||
const mapped: PackageFileItem[] = items.map((f) => ({
|
||||
id: f.id,
|
||||
path: f.path,
|
||||
fileName: f.fileName,
|
||||
extension: f.extension,
|
||||
compressedSize: f.compressedSize.toString(),
|
||||
uncompressedSize: f.uncompressedSize.toString(),
|
||||
crc32: f.crc32,
|
||||
}));
|
||||
|
||||
return {
|
||||
items: mapped,
|
||||
pagination: {
|
||||
page: options.page,
|
||||
limit: options.limit,
|
||||
total,
|
||||
totalPages: Math.ceil(total / options.limit),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export async function searchPackages(options: {
|
||||
query: string;
|
||||
page: number;
|
||||
limit: number;
|
||||
searchIn: "packages" | "files" | "both";
|
||||
}) {
|
||||
const q = options.query;
|
||||
|
||||
if (options.searchIn === "files" || options.searchIn === "both") {
|
||||
// Search in package files, return parent packages
|
||||
const fileMatches = await prisma.packageFile.findMany({
|
||||
where: {
|
||||
OR: [
|
||||
{ fileName: { contains: q, mode: "insensitive" } },
|
||||
{ path: { contains: q, mode: "insensitive" } },
|
||||
],
|
||||
},
|
||||
select: { packageId: true },
|
||||
distinct: ["packageId"],
|
||||
});
|
||||
|
||||
const packageIds = fileMatches.map((f) => f.packageId);
|
||||
|
||||
const packageNameIds =
|
||||
options.searchIn === "both"
|
||||
? (
|
||||
await prisma.package.findMany({
|
||||
where: { fileName: { contains: q, mode: "insensitive" } },
|
||||
select: { id: true },
|
||||
})
|
||||
).map((p) => p.id)
|
||||
: [];
|
||||
|
||||
const allIds = [...new Set([...packageIds, ...packageNameIds])];
|
||||
|
||||
const [items, total] = await Promise.all([
|
||||
prisma.package.findMany({
|
||||
where: { id: { in: allIds } },
|
||||
orderBy: { indexedAt: "desc" },
|
||||
skip: (options.page - 1) * options.limit,
|
||||
take: options.limit,
|
||||
select: {
|
||||
id: true,
|
||||
fileName: true,
|
||||
fileSize: true,
|
||||
contentHash: true,
|
||||
archiveType: true,
|
||||
fileCount: true,
|
||||
isMultipart: true,
|
||||
indexedAt: true,
|
||||
creator: true,
|
||||
previewMsgId: true,
|
||||
sourceChannel: { select: { id: true, title: true } },
|
||||
},
|
||||
}),
|
||||
Promise.resolve(allIds.length),
|
||||
]);
|
||||
|
||||
const mapped: PackageListItem[] = items.map((pkg) => ({
|
||||
id: pkg.id,
|
||||
fileName: pkg.fileName,
|
||||
fileSize: pkg.fileSize.toString(),
|
||||
contentHash: pkg.contentHash,
|
||||
archiveType: pkg.archiveType,
|
||||
fileCount: pkg.fileCount,
|
||||
isMultipart: pkg.isMultipart,
|
||||
hasPreview: pkg.previewMsgId !== null,
|
||||
creator: pkg.creator,
|
||||
indexedAt: pkg.indexedAt.toISOString(),
|
||||
sourceChannel: pkg.sourceChannel,
|
||||
}));
|
||||
|
||||
return {
|
||||
items: mapped,
|
||||
pagination: {
|
||||
page: options.page,
|
||||
limit: options.limit,
|
||||
total,
|
||||
totalPages: Math.ceil(total / options.limit),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// Search packages only
|
||||
return listPackages({
|
||||
page: options.page,
|
||||
limit: options.limit,
|
||||
sortBy: "indexedAt",
|
||||
order: "desc",
|
||||
});
|
||||
}
|
||||
|
||||
export async function getIngestionStatus(): Promise<IngestionAccountStatus[]> {
|
||||
const accounts = await prisma.telegramAccount.findMany({
|
||||
orderBy: { createdAt: "asc" },
|
||||
});
|
||||
|
||||
const statuses: IngestionAccountStatus[] = [];
|
||||
|
||||
for (const account of accounts) {
|
||||
const lastRun = await prisma.ingestionRun.findFirst({
|
||||
where: { accountId: account.id, status: { not: "RUNNING" } },
|
||||
orderBy: { startedAt: "desc" },
|
||||
});
|
||||
|
||||
const currentRun = await prisma.ingestionRun.findFirst({
|
||||
where: { accountId: account.id, status: "RUNNING" },
|
||||
orderBy: { startedAt: "desc" },
|
||||
});
|
||||
|
||||
statuses.push({
|
||||
id: account.id,
|
||||
displayName: account.displayName,
|
||||
phone: account.phone,
|
||||
isActive: account.isActive,
|
||||
authState: account.authState,
|
||||
lastSeenAt: account.lastSeenAt?.toISOString() ?? null,
|
||||
lastRun: lastRun
|
||||
? {
|
||||
id: lastRun.id,
|
||||
status: lastRun.status,
|
||||
startedAt: lastRun.startedAt.toISOString(),
|
||||
finishedAt: lastRun.finishedAt?.toISOString() ?? null,
|
||||
messagesScanned: lastRun.messagesScanned,
|
||||
zipsFound: lastRun.zipsFound,
|
||||
zipsDuplicate: lastRun.zipsDuplicate,
|
||||
zipsIngested: lastRun.zipsIngested,
|
||||
}
|
||||
: null,
|
||||
currentRun: currentRun
|
||||
? {
|
||||
id: currentRun.id,
|
||||
startedAt: currentRun.startedAt.toISOString(),
|
||||
messagesScanned: currentRun.messagesScanned,
|
||||
zipsFound: currentRun.zipsFound,
|
||||
zipsDuplicate: currentRun.zipsDuplicate,
|
||||
zipsIngested: currentRun.zipsIngested,
|
||||
// Live activity tracking
|
||||
currentActivity: currentRun.currentActivity,
|
||||
currentStep: currentRun.currentStep,
|
||||
currentChannel: currentRun.currentChannel,
|
||||
currentFile: currentRun.currentFile,
|
||||
currentFileNum: currentRun.currentFileNum,
|
||||
totalFiles: currentRun.totalFiles,
|
||||
downloadedBytes: currentRun.downloadedBytes?.toString() ?? null,
|
||||
totalBytes: currentRun.totalBytes?.toString() ?? null,
|
||||
downloadPercent: currentRun.downloadPercent,
|
||||
lastActivityAt: currentRun.lastActivityAt?.toISOString() ?? null,
|
||||
}
|
||||
: null,
|
||||
});
|
||||
}
|
||||
|
||||
return statuses;
|
||||
}
|
||||
88
src/lib/telegram/types.ts
Normal file
88
src/lib/telegram/types.ts
Normal file
@@ -0,0 +1,88 @@
|
||||
export interface PackageListItem {
|
||||
id: string;
|
||||
fileName: string;
|
||||
fileSize: string; // BigInt serialized as string
|
||||
contentHash: string;
|
||||
archiveType: "ZIP" | "RAR";
|
||||
fileCount: number;
|
||||
isMultipart: boolean;
|
||||
hasPreview: boolean;
|
||||
creator: string | null;
|
||||
indexedAt: string;
|
||||
sourceChannel: {
|
||||
id: string;
|
||||
title: string;
|
||||
};
|
||||
}
|
||||
|
||||
export interface PackageDetail extends PackageListItem {
|
||||
partCount: number;
|
||||
destChannel: {
|
||||
id: string;
|
||||
title: string;
|
||||
} | null;
|
||||
destMessageId: string | null;
|
||||
sourceMessageId: string;
|
||||
ingestionRun: {
|
||||
id: string;
|
||||
startedAt: string;
|
||||
} | null;
|
||||
}
|
||||
|
||||
export interface PackageFileItem {
|
||||
id: string;
|
||||
path: string;
|
||||
fileName: string;
|
||||
extension: string | null;
|
||||
compressedSize: string;
|
||||
uncompressedSize: string;
|
||||
crc32: string | null;
|
||||
}
|
||||
|
||||
export interface PaginatedResponse<T> {
|
||||
items: T[];
|
||||
pagination: {
|
||||
page: number;
|
||||
limit: number;
|
||||
total: number;
|
||||
totalPages: number;
|
||||
};
|
||||
}
|
||||
|
||||
export interface IngestionAccountStatus {
|
||||
id: string;
|
||||
displayName: string | null;
|
||||
phone: string;
|
||||
isActive: boolean;
|
||||
authState: string;
|
||||
lastSeenAt: string | null;
|
||||
lastRun: {
|
||||
id: string;
|
||||
status: string;
|
||||
startedAt: string;
|
||||
finishedAt: string | null;
|
||||
messagesScanned: number;
|
||||
zipsFound: number;
|
||||
zipsDuplicate: number;
|
||||
zipsIngested: number;
|
||||
} | null;
|
||||
currentRun: {
|
||||
id: string;
|
||||
startedAt: string;
|
||||
messagesScanned: number;
|
||||
zipsFound: number;
|
||||
zipsDuplicate: number;
|
||||
zipsIngested: number;
|
||||
// Live activity tracking
|
||||
currentActivity: string | null;
|
||||
currentStep: string | null;
|
||||
currentChannel: string | null;
|
||||
currentFile: string | null;
|
||||
currentFileNum: number | null;
|
||||
totalFiles: number | null;
|
||||
downloadedBytes: string | null; // BigInt serialized as string
|
||||
totalBytes: string | null; // BigInt serialized as string
|
||||
downloadPercent: number | null;
|
||||
lastActivityAt: string | null;
|
||||
} | null;
|
||||
}
|
||||
Reference in New Issue
Block a user