feat: add Telegram integration with forum topic support and creator tracking

Adds full Telegram ZIP ingestion pipeline: TDLib worker service scans source
channels for archive files, deduplicates by content hash, extracts metadata,
uploads to archive channel, and indexes in Postgres. Forum supergroups are
scanned per-topic with topic names used as creator. Filename-based creator
extraction (e.g. "Mammoth Factory - 2026-01.zip") serves as fallback.

Includes admin UI for managing accounts/channels, simplified account setup
(API credentials via env vars), auth code/password submission dialog,
package browser with creator column, and live ingestion activity tracking.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
xCyanGrizzly
2026-02-24 16:02:06 +01:00
parent beb9cfb312
commit b427193d17
70 changed files with 8627 additions and 2 deletions

View File

@@ -0,0 +1,13 @@
import { NextResponse } from "next/server";
import { authenticateApiRequest } from "@/lib/telegram/api-auth";
import { getIngestionStatus } from "@/lib/telegram/queries";
export const dynamic = "force-dynamic";
export async function GET(request: Request) {
const authResult = await authenticateApiRequest(request);
if ("error" in authResult) return authResult.error;
const accounts = await getIngestionStatus();
return NextResponse.json({ accounts });
}

View File

@@ -0,0 +1,77 @@
import { NextResponse } from "next/server";
import { authenticateApiRequest } from "@/lib/telegram/api-auth";
import { triggerIngestionSchema } from "@/schemas/telegram";
import { prisma } from "@/lib/prisma";
export const dynamic = "force-dynamic";
export async function POST(request: Request) {
const authResult = await authenticateApiRequest(request, true);
if ("error" in authResult) return authResult.error;
let body: unknown = {};
try {
body = await request.json();
} catch {
// Empty body is fine — triggers all accounts
}
const parsed = triggerIngestionSchema.safeParse(body);
if (!parsed.success) {
return NextResponse.json(
{ error: "Invalid parameters", details: parsed.error.flatten() },
{ status: 400 }
);
}
// Find accounts to trigger
const where: { isActive: boolean; authState: "AUTHENTICATED"; id?: string } = {
isActive: true,
authState: "AUTHENTICATED",
};
if (parsed.data.accountId) {
where.id = parsed.data.accountId;
}
const accounts = await prisma.telegramAccount.findMany({
where,
select: { id: true },
});
if (accounts.length === 0) {
return NextResponse.json(
{ triggered: false, message: "No eligible accounts found" },
{ status: 404 }
);
}
// Create ingestion runs marked as RUNNING — the worker will pick these up
// when it next polls, or we use pg_notify for immediate pickup
for (const account of accounts) {
// Only create if no run is already RUNNING for this account
const existing = await prisma.ingestionRun.findFirst({
where: { accountId: account.id, status: "RUNNING" },
});
if (!existing) {
await prisma.ingestionRun.create({
data: { accountId: account.id, status: "RUNNING" },
});
}
}
// Send pg_notify for immediate worker pickup
try {
await prisma.$queryRawUnsafe(
`SELECT pg_notify('ingestion_trigger', $1)`,
accounts.map((a) => a.id).join(",")
);
} catch {
// pg_notify is best-effort — worker will pick up on next cycle anyway
}
return NextResponse.json({
triggered: true,
accountIds: accounts.map((a) => a.id),
message: `Ingestion queued for ${accounts.length} account(s)`,
});
}

View File

@@ -0,0 +1,17 @@
import { NextResponse } from "next/server";
import { authenticateApiRequest } from "@/lib/telegram/api-auth";
import { listAccountChannelLinks } from "@/lib/telegram/admin-queries";
export const dynamic = "force-dynamic";
export async function GET(
request: Request,
{ params }: { params: Promise<{ accountId: string }> }
) {
const authResult = await authenticateApiRequest(request, true);
if ("error" in authResult) return authResult.error;
const { accountId } = await params;
const links = await listAccountChannelLinks(accountId);
return NextResponse.json(links);
}

View File

@@ -0,0 +1,17 @@
import { NextResponse } from "next/server";
import { authenticateApiRequest } from "@/lib/telegram/api-auth";
import { getUnlinkedChannels } from "@/lib/telegram/admin-queries";
export const dynamic = "force-dynamic";
export async function GET(
request: Request,
{ params }: { params: Promise<{ accountId: string }> }
) {
const authResult = await authenticateApiRequest(request, true);
if ("error" in authResult) return authResult.error;
const { accountId } = await params;
const channels = await getUnlinkedChannels(accountId);
return NextResponse.json(channels);
}

View File

@@ -0,0 +1,32 @@
import { NextResponse } from "next/server";
import { authenticateApiRequest } from "@/lib/telegram/api-auth";
import { listPackageFiles } from "@/lib/telegram/queries";
import { listFilesSchema } from "@/schemas/telegram";
export const dynamic = "force-dynamic";
export async function GET(
request: Request,
{ params }: { params: Promise<{ id: string }> }
) {
const authResult = await authenticateApiRequest(request);
if ("error" in authResult) return authResult.error;
const { id } = await params;
const { searchParams } = new URL(request.url);
const parsed = listFilesSchema.safeParse(Object.fromEntries(searchParams));
if (!parsed.success) {
return NextResponse.json(
{ error: "Invalid parameters", details: parsed.error.flatten() },
{ status: 400 }
);
}
const result = await listPackageFiles({
packageId: id,
...parsed.data,
});
return NextResponse.json(result);
}

View File

@@ -0,0 +1,42 @@
import { NextResponse } from "next/server";
import { prisma } from "@/lib/prisma";
import { authenticateApiRequest } from "@/lib/telegram/api-auth";
/**
* GET /api/zips/:id/preview
* Returns the preview thumbnail image as JPEG binary.
* Cached for 1 hour (immutable once set).
*/
export async function GET(
request: Request,
{ params }: { params: Promise<{ id: string }> }
) {
const authResult = await authenticateApiRequest(request);
if ("error" in authResult) return authResult.error;
const { id } = await params;
const pkg = await prisma.package.findUnique({
where: { id },
select: { previewData: true },
});
if (!pkg || !pkg.previewData) {
return new NextResponse(null, { status: 404 });
}
// previewData is stored as Bytes (Buffer) from Prisma
const buffer =
pkg.previewData instanceof Buffer
? pkg.previewData
: Buffer.from(pkg.previewData);
return new NextResponse(buffer, {
status: 200,
headers: {
"Content-Type": "image/jpeg",
"Content-Length": String(buffer.length),
"Cache-Control": "public, max-age=3600, immutable",
},
});
}

View File

@@ -0,0 +1,22 @@
import { NextResponse } from "next/server";
import { authenticateApiRequest } from "@/lib/telegram/api-auth";
import { getPackageById } from "@/lib/telegram/queries";
export const dynamic = "force-dynamic";
export async function GET(
request: Request,
{ params }: { params: Promise<{ id: string }> }
) {
const authResult = await authenticateApiRequest(request);
if ("error" in authResult) return authResult.error;
const { id } = await params;
const pkg = await getPackageById(id);
if (!pkg) {
return NextResponse.json({ error: "Package not found" }, { status: 404 });
}
return NextResponse.json(pkg);
}

24
src/app/api/zips/route.ts Normal file
View File

@@ -0,0 +1,24 @@
import { NextResponse } from "next/server";
import { authenticateApiRequest } from "@/lib/telegram/api-auth";
import { listPackages } from "@/lib/telegram/queries";
import { listPackagesSchema } from "@/schemas/telegram";
export const dynamic = "force-dynamic";
export async function GET(request: Request) {
const authResult = await authenticateApiRequest(request);
if ("error" in authResult) return authResult.error;
const { searchParams } = new URL(request.url);
const parsed = listPackagesSchema.safeParse(Object.fromEntries(searchParams));
if (!parsed.success) {
return NextResponse.json(
{ error: "Invalid parameters", details: parsed.error.flatten() },
{ status: 400 }
);
}
const result = await listPackages(parsed.data);
return NextResponse.json(result);
}

View File

@@ -0,0 +1,25 @@
import { NextResponse } from "next/server";
import { authenticateApiRequest } from "@/lib/telegram/api-auth";
import { searchPackages } from "@/lib/telegram/queries";
import { searchSchema } from "@/schemas/telegram";
export const dynamic = "force-dynamic";
export async function GET(request: Request) {
const authResult = await authenticateApiRequest(request);
if ("error" in authResult) return authResult.error;
const { searchParams } = new URL(request.url);
const parsed = searchSchema.safeParse(Object.fromEntries(searchParams));
if (!parsed.success) {
return NextResponse.json(
{ error: "Invalid parameters", details: parsed.error.flatten() },
{ status: 400 }
);
}
const { q, ...rest } = parsed.data;
const result = await searchPackages({ query: q, ...rest });
return NextResponse.json(result);
}