mirror of
https://github.com/xCyanGrizzly/DragonsStash.git
synced 2026-05-10 22:01:16 +00:00
Compare commits
9 Commits
copilot/fi
...
copilot/fi
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
651e9e6bdd | ||
|
|
8d508d5a86 | ||
|
|
2bb3caf7d9 | ||
|
|
8d95752106 | ||
|
|
22419106c1 | ||
|
|
e45de85c69 | ||
|
|
71a2e6a5e8 | ||
|
|
1436b630e2 | ||
|
|
43af23d3be |
@@ -13,6 +13,8 @@ AUTH_GITHUB_ID=""
|
||||
AUTH_GITHUB_SECRET=""
|
||||
|
||||
# App
|
||||
# APP_PORT controls the port the container listens on AND how it is exposed on the host.
|
||||
# If you change APP_PORT, also update NEXT_PUBLIC_APP_URL to match.
|
||||
NEXT_PUBLIC_APP_URL="http://localhost:3000"
|
||||
APP_PORT=3000
|
||||
|
||||
|
||||
@@ -54,6 +54,7 @@ RUN chmod +x docker-entrypoint.sh
|
||||
|
||||
USER nextjs
|
||||
|
||||
# Default port — overridden at runtime by the PORT env var (set via docker-compose APP_PORT)
|
||||
EXPOSE 3000
|
||||
ENV PORT=3000
|
||||
ENV HOSTNAME="0.0.0.0"
|
||||
|
||||
@@ -5,7 +5,7 @@ services:
|
||||
dockerfile: Dockerfile
|
||||
pull_policy: never
|
||||
ports:
|
||||
- "${APP_PORT:-3000}:3000"
|
||||
- "${APP_PORT:-3000}:${APP_PORT:-3000}"
|
||||
environment:
|
||||
- DATABASE_URL=postgresql://${POSTGRES_USER:-dragons}:${POSTGRES_PASSWORD:-stash}@db:5432/${POSTGRES_DB:-dragonsstash}
|
||||
- AUTH_SECRET=${AUTH_SECRET:?Set AUTH_SECRET in .env}
|
||||
@@ -17,11 +17,13 @@ services:
|
||||
- BOT_TOKEN=${BOT_TOKEN:-}
|
||||
- BOT_USERNAME=${BOT_USERNAME:-}
|
||||
- LOG_LEVEL=${LOG_LEVEL:-info}
|
||||
- WORKER_INTERVAL_MINUTES=${WORKER_INTERVAL_MINUTES:-60}
|
||||
- PORT=${APP_PORT:-3000}
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "--spider", "http://localhost:3000/api/health"]
|
||||
test: ["CMD-SHELL", "wget -q --spider http://localhost:$$PORT/api/health || exit 1"]
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
|
||||
2
package-lock.json
generated
2
package-lock.json
generated
@@ -49,7 +49,7 @@
|
||||
"ts-node": "^10.9.2",
|
||||
"tsx": "^4.21.0",
|
||||
"tw-animate-css": "^1.4.0",
|
||||
"typescript": "^5"
|
||||
"typescript": "5.9.3"
|
||||
}
|
||||
},
|
||||
"node_modules/@alloc/quick-lru": {
|
||||
|
||||
@@ -58,6 +58,6 @@
|
||||
"ts-node": "^10.9.2",
|
||||
"tsx": "^4.21.0",
|
||||
"tw-animate-css": "^1.4.0",
|
||||
"typescript": "^5"
|
||||
"typescript": "5.9.3"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
Power,
|
||||
ArrowDownToLine,
|
||||
ArrowUpFromLine,
|
||||
RefreshCcw,
|
||||
} from "lucide-react";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { Button } from "@/components/ui/button";
|
||||
@@ -23,12 +24,14 @@ interface ChannelColumnsProps {
|
||||
onToggleActive: (id: string) => void;
|
||||
onDelete: (id: string) => void;
|
||||
onSetType: (id: string, type: "SOURCE" | "DESTINATION") => void;
|
||||
onRescan: (id: string) => void;
|
||||
}
|
||||
|
||||
export function getChannelColumns({
|
||||
onToggleActive,
|
||||
onDelete,
|
||||
onSetType,
|
||||
onRescan,
|
||||
}: ChannelColumnsProps): ColumnDef<ChannelRow, unknown>[] {
|
||||
return [
|
||||
{
|
||||
@@ -121,6 +124,14 @@ export function getChannelColumns({
|
||||
Set as Source
|
||||
</DropdownMenuItem>
|
||||
)}
|
||||
{row.original.type === "SOURCE" && (
|
||||
<DropdownMenuItem
|
||||
onClick={() => onRescan(row.original.id)}
|
||||
>
|
||||
<RefreshCcw className="mr-2 h-3.5 w-3.5" />
|
||||
Rescan Channel
|
||||
</DropdownMenuItem>
|
||||
)}
|
||||
<DropdownMenuItem
|
||||
onClick={() => onToggleActive(row.original.id)}
|
||||
>
|
||||
|
||||
@@ -2,26 +2,36 @@
|
||||
|
||||
import { useState, useTransition } from "react";
|
||||
import { toast } from "sonner";
|
||||
import { Download } from "lucide-react";
|
||||
import { getChannelColumns } from "./channel-columns";
|
||||
import { DestinationCard } from "./destination-card";
|
||||
import { ChannelPickerDialog } from "./channel-picker-dialog";
|
||||
import {
|
||||
deleteChannel,
|
||||
toggleChannelActive,
|
||||
setChannelType,
|
||||
rescanChannel,
|
||||
} from "../actions";
|
||||
import { DataTable } from "@/components/shared/data-table";
|
||||
import { DeleteDialog } from "@/components/shared/delete-dialog";
|
||||
import type { ChannelRow, GlobalDestination } from "@/lib/telegram/admin-queries";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import type { AccountRow, ChannelRow, GlobalDestination } from "@/lib/telegram/admin-queries";
|
||||
import { useDataTable } from "@/hooks/use-data-table";
|
||||
|
||||
interface ChannelsTabProps {
|
||||
channels: ChannelRow[];
|
||||
globalDestination: GlobalDestination;
|
||||
accounts: AccountRow[];
|
||||
}
|
||||
|
||||
export function ChannelsTab({ channels, globalDestination }: ChannelsTabProps) {
|
||||
export function ChannelsTab({ channels, globalDestination, accounts }: ChannelsTabProps) {
|
||||
const [isPending, startTransition] = useTransition();
|
||||
const [deleteId, setDeleteId] = useState<string | null>(null);
|
||||
const [rescanId, setRescanId] = useState<string | null>(null);
|
||||
const [fetchChannelsAccountId, setFetchChannelsAccountId] = useState<string | null>(null);
|
||||
|
||||
// Find the first authenticated account for "Fetch Channels"
|
||||
const authenticatedAccounts = accounts.filter((a) => a.authState === "AUTHENTICATED" && a.isActive);
|
||||
|
||||
const columns = getChannelColumns({
|
||||
onToggleActive: (id) => {
|
||||
@@ -39,6 +49,7 @@ export function ChannelsTab({ channels, globalDestination }: ChannelsTabProps) {
|
||||
else toast.error(result.error);
|
||||
});
|
||||
},
|
||||
onRescan: (id) => setRescanId(id),
|
||||
});
|
||||
|
||||
const { table } = useDataTable({
|
||||
@@ -60,19 +71,51 @@ export function ChannelsTab({ channels, globalDestination }: ChannelsTabProps) {
|
||||
});
|
||||
};
|
||||
|
||||
const handleRescan = () => {
|
||||
if (!rescanId) return;
|
||||
startTransition(async () => {
|
||||
const result = await rescanChannel(rescanId);
|
||||
if (result.success) {
|
||||
toast.success("Channel scan progress reset — it will be fully rescanned on the next sync");
|
||||
setRescanId(null);
|
||||
} else {
|
||||
toast.error(result.error);
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
const handleFetchChannels = () => {
|
||||
if (authenticatedAccounts.length > 0) {
|
||||
setFetchChannelsAccountId(authenticatedAccounts[0].id);
|
||||
} else {
|
||||
toast.error("No authenticated accounts available. Add and authenticate an account first.");
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="space-y-4">
|
||||
<DestinationCard destination={globalDestination} />
|
||||
|
||||
<div className="flex items-center gap-2">
|
||||
<Button
|
||||
variant="outline"
|
||||
onClick={handleFetchChannels}
|
||||
disabled={authenticatedAccounts.length === 0}
|
||||
>
|
||||
<Download className="mr-2 h-4 w-4" />
|
||||
Fetch Channels
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
{channels.length > 0 && (
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Source channels are added per-account via the "Fetch Channels" button on the Accounts tab.
|
||||
Channels discovered via "Fetch Channels" are automatically activated as sources.
|
||||
</p>
|
||||
)}
|
||||
|
||||
<DataTable
|
||||
table={table}
|
||||
emptyMessage="No channels yet. Use "Fetch Channels" on an account to discover and add source channels."
|
||||
emptyMessage="No channels yet. Click "Fetch Channels" above to discover and add source channels."
|
||||
/>
|
||||
|
||||
<DeleteDialog
|
||||
@@ -83,6 +126,24 @@ export function ChannelsTab({ channels, globalDestination }: ChannelsTabProps) {
|
||||
onConfirm={handleDelete}
|
||||
isLoading={isPending}
|
||||
/>
|
||||
|
||||
<DeleteDialog
|
||||
open={!!rescanId}
|
||||
onOpenChange={(open) => !open && setRescanId(null)}
|
||||
title="Rescan Channel"
|
||||
description="This will reset all scan progress for this channel. On the next sync the worker will re-process every message from the beginning. Packages that are already in the library will be skipped (deduplication by hash), but any missing files will be re-downloaded and re-uploaded. This may take a long time for large channels."
|
||||
confirmLabel="Rescan"
|
||||
onConfirm={handleRescan}
|
||||
isLoading={isPending}
|
||||
/>
|
||||
|
||||
<ChannelPickerDialog
|
||||
accountId={fetchChannelsAccountId}
|
||||
open={!!fetchChannelsAccountId}
|
||||
onOpenChange={(open) => {
|
||||
if (!open) setFetchChannelsAccountId(null);
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ interface TelegramAdminProps {
|
||||
ingestionStatus: IngestionAccountStatus[];
|
||||
globalDestination: GlobalDestination;
|
||||
sendHistory: SendHistoryRow[];
|
||||
workerIntervalMinutes: number;
|
||||
}
|
||||
|
||||
export function TelegramAdmin({
|
||||
@@ -24,6 +25,7 @@ export function TelegramAdmin({
|
||||
ingestionStatus,
|
||||
globalDestination,
|
||||
sendHistory,
|
||||
workerIntervalMinutes,
|
||||
}: TelegramAdminProps) {
|
||||
return (
|
||||
<div className="space-y-4">
|
||||
@@ -32,7 +34,7 @@ export function TelegramAdmin({
|
||||
description="Manage Telegram accounts, channels, and ingestion"
|
||||
/>
|
||||
|
||||
<WorkerStatusPanel initialStatus={ingestionStatus} />
|
||||
<WorkerStatusPanel initialStatus={ingestionStatus} initialIntervalMinutes={workerIntervalMinutes} />
|
||||
|
||||
<Tabs defaultValue="accounts" className="space-y-4">
|
||||
<TabsList>
|
||||
@@ -51,7 +53,7 @@ export function TelegramAdmin({
|
||||
<AccountsTab accounts={accounts} />
|
||||
</TabsContent>
|
||||
<TabsContent value="channels">
|
||||
<ChannelsTab channels={channels} globalDestination={globalDestination} />
|
||||
<ChannelsTab channels={channels} globalDestination={globalDestination} accounts={accounts} />
|
||||
</TabsContent>
|
||||
<TabsContent value="sends">
|
||||
<BotSendsTab history={sendHistory} />
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
"use client";
|
||||
|
||||
import { useEffect, useState, useCallback } from "react";
|
||||
import { useEffect, useState, useCallback, useTransition } from "react";
|
||||
import {
|
||||
Loader2,
|
||||
CheckCircle2,
|
||||
@@ -14,10 +14,13 @@ import { Card, CardContent } from "@/components/ui/card";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { cn } from "@/lib/utils";
|
||||
import { toast } from "sonner";
|
||||
import { triggerIngestion } from "../actions";
|
||||
import type { IngestionAccountStatus } from "@/lib/telegram/types";
|
||||
|
||||
interface WorkerStatusPanelProps {
|
||||
initialStatus: IngestionAccountStatus[];
|
||||
initialIntervalMinutes?: number;
|
||||
}
|
||||
|
||||
const AUTH_STATE_CONFIG: Record<
|
||||
@@ -39,15 +42,28 @@ const AUTH_STATE_CONFIG: Record<
|
||||
EXPIRED: { label: "Expired", color: "text-red-500", icon: "x" },
|
||||
};
|
||||
|
||||
export function WorkerStatusPanel({ initialStatus }: WorkerStatusPanelProps) {
|
||||
export function WorkerStatusPanel({ initialStatus, initialIntervalMinutes = 60 }: WorkerStatusPanelProps) {
|
||||
const [accounts, setAccounts] = useState(initialStatus);
|
||||
const [error, setError] = useState(false);
|
||||
const [nextRunCountdown, setNextRunCountdown] = useState<string | null>(null);
|
||||
const [workerIntervalMinutes, setWorkerIntervalMinutes] = useState(initialIntervalMinutes);
|
||||
const [isPending, startTransition] = useTransition();
|
||||
|
||||
// Find active run
|
||||
const activeRun = accounts.find((a) => a.currentRun);
|
||||
const isRunning = !!activeRun;
|
||||
|
||||
const handleSyncNow = useCallback(() => {
|
||||
startTransition(async () => {
|
||||
const result = await triggerIngestion();
|
||||
if (result.success) {
|
||||
toast.success("Sync triggered — worker will start shortly");
|
||||
} else {
|
||||
toast.error(result.error ?? "Failed to trigger sync");
|
||||
}
|
||||
});
|
||||
}, []);
|
||||
|
||||
// Poll for status
|
||||
useEffect(() => {
|
||||
let timer: ReturnType<typeof setTimeout>;
|
||||
@@ -60,6 +76,9 @@ export function WorkerStatusPanel({ initialStatus }: WorkerStatusPanelProps) {
|
||||
const data = await res.json();
|
||||
if (mounted) {
|
||||
setAccounts(data.accounts ?? []);
|
||||
if (data.workerIntervalMinutes) {
|
||||
setWorkerIntervalMinutes(data.workerIntervalMinutes);
|
||||
}
|
||||
setError(false);
|
||||
}
|
||||
} catch {
|
||||
@@ -86,7 +105,7 @@ export function WorkerStatusPanel({ initialStatus }: WorkerStatusPanelProps) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Estimate next run based on last run finish time + interval (5 min + up to 5 min jitter)
|
||||
// Estimate next run based on last run finish time + configured interval + up to 5 min jitter
|
||||
const lastFinished = accounts
|
||||
.filter((a) => a.lastRun?.finishedAt)
|
||||
.map((a) => new Date(a.lastRun!.finishedAt!).getTime())
|
||||
@@ -97,7 +116,7 @@ export function WorkerStatusPanel({ initialStatus }: WorkerStatusPanelProps) {
|
||||
return;
|
||||
}
|
||||
|
||||
const intervalMs = 5 * 60 * 1000; // 5 min base
|
||||
const intervalMs = workerIntervalMinutes * 60 * 1000;
|
||||
const estimatedNext = lastFinished + intervalMs;
|
||||
|
||||
const tick = () => {
|
||||
@@ -116,7 +135,7 @@ export function WorkerStatusPanel({ initialStatus }: WorkerStatusPanelProps) {
|
||||
tick();
|
||||
const interval = setInterval(tick, 1_000);
|
||||
return () => clearInterval(interval);
|
||||
}, [isRunning, accounts]);
|
||||
}, [isRunning, accounts, workerIntervalMinutes]);
|
||||
|
||||
if (accounts.length === 0 && !error) {
|
||||
return (
|
||||
@@ -182,7 +201,12 @@ export function WorkerStatusPanel({ initialStatus }: WorkerStatusPanelProps) {
|
||||
) : isRunning && activeRun?.currentRun ? (
|
||||
<RunningStatus run={activeRun.currentRun} />
|
||||
) : (
|
||||
<IdleStatus accounts={accounts} nextRunCountdown={nextRunCountdown} />
|
||||
<IdleStatus
|
||||
accounts={accounts}
|
||||
nextRunCountdown={nextRunCountdown}
|
||||
onSyncNow={handleSyncNow}
|
||||
isSyncing={isPending}
|
||||
/>
|
||||
)}
|
||||
</CardContent>
|
||||
</Card>
|
||||
@@ -256,9 +280,13 @@ function RunningStatus({
|
||||
function IdleStatus({
|
||||
accounts,
|
||||
nextRunCountdown,
|
||||
onSyncNow,
|
||||
isSyncing,
|
||||
}: {
|
||||
accounts: IngestionAccountStatus[];
|
||||
nextRunCountdown: string | null;
|
||||
onSyncNow: () => void;
|
||||
isSyncing: boolean;
|
||||
}) {
|
||||
const lastRun = accounts
|
||||
.filter((a) => a.lastRun)
|
||||
@@ -321,14 +349,32 @@ function IdleStatus({
|
||||
)}
|
||||
</div>
|
||||
|
||||
{nextRunCountdown && hasAuthenticated && (
|
||||
<div className="flex items-center gap-1.5 shrink-0">
|
||||
<RefreshCw className="h-3 w-3 text-muted-foreground" />
|
||||
<span className="text-xs text-muted-foreground tabular-nums">
|
||||
Next: {nextRunCountdown}
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
<div className="flex items-center gap-2 shrink-0">
|
||||
{nextRunCountdown && hasAuthenticated && (
|
||||
<div className="flex items-center gap-1.5">
|
||||
<RefreshCw className="h-3 w-3 text-muted-foreground" />
|
||||
<span className="text-xs text-muted-foreground tabular-nums">
|
||||
Next: {nextRunCountdown}
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
{hasAuthenticated && (
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
className="h-7 text-xs px-2"
|
||||
onClick={onSyncNow}
|
||||
disabled={isSyncing}
|
||||
>
|
||||
{isSyncing ? (
|
||||
<Loader2 className="h-3 w-3 animate-spin mr-1" />
|
||||
) : (
|
||||
<RefreshCw className="h-3 w-3 mr-1" />
|
||||
)}
|
||||
Sync Now
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -297,6 +297,52 @@ export async function triggerChannelSync(): Promise<ActionResult> {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset all scan progress for a channel so the worker will re-process it
|
||||
* from the very beginning on the next ingestion cycle.
|
||||
*
|
||||
* This clears:
|
||||
* - `lastProcessedMessageId` on every AccountChannelMap linked to this channel
|
||||
* - All TopicProgress records for those maps (for forum channels)
|
||||
*/
|
||||
export async function rescanChannel(channelId: string): Promise<ActionResult> {
|
||||
const admin = await requireAdmin();
|
||||
if (!admin.success) return admin;
|
||||
|
||||
const channel = await prisma.telegramChannel.findUnique({
|
||||
where: { id: channelId },
|
||||
});
|
||||
if (!channel) return { success: false, error: "Channel not found" };
|
||||
|
||||
try {
|
||||
// Find all account-channel maps for this channel
|
||||
const maps = await prisma.accountChannelMap.findMany({
|
||||
where: { channelId },
|
||||
select: { id: true },
|
||||
});
|
||||
|
||||
const mapIds = maps.map((m) => m.id);
|
||||
|
||||
// Delete all topic progress records for these maps (forum channels)
|
||||
if (mapIds.length > 0) {
|
||||
await prisma.topicProgress.deleteMany({
|
||||
where: { accountChannelMapId: { in: mapIds } },
|
||||
});
|
||||
}
|
||||
|
||||
// Reset the scan cursor so the worker re-processes from the start
|
||||
await prisma.accountChannelMap.updateMany({
|
||||
where: { channelId },
|
||||
data: { lastProcessedMessageId: null },
|
||||
});
|
||||
|
||||
revalidatePath(REVALIDATE_PATH);
|
||||
return { success: true, data: undefined };
|
||||
} catch {
|
||||
return { success: false, error: "Failed to reset channel scan progress" };
|
||||
}
|
||||
}
|
||||
|
||||
// ── Account-Channel link actions ──
|
||||
|
||||
export async function linkChannel(
|
||||
@@ -377,7 +423,7 @@ export async function triggerIngestion(
|
||||
try {
|
||||
await prisma.$queryRawUnsafe(
|
||||
`SELECT pg_notify('ingestion_trigger', $1)`,
|
||||
accounts.map((a) => a.id).join(",")
|
||||
accounts.map((a: { id: string }) => a.id).join(",")
|
||||
);
|
||||
} catch {
|
||||
// Best-effort
|
||||
@@ -407,7 +453,7 @@ export async function saveChannelSelections(
|
||||
try {
|
||||
let linked = 0;
|
||||
for (const ch of channels) {
|
||||
// Upsert the channel record (new channels default to disabled)
|
||||
// Upsert the channel record and activate it (user explicitly selected it)
|
||||
const channel = await prisma.telegramChannel.upsert({
|
||||
where: { telegramId: BigInt(ch.telegramId) },
|
||||
create: {
|
||||
@@ -415,11 +461,12 @@ export async function saveChannelSelections(
|
||||
title: ch.title,
|
||||
type: "SOURCE",
|
||||
isForum: ch.isForum,
|
||||
isActive: false,
|
||||
isActive: true,
|
||||
},
|
||||
update: {
|
||||
title: ch.title,
|
||||
isForum: ch.isForum,
|
||||
isActive: true,
|
||||
},
|
||||
});
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ export default async function TelegramPage() {
|
||||
}),
|
||||
]);
|
||||
|
||||
const serializedHistory = sendHistory.map((r) => ({
|
||||
const serializedHistory = sendHistory.map((r: typeof sendHistory[number]) => ({
|
||||
id: r.id,
|
||||
packageName: r.package.fileName,
|
||||
recipientName: r.telegramLink.telegramName,
|
||||
@@ -42,6 +42,7 @@ export default async function TelegramPage() {
|
||||
ingestionStatus={ingestionStatus}
|
||||
globalDestination={globalDestination}
|
||||
sendHistory={serializedHistory}
|
||||
workerIntervalMinutes={parseInt(process.env.WORKER_INTERVAL_MINUTES ?? "60", 10)}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -9,5 +9,9 @@ export async function GET(request: Request) {
|
||||
if ("error" in authResult) return authResult.error;
|
||||
|
||||
const accounts = await getIngestionStatus();
|
||||
return NextResponse.json({ accounts });
|
||||
const workerIntervalMinutes = parseInt(
|
||||
process.env.WORKER_INTERVAL_MINUTES ?? "60",
|
||||
10
|
||||
);
|
||||
return NextResponse.json({ accounts, workerIntervalMinutes });
|
||||
}
|
||||
|
||||
@@ -45,33 +45,20 @@ export async function POST(request: Request) {
|
||||
);
|
||||
}
|
||||
|
||||
// Create ingestion runs marked as RUNNING — the worker will pick these up
|
||||
// when it next polls, or we use pg_notify for immediate pickup
|
||||
for (const account of accounts) {
|
||||
// Only create if no run is already RUNNING for this account
|
||||
const existing = await prisma.ingestionRun.findFirst({
|
||||
where: { accountId: account.id, status: "RUNNING" },
|
||||
});
|
||||
if (!existing) {
|
||||
await prisma.ingestionRun.create({
|
||||
data: { accountId: account.id, status: "RUNNING" },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Send pg_notify for immediate worker pickup
|
||||
// Send pg_notify for immediate worker pickup.
|
||||
// The worker creates its own IngestionRun records with proper activity tracking.
|
||||
try {
|
||||
await prisma.$queryRawUnsafe(
|
||||
`SELECT pg_notify('ingestion_trigger', $1)`,
|
||||
accounts.map((a) => a.id).join(",")
|
||||
accounts.map((a: { id: string }) => a.id).join(",")
|
||||
);
|
||||
} catch {
|
||||
// pg_notify is best-effort — worker will pick up on next cycle anyway
|
||||
// pg_notify is best-effort — worker will pick up on next scheduled cycle anyway
|
||||
}
|
||||
|
||||
return NextResponse.json({
|
||||
triggered: true,
|
||||
accountIds: accounts.map((a) => a.id),
|
||||
message: `Ingestion queued for ${accounts.length} account(s)`,
|
||||
accountIds: accounts.map((a: { id: string }) => a.id),
|
||||
message: `Ingestion triggered for ${accounts.length} account(s)`,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -18,6 +18,8 @@ interface DeleteDialogProps {
|
||||
description?: string;
|
||||
onConfirm: () => void;
|
||||
isLoading?: boolean;
|
||||
confirmLabel?: string;
|
||||
confirmLoadingLabel?: string;
|
||||
}
|
||||
|
||||
export function DeleteDialog({
|
||||
@@ -27,6 +29,8 @@ export function DeleteDialog({
|
||||
description = "This action cannot be undone.",
|
||||
onConfirm,
|
||||
isLoading,
|
||||
confirmLabel = "Delete",
|
||||
confirmLoadingLabel,
|
||||
}: DeleteDialogProps) {
|
||||
return (
|
||||
<AlertDialog open={open} onOpenChange={onOpenChange}>
|
||||
@@ -42,7 +46,7 @@ export function DeleteDialog({
|
||||
disabled={isLoading}
|
||||
className="bg-destructive text-destructive-foreground hover:bg-destructive/90"
|
||||
>
|
||||
{isLoading ? "Deleting..." : "Delete"}
|
||||
{isLoading ? (confirmLoadingLabel ?? `${confirmLabel}...`) : confirmLabel}
|
||||
</AlertDialogAction>
|
||||
</AlertDialogFooter>
|
||||
</AlertDialogContent>
|
||||
|
||||
@@ -18,6 +18,10 @@ import {
|
||||
const log = childLogger("fetch-listener");
|
||||
|
||||
let pgClient: pg.PoolClient | null = null;
|
||||
let stopped = false;
|
||||
|
||||
/** Delay (ms) before attempting to reconnect after a connection loss. */
|
||||
const RECONNECT_DELAY_MS = 5_000;
|
||||
|
||||
/**
|
||||
* Start listening for pg_notify signals from the web app.
|
||||
@@ -27,30 +31,75 @@ let pgClient: pg.PoolClient | null = null;
|
||||
* - `generate_invite` — payload = channelId → generate invite link for destination
|
||||
* - `create_destination` — payload = JSON { requestId, title } → create supergroup via TDLib
|
||||
* - `ingestion_trigger` — trigger an immediate ingestion cycle
|
||||
*
|
||||
* If the underlying connection is lost, the listener automatically reconnects
|
||||
* so that pg_notify signals are never silently dropped.
|
||||
*/
|
||||
export async function startFetchListener(): Promise<void> {
|
||||
pgClient = await pool.connect();
|
||||
await pgClient.query("LISTEN channel_fetch");
|
||||
await pgClient.query("LISTEN generate_invite");
|
||||
await pgClient.query("LISTEN create_destination");
|
||||
await pgClient.query("LISTEN ingestion_trigger");
|
||||
stopped = false;
|
||||
await connectListener();
|
||||
}
|
||||
|
||||
pgClient.on("notification", (msg) => {
|
||||
if (msg.channel === "channel_fetch" && msg.payload) {
|
||||
handleChannelFetch(msg.payload);
|
||||
} else if (msg.channel === "generate_invite" && msg.payload) {
|
||||
handleGenerateInvite(msg.payload);
|
||||
} else if (msg.channel === "create_destination" && msg.payload) {
|
||||
handleCreateDestination(msg.payload);
|
||||
} else if (msg.channel === "ingestion_trigger") {
|
||||
handleIngestionTrigger();
|
||||
async function connectListener(): Promise<void> {
|
||||
try {
|
||||
pgClient = await pool.connect();
|
||||
await pgClient.query("LISTEN channel_fetch");
|
||||
await pgClient.query("LISTEN generate_invite");
|
||||
await pgClient.query("LISTEN create_destination");
|
||||
await pgClient.query("LISTEN ingestion_trigger");
|
||||
|
||||
pgClient.on("notification", (msg) => {
|
||||
if (msg.channel === "channel_fetch" && msg.payload) {
|
||||
handleChannelFetch(msg.payload);
|
||||
} else if (msg.channel === "generate_invite" && msg.payload) {
|
||||
handleGenerateInvite(msg.payload);
|
||||
} else if (msg.channel === "create_destination" && msg.payload) {
|
||||
handleCreateDestination(msg.payload);
|
||||
} else if (msg.channel === "ingestion_trigger") {
|
||||
handleIngestionTrigger();
|
||||
}
|
||||
});
|
||||
|
||||
// Reconnect automatically when the connection ends unexpectedly
|
||||
pgClient.on("end", () => {
|
||||
if (!stopped) {
|
||||
log.warn("Fetch listener connection lost — reconnecting");
|
||||
pgClient = null;
|
||||
scheduleReconnect();
|
||||
}
|
||||
});
|
||||
|
||||
pgClient.on("error", (err) => {
|
||||
log.error({ err }, "Fetch listener connection error");
|
||||
if (!stopped && pgClient) {
|
||||
try {
|
||||
pgClient.release(true);
|
||||
} catch (releaseErr) {
|
||||
log.debug({ err: releaseErr }, "Failed to release pg client after error");
|
||||
}
|
||||
pgClient = null;
|
||||
scheduleReconnect();
|
||||
}
|
||||
});
|
||||
|
||||
log.info("Fetch listener started (channel_fetch, generate_invite, create_destination, ingestion_trigger)");
|
||||
} catch (err) {
|
||||
log.error({ err }, "Failed to start fetch listener — retrying");
|
||||
scheduleReconnect();
|
||||
}
|
||||
}
|
||||
|
||||
function scheduleReconnect(): void {
|
||||
if (stopped) return;
|
||||
setTimeout(() => {
|
||||
if (!stopped) {
|
||||
connectListener();
|
||||
}
|
||||
});
|
||||
|
||||
log.info("Fetch listener started (channel_fetch, generate_invite, create_destination, ingestion_trigger)");
|
||||
}, RECONNECT_DELAY_MS);
|
||||
}
|
||||
|
||||
export function stopFetchListener(): void {
|
||||
stopped = true;
|
||||
if (pgClient) {
|
||||
pgClient.release();
|
||||
pgClient = null;
|
||||
|
||||
@@ -351,6 +351,10 @@ export async function runWorkerForAccount(
|
||||
|
||||
const totalChannels = channelMappings.length;
|
||||
|
||||
if (totalChannels === 0) {
|
||||
accountLog.info("No active source channels linked to this account — nothing to ingest");
|
||||
}
|
||||
|
||||
for (let chIdx = 0; chIdx < channelMappings.length; chIdx++) {
|
||||
const mapping = channelMappings[chIdx];
|
||||
const channel = mapping.channel;
|
||||
@@ -451,8 +455,8 @@ export async function runWorkerForAccount(
|
||||
counters.messagesScanned += scanResult.totalScanned;
|
||||
|
||||
if (scanResult.archives.length === 0) {
|
||||
accountLog.debug(
|
||||
{ channelId: channel.id, topic: topic.name },
|
||||
accountLog.info(
|
||||
{ channelId: channel.id, topic: topic.name, totalScanned: scanResult.totalScanned },
|
||||
"No new archives in topic"
|
||||
);
|
||||
continue;
|
||||
@@ -525,7 +529,7 @@ export async function runWorkerForAccount(
|
||||
counters.messagesScanned += scanResult.totalScanned;
|
||||
|
||||
if (scanResult.archives.length === 0) {
|
||||
accountLog.debug({ channelId: channel.id }, "No new archives");
|
||||
accountLog.info({ channelId: channel.id, title: channel.title, totalScanned: scanResult.totalScanned }, "No new archives in channel");
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user