fix(worker): clamp RAR header re-read to 8MB to bound corrupt-archive reads

Add MAX_RAR_HEADER_BYTES constant to prevent unbounded ranged reads when a RAR block's HeaderSize is bogus. Real RAR block headers are far smaller; this guards against amplification attacks on corrupt/desynced archives. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
feat(worker): enable RAR ranged listing + format-aware destination reads
2026-09-21 13:31:42 +00:00 · 2026-07-27 12:34:00 +02:00 · 2026-07-27 12:06:46 +02:00 · 2026-07-27 12:03:14 +02:00 · 2026-07-27 11:58:33 +02:00 · 2026-07-27 11:55:59 +02:00
101 changed files with 14061 additions and 647 deletions
@@ -54,9 +54,24 @@ steps:
      password:
        from_secret: gitea_password
  - name: build-backup
    image: plugins/docker
    depends_on: [clone]
    settings:
      repo: git.samagsteribbe.nl/admin/dragonsstash-backup
      registry: git.samagsteribbe.nl
      dockerfile: backup/Dockerfile
      tags:
        - latest
        - "${DRONE_COMMIT_SHA:0:8}"
      username:
        from_secret: gitea_username
      password:
        from_secret: gitea_password
  - name: deploy
    image: alpine
-    depends_on: [build-app, build-worker, build-bot]
+    depends_on: [build-app, build-worker, build-bot, build-backup]
    environment:
      SSH_KEY:
        from_secret: ssh_key
@@ -36,3 +36,12 @@ TDLIB_STATE_DIR="/data/tdlib"
 WORKER_MAX_ZIP_SIZE_MB=4096
 MULTIPART_TIMEOUT_HOURS=0
 LOG_LEVEL="info"
 # Backup (NAS via SMB/CIFS + restic)
 NAS_HOST=""              # Synology NAS IP or hostname reachable from this host
 NAS_SHARE=""             # SMB share name, e.g. dragonsstash_backups
 NAS_USERNAME=""          # SMB user with read/write on the share
 NAS_PASSWORD=""          # SMB user password (avoid commas — they delimit cifs mount opts)
 RESTIC_PASSWORD=""       # generate with: openssl rand -base64 32
 KUMA_PUSH_URL=""         # optional: Uptime Kuma Push monitor URL; leave empty to disable alerting
 TZ="Etc/UTC"
@@ -54,3 +54,4 @@ src/generated
 # temp files
 nul
 tmpclaude-*
 .worktrees/
@@ -0,0 +1,13 @@
 FROM alpine:3.20
 # Note: use busybox's built-in crond (Alpine base), NOT the dcron package —
 # dcron's crond fails with "setpgid: Operation not permitted" in this runtime.
 RUN apk add --no-cache restic postgresql16-client curl tzdata tar bash
 COPY backup/backup.sh /backup.sh
 COPY backup/entrypoint.sh /entrypoint.sh
 COPY backup/crontab /etc/crontabs/root
 RUN chmod +x /backup.sh /entrypoint.sh
 ENTRYPOINT ["/entrypoint.sh"]
@@ -0,0 +1,32 @@
 #!/bin/bash
 set -euo pipefail
 report_failure() {
  [ -n "${KUMA_PUSH_URL:-}" ] || return 0
  curl -fsS "$KUMA_PUSH_URL" --get \
    --data-urlencode "status=down" \
    --data-urlencode "msg=$BASH_COMMAND failed" || true
 }
 trap report_failure ERR
 DUMP_FILE=/tmp/dragonsstash.dump
 TAR_FILE=/tmp/tdlib.tar.gz
 trap 'rm -f "$DUMP_FILE" "$TAR_FILE"' EXIT
 pg_dump -h dragonsstash-db -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Fc -f "$DUMP_FILE"
 # TDLib volumes are tarred live (best-effort, per design). A file changing
 # mid-read makes GNU tar exit 1 (warning) — that is expected here and must not
 # abort the backup. Only a genuine error (exit >= 2) is fatal.
 tar --warning=no-file-changed -czf "$TAR_FILE" -C /data tdlib-worker tdlib-bot \
  || { rc=$?; [ "$rc" -le 1 ] || exit "$rc"; }
 restic backup "$DUMP_FILE" "$TAR_FILE"
 restic forget --keep-daily 14 --prune
 if [ -n "${KUMA_PUSH_URL:-}" ]; then
  curl -fsS "$KUMA_PUSH_URL" --get \
    --data-urlencode "status=up" \
    --data-urlencode "msg=OK"
 fi
@@ -0,0 +1,2 @@
 0 3 * * * /backup.sh >> /proc/1/fd/1 2>&1
 0 4 * * 0 restic check >> /proc/1/fd/1 2>&1
@@ -0,0 +1,11 @@
 #!/bin/bash
 set -uo pipefail
 # Ensure the repo exists, but never crash-loop on it: a transient error reading
 # the repo (CIFS hiccup, stale lock) must not kill PID 1. `restic init` failing
 # because the repo already exists is expected and harmless here.
 if ! restic cat config >/dev/null 2>&1; then
  restic init || echo "restic init skipped (repo already exists or temporarily unreachable)"
 fi
 exec crond -f -l 2
@@ -12,8 +12,8 @@
        "@prisma/client": "^7.4.0",
        "pg": "^8.18.0",
        "pino": "^9.6.0",
-        "prebuilt-tdlib": "^0.1008050.0",
+        "prebuilt-tdlib": "^0.1008064.0",
-        "tdl": "^8.0.0"
+        "tdl": "^8.1.0"
      },
      "devDependencies": {
        "@types/node": "^20",
@@ -566,9 +566,9 @@
      "license": "MIT"
    },
    "node_modules/@prebuilt-tdlib/darwin-arm64": {
-      "version": "0.1008050.0",
+      "version": "0.1008064.0",
-      "resolved": "https://registry.npmjs.org/@prebuilt-tdlib/darwin-arm64/-/darwin-arm64-0.1008050.0.tgz",
+      "resolved": "https://registry.npmjs.org/@prebuilt-tdlib/darwin-arm64/-/darwin-arm64-0.1008064.0.tgz",
-      "integrity": "sha512-XrWN7M1gfvnzOBRX0YdXVfhSxIDSs/ZJ16QJ0ILDKe+grOFl/cfl7lwB/hK/MlHC6Rev56f5X7xaWnjMh0vktQ==",
+      "integrity": "sha512-Oq5us+o0g68Jag74RIV3LdLkZxQxJMcOdrVbgmyE7Unk+WcifqTb/gZw1rS6BrW+2SX2LNeGY4zQqqBTNDr17Q==",
      "cpu": [
        "arm64"
      ],
@@ -579,9 +579,9 @@
      ]
    },
    "node_modules/@prebuilt-tdlib/darwin-x64": {
-      "version": "0.1008050.0",
+      "version": "0.1008064.0",
-      "resolved": "https://registry.npmjs.org/@prebuilt-tdlib/darwin-x64/-/darwin-x64-0.1008050.0.tgz",
+      "resolved": "https://registry.npmjs.org/@prebuilt-tdlib/darwin-x64/-/darwin-x64-0.1008064.0.tgz",
-      "integrity": "sha512-a1UfBW0lYx4tUy5viMPtsbqBfBncCAgDu3FPjljfYTHjP8wfkKFxpp5+8wdxhyqdy3QriWaipVtUXQgOeEWMJg==",
+      "integrity": "sha512-Pz11xjET2Y3uUJKxkWKBc0dmOtlykmBdZ9D6Ahh+EsoLDLIWHm7M91p6nZT396YZ4n2BL+FtDYK65Ae3LDIA5g==",
      "cpu": [
        "x64"
      ],
@@ -592,9 +592,22 @@
      ]
    },
    "node_modules/@prebuilt-tdlib/linux-arm64-glibc": {
-      "version": "0.1008050.0",
+      "version": "0.1008064.0",
-      "resolved": "https://registry.npmjs.org/@prebuilt-tdlib/linux-arm64-glibc/-/linux-arm64-glibc-0.1008050.0.tgz",
+      "resolved": "https://registry.npmjs.org/@prebuilt-tdlib/linux-arm64-glibc/-/linux-arm64-glibc-0.1008064.0.tgz",
-      "integrity": "sha512-HRGspdQYzaBkU+W2M8uY5OgOkmgfTkyHkTYan/dn7EE/38QdIFW0YTvmGrl3DoFV2PA+SeJQw0xqK8tMSyHKaA==",
+      "integrity": "sha512-1kML9+RCfTOTWLzxq2klCN962/XwYhd+SGd4BxOwcmvPniYDNRUXtgMi3qRyV/Flola8dchGFrqZJU4kNZNLuQ==",
      "cpu": [
        "arm64"
      ],
      "license": "0BSD",
      "optional": true,
      "os": [
        "linux"
      ]
    },
    "node_modules/@prebuilt-tdlib/linux-arm64-musl": {
      "version": "0.1008064.0",
      "resolved": "https://registry.npmjs.org/@prebuilt-tdlib/linux-arm64-musl/-/linux-arm64-musl-0.1008064.0.tgz",
      "integrity": "sha512-tN9FJOR8VDfmOoTHMivAqBfQ/d9Bry9T/9cGSTcms3H4ORun/WO5U5zT8VqadAsqjuiQ8Y9HaUqqz65xBDtcgw==",
      "cpu": [
        "arm64"
      ],
@@ -605,9 +618,9 @@
      ]
    },
    "node_modules/@prebuilt-tdlib/linux-x64-glibc": {
-      "version": "0.1008050.0",
+      "version": "0.1008064.0",
-      "resolved": "https://registry.npmjs.org/@prebuilt-tdlib/linux-x64-glibc/-/linux-x64-glibc-0.1008050.0.tgz",
+      "resolved": "https://registry.npmjs.org/@prebuilt-tdlib/linux-x64-glibc/-/linux-x64-glibc-0.1008064.0.tgz",
-      "integrity": "sha512-Yf6ve3Dzxc66kV1cijFLn7EXKhPN5YHTjtJABEaCR5euetCI2wZp/1uBsXvyYTuFXqQbMfjO3xUCXUIBhLoChw==",
+      "integrity": "sha512-7fyCp2uk0BdeHKJ9PyQOCditC9vBXeeIjYPAKKBcrkum5bi1e9txy2g5kkGjqwUkN0ntIniS5QfHEyr17Idr9g==",
      "cpu": [
        "x64"
      ],
@@ -617,10 +630,30 @@
        "linux"
      ]
    },
    "node_modules/@prebuilt-tdlib/linux-x64-musl": {
      "version": "0.1008064.0",
      "resolved": "https://registry.npmjs.org/@prebuilt-tdlib/linux-x64-musl/-/linux-x64-musl-0.1008064.0.tgz",
      "integrity": "sha512-e2zRucrRrrK6M04iQWMfwtrts+VvVtyUwtTP1hF2g3a6jW+AHMzFoB9Wu8fWr+vuJflLIQ6sG9r3lI07Q8NenQ==",
      "cpu": [
        "x64"
      ],
      "license": "0BSD",
      "optional": true,
      "os": [
        "linux"
      ]
    },
    "node_modules/@prebuilt-tdlib/types": {
      "version": "0.1008064.0",
      "resolved": "https://registry.npmjs.org/@prebuilt-tdlib/types/-/types-0.1008064.0.tgz",
      "integrity": "sha512-eqr1+fiHZ+Gj4lwcITzMp6FwPg8UrxlxxaFjhiJRHL9BlbmD2QkCRHac4wW1Sx8Dzwzd7f+xO21Pgi7TBRSwmw==",
      "license": "0BSD",
      "optional": true
    },
    "node_modules/@prebuilt-tdlib/win32-x64": {
-      "version": "0.1008050.0",
+      "version": "0.1008064.0",
-      "resolved": "https://registry.npmjs.org/@prebuilt-tdlib/win32-x64/-/win32-x64-0.1008050.0.tgz",
+      "resolved": "https://registry.npmjs.org/@prebuilt-tdlib/win32-x64/-/win32-x64-0.1008064.0.tgz",
-      "integrity": "sha512-4v8tU5bodMcLhzrWWXzIzqdHBIpq0wim+7sDmQWQIMy3kDeIzVtpuM+vQjxrGoeH9oWr2WXSRKuj93ld7G5NbQ==",
+      "integrity": "sha512-rkacZWexQw52/EUaLAmbsu2+P3C1/AtinlCjfiX07oQAEg3327BCEZqrcY0ER83D8+MMf2pfwMPCDJKytr4hcg==",
      "cpu": [
        "x64"
      ],
@@ -1669,16 +1702,19 @@
      }
    },
    "node_modules/prebuilt-tdlib": {
-      "version": "0.1008050.0",
+      "version": "0.1008064.0",
-      "resolved": "https://registry.npmjs.org/prebuilt-tdlib/-/prebuilt-tdlib-0.1008050.0.tgz",
+      "resolved": "https://registry.npmjs.org/prebuilt-tdlib/-/prebuilt-tdlib-0.1008064.0.tgz",
-      "integrity": "sha512-CfeQE1rG51d2iC6m72fzrbCW4mqI17ugil9pVurWHtfUJi1Fcn7zadpTzDoUl4oc1dEtKgM7S24DVP67gcl4SQ==",
+      "integrity": "sha512-jJLowKZoH4slXYrkTkKlEgyGsIGv61AWjDZcxxVxJYu21X3kmukGwbCpk4ML99cJp2CwRsD41GCEQBkKJAwCUg==",
      "license": "MIT",
      "optionalDependencies": {
-        "@prebuilt-tdlib/darwin-arm64": "0.1008050.0",
+        "@prebuilt-tdlib/darwin-arm64": "0.1008064.0",
-        "@prebuilt-tdlib/darwin-x64": "0.1008050.0",
+        "@prebuilt-tdlib/darwin-x64": "0.1008064.0",
-        "@prebuilt-tdlib/linux-arm64-glibc": "0.1008050.0",
+        "@prebuilt-tdlib/linux-arm64-glibc": "0.1008064.0",
-        "@prebuilt-tdlib/linux-x64-glibc": "0.1008050.0",
+        "@prebuilt-tdlib/linux-arm64-musl": "0.1008064.0",
-        "@prebuilt-tdlib/win32-x64": "0.1008050.0"
+        "@prebuilt-tdlib/linux-x64-glibc": "0.1008064.0",
        "@prebuilt-tdlib/linux-x64-musl": "0.1008064.0",
        "@prebuilt-tdlib/types": "0.1008064.0",
        "@prebuilt-tdlib/win32-x64": "0.1008064.0"
      }
    },
    "node_modules/prisma": {
@@ -1971,13 +2007,13 @@
      "license": "MIT"
    },
    "node_modules/tdl": {
-      "version": "8.0.2",
+      "version": "8.1.0",
-      "resolved": "https://registry.npmjs.org/tdl/-/tdl-8.0.2.tgz",
+      "resolved": "https://registry.npmjs.org/tdl/-/tdl-8.1.0.tgz",
-      "integrity": "sha512-KYxlJ4eao7FUu91U1dCDkaHmK70JAyZ1KqitkKqpPC7rxAiXWhaYxddWvt84UxIYoWbgdd0B70FYJ4p/YqpFCA==",
+      "integrity": "sha512-idpw60gjJdiJALQg0+6UbxtJTMxVhzZAgCO6QzL81gqBYCkEFjm9zM9HwTTQGeOaAavw4yRHymR68yUUiCoKrA==",
      "hasInstallScript": true,
      "license": "MIT",
      "dependencies": {
-        "debug": "^4.4.0",
+        "debug": "^4.4.3",
        "node-addon-api": "^7.1.1",
        "node-gyp-build": "^4.8.4"
      },
@@ -13,8 +13,8 @@
    "@prisma/client": "^7.4.0",
    "pg": "^8.18.0",
    "pino": "^9.6.0",
-    "prebuilt-tdlib": "^0.1008050.0",
+    "prebuilt-tdlib": "^0.1008064.0",
-    "tdl": "^8.0.0"
+    "tdl": "^8.1.0"
  },
  "devDependencies": {
    "@types/node": "^20",
@@ -28,6 +28,8 @@ services:
      timeout: 5s
      retries: 3
      start_period: 60s
    volumes:
      - manual_uploads:/data/uploads
    restart: unless-stopped
    deploy:
      resources:
@@ -54,6 +56,7 @@ services:
    volumes:
      - tdlib_state:/data/tdlib
      - tmp_zips:/tmp/zips
      - manual_uploads:/data/uploads
    depends_on:
      db:
        condition: service_healthy
@@ -94,6 +97,35 @@ services:
    networks:
      - backend
  backup:
    build:
      context: .
      dockerfile: backup/Dockerfile
    pull_policy: never
    environment:
      - POSTGRES_USER=${POSTGRES_USER:-dragons}
      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-stash}
      - PGPASSWORD=${POSTGRES_PASSWORD:-stash}
      - POSTGRES_DB=${POSTGRES_DB:-dragonsstash}
      - RESTIC_REPOSITORY=/backups/restic-repo
      - RESTIC_PASSWORD=${RESTIC_PASSWORD:?Set RESTIC_PASSWORD in .env}
      - KUMA_PUSH_URL=${KUMA_PUSH_URL:-}
      - TZ=${TZ:-Etc/UTC}
    volumes:
      - tdlib_state:/data/tdlib-worker:ro
      - tdlib_bot_state:/data/tdlib-bot:ro
      - nas_backups:/backups
    depends_on:
      db:
        condition: service_healthy
    restart: unless-stopped
    deploy:
      resources:
        limits:
          memory: 1G
    networks:
      - backend
  db:
    image: postgres:16-alpine
    environment:
@@ -113,14 +145,22 @@ services:
        limits:
          memory: 1G
    networks:
-      - frontend
+      frontend: {}
-      - backend
+      backend:
        aliases:
          - dragonsstash-db
 volumes:
  postgres_data:
  tdlib_state:
  tdlib_bot_state:
  tmp_zips:
  manual_uploads:
  nas_backups:
    driver_opts:
      type: cifs
      o: "username=${NAS_USERNAME},password=${NAS_PASSWORD},vers=3.0,uid=0,gid=0,file_mode=0660,dir_mode=0770"
      device: "//${NAS_HOST}/${NAS_SHARE}"
 networks:
  frontend:
@@ -0,0 +1,454 @@
 # Send All From Creator Implementation Plan
 > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
 **Goal:** Add a toolbar button on the STL packages page that queues every sendable package from the currently-filtered creator for delivery to the user's Telegram.
 **Architecture:** A new server action `sendAllFromCreatorAction(creatorName)` mirrors the existing `sendAllInGroupAction`: it fetches sendable packages for the creator, dedups against live `BotSendRequest`s, creates a `BotSendRequest` per package, and fires `pg_notify('bot_send', id)`. The bot's existing `send-listener` consumes these unchanged. The client table renders a "Send all from [Creator]" button in the packages-tab toolbar only when `?creator=<name>` is active, wired to the action with a confirm dialog and a count toast.
 **Tech Stack:** Next.js 16 App Router, TypeScript, Prisma v7, server actions, `pg_notify`, sonner toasts, lucide-react icons.
 **Testing note:** This repo has no test framework (`CLAUDE.md`: "Testing is manual"). Verification for each task is `npm run lint` + `npm run build`, plus manual UI checks at the end. Do not scaffold a test harness.
 ---
 ### Task 1: Add `sendAllFromCreatorAction` server action
 **Files:**
 - Modify: `src/app/(app)/stls/actions.ts` (append new action after `sendAllInGroupAction`, which ends at line 591)
 Reference implementation to mirror: `sendAllInGroupAction` at `src/app/(app)/stls/actions.ts:515-591`.
 Existing imports already present in the file (do not re-add): `auth` from `@/lib/auth`, `prisma` from `@/lib/prisma`, `ActionResult` from `@/types/api.types`, `revalidatePath` from `next/cache`.
 `ActionResult` is generic: `ActionResult<T = void>` (`src/types/api.types.ts`).
 - [ ] **Step 1: Append the new action**
 Add this to the end of `src/app/(app)/stls/actions.ts`:
 ```ts
 export async function sendAllFromCreatorAction(
  creatorName: string
 ): Promise<ActionResult<{ queued: number; skipped: number }>> {
  const session = await auth();
  if (!session?.user?.id) return { success: false, error: "Unauthorized" };
  const creator = creatorName.trim();
  if (!creator) {
    return { success: false, error: "No creator specified" };
  }
  try {
    const telegramLink = await prisma.telegramLink.findUnique({
      where: { userId: session.user.id },
    });
    if (!telegramLink) {
      return { success: false, error: "No linked Telegram account. Link one in Settings." };
    }
    const sendablePackages = await prisma.package.findMany({
      where: {
        creator,
        destChannelId: { not: null },
        destMessageId: { not: null },
      },
      select: { id: true },
    });
    if (sendablePackages.length === 0) {
      return { success: false, error: "No uploaded packages found for this creator" };
    }
    let queued = 0;
    let skipped = 0;
    for (const pkg of sendablePackages) {
      // Only create if no existing PENDING/SENDING request for this package+link combo
      const existing = await prisma.botSendRequest.findFirst({
        where: {
          packageId: pkg.id,
          telegramLinkId: telegramLink.id,
          status: { in: ["PENDING", "SENDING"] },
        },
      });
      if (existing) {
        skipped++;
        continue;
      }
      const sendRequest = await prisma.botSendRequest.create({
        data: {
          packageId: pkg.id,
          telegramLinkId: telegramLink.id,
          requestedByUserId: session.user.id,
          status: "PENDING",
        },
      });
      // Notify the bot via pg_notify
      try {
        await prisma.$queryRawUnsafe(
          `SELECT pg_notify('bot_send', $1)`,
          sendRequest.id
        );
      } catch {
        // Best-effort — the bot also polls periodically
      }
      queued++;
    }
    revalidatePath("/stls");
    return { success: true, data: { queued, skipped } };
  } catch {
    return { success: false, error: "Failed to send creator packages" };
  }
 }
 ```
 - [ ] **Step 2: Verify lint + typecheck pass**
 Run: `npm run lint`
 Expected: no new errors in `src/app/(app)/stls/actions.ts`.
 - [ ] **Step 3: Commit**
 ```bash
 git add src/app/\(app\)/stls/actions.ts
 git commit -m "feat(stls): add sendAllFromCreatorAction server action"
 ```
 ---
 ### Task 2: Wire up the toolbar button in the STL table
 **Files:**
 - Modify: `src/app/(app)/stls/_components/stl-table.tsx`
  - lucide import (line 6)
  - actions import block (lines 44-54)
  - handler (after `handleSendAllInGroup`, which ends at line 278)
  - `activeCreator` derivation (near `activeTag` at line 445)
  - toolbar JSX (packages-tab toolbar row starting at line 478)
 - [ ] **Step 1: Add the `Send` icon to the lucide import**
 Change line 6 from:
 ```ts
 import { Search, Layers, Upload } from "lucide-react";
 ```
 to:
 ```ts
 import { Search, Layers, Upload, Send } from "lucide-react";
 ```
 - [ ] **Step 2: Import the new action**
 In the actions import block (lines 44-54), add `sendAllFromCreatorAction` next to `sendAllInGroupAction`:
 ```ts
 import {
  updatePackageCreator,
  updatePackageTags,
  renameGroupAction,
  dissolveGroupAction,
  createGroupAction,
  removeFromGroupAction,
  sendAllInGroupAction,
  sendAllFromCreatorAction,
  updateGroupPreviewAction,
  mergeGroupsAction,
 } from "../actions";
 ```
 - [ ] **Step 3: Derive the active creator from the URL**
 Immediately after the `activeTag` line (`src/app/(app)/stls/_components/stl-table.tsx:445`):
 ```ts
  const activeTag = searchParams.get("tag") ?? "";
 ```
 add:
 ```ts
  const activeCreator = searchParams.get("creator") ?? "";
 ```
 - [ ] **Step 4: Add the click handler**
 After `handleSendAllInGroup` (which ends at line 278), add:
 ```ts
  const handleSendAllFromCreator = useCallback(() => {
    if (!confirm(`Send all packages from "${activeCreator}" to your Telegram?`)) return;
    startTransition(async () => {
      const result = await sendAllFromCreatorAction(activeCreator);
      if (result.success) {
        const { queued, skipped } = result.data;
        toast.success(
          `Queued ${queued} package${queued === 1 ? "" : "s"} from ${activeCreator}` +
            (skipped ? ` (${skipped} already queued)` : "")
        );
        router.refresh();
      } else {
        toast.error(result.error);
      }
    });
  }, [activeCreator, router]);
 ```
 - [ ] **Step 5: Render the toolbar button**
 In the packages-tab toolbar (`flex flex-wrap items-center gap-2` row at line 478), add the button right after the "Upload Files" button (which closes at line 507, before the `selectedPackages.size >= 2` block at line 508):
 ```tsx
            {activeCreator && (
              <Button
                variant="outline"
                size="sm"
                className="h-9 gap-1.5"
                onClick={handleSendAllFromCreator}
              >
                <Send className="h-3.5 w-3.5" />
                Send all from {activeCreator}
              </Button>
            )}
 ```
 - [ ] **Step 6: Verify lint + build pass**
 Run: `npm run lint && npm run build`
 Expected: no new errors; build completes.
 - [ ] **Step 7: Manual verification**
 1. `npm run dev`, open `/stls`.
 2. Click a creator name in the Creator column to apply `?creator=<name>` (or navigate to `/stls?creator=<known creator>`).
 3. Confirm the "Send all from [Creator]" button appears in the toolbar.
 4. Remove the creator filter → confirm the button disappears.
 5. Click the button → confirm the dialog appears; on confirm, a toast reports the queued count.
 6. (If a linked Telegram account + uploaded packages exist) confirm packages arrive via the bot.
 - [ ] **Step 8: Commit**
 ```bash
 git add src/app/\(app\)/stls/_components/stl-table.tsx
 git commit -m "feat(stls): add 'send all from creator' toolbar button"
 ```
 ---
 ### Task 3: Searchable creator filter combobox (makes the filter reachable)
 **Why:** Discovered during execution — the `?creator=` filter that reveals the Task 2
 button had no UI trigger. The creator table cell click only opens the edit prompt
 (`package-columns.tsx:339` → `onSetCreator`). This task adds a searchable
 "All Creators" combobox to the packages-tab toolbar (like the existing Tags select,
 but type-to-filter since there can be many creators). The creator cell stays
 edit-only.
 **Files:**
 - Modify: `src/lib/telegram/queries.ts` — add `getAllPackageCreators()`.
 - Modify: `src/app/(app)/stls/page.tsx` — fetch + pass `availableCreators`.
 - Create: `src/app/(app)/stls/_components/creator-filter.tsx` — the combobox.
 - Modify: `src/app/(app)/stls/_components/stl-table.tsx` — new prop, handler, render.
 - [ ] **Step 1: Add the distinct-creators query**
 Append to `src/lib/telegram/queries.ts` (mirrors `getAllPackageTags` at line 530):
 ```ts
 export async function getAllPackageCreators(): Promise<string[]> {
  const result = await prisma.$queryRaw<{ creator: string }[]>`
    SELECT DISTINCT creator FROM packages
    WHERE creator IS NOT NULL AND creator <> ''
    ORDER BY creator
  `;
  return result.map((r) => r.creator);
 }
 ```
 - [ ] **Step 2: Create the combobox component**
 Create `src/app/(app)/stls/_components/creator-filter.tsx` (mirrors the
 Popover+Command pattern in `src/components/shared/data-table-faceted-filter.tsx`):
 ```tsx
 "use client";
 import { useState } from "react";
 import { Check, ChevronsUpDown } from "lucide-react";
 import { cn } from "@/lib/utils";
 import { Button } from "@/components/ui/button";
 import {
  Command,
  CommandEmpty,
  CommandGroup,
  CommandInput,
  CommandItem,
  CommandList,
 } from "@/components/ui/command";
 import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover";
 interface CreatorFilterProps {
  creators: string[];
  value: string; // active creator, "" when none
  onChange: (creator: string) => void; // "" clears the filter
 }
 export function CreatorFilter({ creators, value, onChange }: CreatorFilterProps) {
  const [open, setOpen] = useState(false);
  return (
    <Popover open={open} onOpenChange={setOpen}>
      <PopoverTrigger asChild>
        <Button
          variant="outline"
          size="sm"
          role="combobox"
          aria-expanded={open}
          className="h-9 w-[200px] justify-between"
        >
          <span className="truncate">{value || "All Creators"}</span>
          <ChevronsUpDown className="ml-2 h-4 w-4 shrink-0 opacity-50" />
        </Button>
      </PopoverTrigger>
      <PopoverContent className="w-[240px] p-0" align="start">
        <Command>
          <CommandInput placeholder="Search creators..." className="h-9" />
          <CommandList>
            <CommandEmpty>No creators found.</CommandEmpty>
            <CommandGroup>
              <CommandItem
                value="__all__"
                onSelect={() => {
                  onChange("");
                  setOpen(false);
                }}
              >
                <Check
                  className={cn("mr-2 h-4 w-4", value === "" ? "opacity-100" : "opacity-0")}
                />
                All Creators
              </CommandItem>
              {creators.map((creator) => (
                <CommandItem
                  key={creator}
                  value={creator}
                  onSelect={() => {
                    onChange(creator);
                    setOpen(false);
                  }}
                >
                  <Check
                    className={cn(
                      "mr-2 h-4 w-4",
                      value === creator ? "opacity-100" : "opacity-0"
                    )}
                  />
                  <span className="truncate">{creator}</span>
                </CommandItem>
              ))}
            </CommandGroup>
          </CommandList>
        </Command>
      </PopoverContent>
    </Popover>
  );
 }
 ```
 - [ ] **Step 3: Wire the query through the page**
 In `src/app/(app)/stls/page.tsx`:
 - Add `getAllPackageCreators` to the import from `@/lib/telegram/queries` (line 3).
 - Add `getAllPackageCreators()` to the `Promise.all` (line 27) and destructure
  `availableCreators`:
  ```ts
  const [result, ingestionStatus, availableTags, availableCreators, skippedCount, ungroupedCount] =
    await Promise.all([
      // ...existing entries unchanged...
      getIngestionStatus(),
      getAllPackageTags(),
      getAllPackageCreators(),
      countSkippedPackages(),
      countUngroupedPackages(),
    ]);
  ```
  (Insert `getAllPackageCreators()` immediately after `getAllPackageTags()`, and add
  `availableCreators` in the matching position of the destructure.)
 - Pass the prop to `<StlTable>`: `availableCreators={availableCreators}` (next to
  `availableTags={availableTags}`).
 - [ ] **Step 4: Add prop + handler + render in the table**
 In `src/app/(app)/stls/_components/stl-table.tsx`:
 - Import the component: `import { CreatorFilter } from "./creator-filter";`
 - Add to `StlTableProps` (next to `availableTags: string[];`): `availableCreators: string[];`
 - Add to the destructured params (next to `availableTags,`): `availableCreators,`
 - Add the handler after `updateTagFilter` (ends ~line 210):
  ```ts
  const updateCreatorFilter = useCallback(
    (value: string) => {
      const params = new URLSearchParams(searchParams.toString());
      if (value) {
        params.set("creator", value);
        params.set("page", "1");
      } else {
        params.delete("creator");
      }
      router.push(`${pathname}?${params.toString()}`, { scroll: false });
    },
    [router, pathname, searchParams]
  );
  ```
 - Render it in the toolbar right after the Tags `Select` block (the
  `{availableTags.length > 0 && (...)}` block, ~lines 507-522):
  ```tsx
  {availableCreators.length > 0 && (
    <CreatorFilter
      creators={availableCreators}
      value={activeCreator}
      onChange={updateCreatorFilter}
    />
  )}
  ```
 - [ ] **Step 5: Verify + manual test**
 Run: `npx tsc --noEmit` (filter for the touched files — expect none new). Note: full
 `npm run build` fails on a pre-existing stale Prisma client issue in
 `src/lib/telegram/*` unrelated to this change; do not attempt to fix it.
 Manual: open `/stls`, use the "All Creators" combobox, type to filter, pick a
 creator → list filters and the "Send all from [Creator]" button appears; pick
 "All Creators" → filter clears and button disappears.
 - [ ] **Step 6: Commit**
 ```bash
 git add "src/lib/telegram/queries.ts" "src/app/(app)/stls/page.tsx" \
  "src/app/(app)/stls/_components/creator-filter.tsx" \
  "src/app/(app)/stls/_components/stl-table.tsx"
 git commit -m "feat(stls): add searchable creator filter combobox to toolbar"
 ```
 ---
 ## Self-Review
 **Spec coverage:**
 - Req 1 (button when filtered by creator) → Task 2 Step 5 (`{activeCreator && ...}`).
 - Req 2 (hidden when no filter) → Task 2 Step 5 conditional.
 - Req 3 (confirm dialog) → Task 2 Step 4 `confirm(...)`.
 - Req 4 (all pages, not current page) → Task 1 queries `prisma.package.findMany` by `creator`, unpaginated.
 - Req 5 (skip not-uploaded + dedup live requests) → Task 1 `where` filter on `destChannelId`/`destMessageId` + `existing` check.
 - Req 6 (report counts) → Task 1 returns `{ queued, skipped }`; Task 2 Step 4 toast.
 - Req 7 (no polling) → Task 2 handler queues + `router.refresh()`, no poll loop.
 - Blank-creator guard → Task 1 `creator.trim()` check.
 **Placeholder scan:** No TBD/TODO/placeholder steps; all code shown in full.
 **Type consistency:** Action returns `ActionResult<{ queued: number; skipped: number }>`; handler destructures `result.data.{queued,skipped}` inside the `result.success` branch (where `data` is typed). `sendAllFromCreatorAction` name matches between Task 1 definition, Task 2 import, and Task 2 call site. `activeCreator` defined once (Step 3) and used in Steps 4-5.
@@ -0,0 +1,555 @@
 # NAS Backup for Postgres + TDLib State Implementation Plan
 > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
 **Goal:** Add a `backup` container to the DragonsStash stack that takes daily, encrypted, deduplicated backups of the Postgres database and both TDLib state volumes, and ships them to a Synology NAS over NFS.
 **Architecture:** A small Alpine-based image (restic + postgresql16-client + curl + dcron) runs as its own compose service. A crontab fires `backup.sh` daily at 03:00, which dumps Postgres, tars the TDLib volumes, hands both to `restic backup` against an NFS-backed Docker volume, prunes with `restic forget --keep-daily 14`, and reports success/failure to an Uptime Kuma push monitor. Matches the existing `worker`/`bot` pattern: build context in the repo's `docker-compose.yml`, prebuilt image in `/opt/stacks/DragonsStash/docker-compose.yml`, built and pushed by `.drone.yml`.
 **Tech Stack:** Alpine 3.20, restic 0.16, postgresql16-client, dcron, bash, Docker Compose NFS volume driver.
 ## Global Constraints
 - Retention: `restic forget --keep-daily 14 --prune` (14-day window, per approved spec).
 - Schedule: daily backup at 03:00, weekly `restic check` at 04:00 Sunday.
 - No Docker socket mount, no `privileged: true` — the backup container must not be able to control sibling containers.
 - No host-level mount — NFS access only via Docker's native `driver_opts: type: nfs` volume, never `/etc/fstab`.
 - Encryption and retention are restic's job — no hand-rolled `age`/`gpg`/`find -mtime` logic.
 - TDLib volumes are tarred live (best-effort) — never pause `worker`/`bot` for the backup.
 - Restore is a manual, documented procedure only — never scripted/automated.
 **Required user input before Task 5 can run:** `NAS_HOST` and `NAS_EXPORT_PATH` (the Synology NFS share details) and a Kuma Push-monitor URL (`KUMA_PUSH_URL`, created manually in the existing Uptime Kuma instance, ~26h expected heartbeat interval). Tasks 1–4 need none of these and can proceed immediately; do not substitute placeholder values for them in Task 5 — stop and ask the user instead.
 ---
 ### Task 1: Backup image (Dockerfile + entrypoint)
 **Files:**
 - Create: `backup/Dockerfile`
 - Create: `backup/entrypoint.sh`
 - Create: `backup/crontab`
 **Interfaces:**
 - Produces: a buildable image tagged `dragonsstash-backup:test` locally, with `/entrypoint.sh` as `ENTRYPOINT`, `/backup.sh` present at the image root (written in Task 2 — this task only needs the `COPY` line and a placeholder-free stub isn't acceptable, so create an empty‑body-but-real `backup/backup.sh` here containing just `#!/bin/bash` + `exit 0`, and Task 2 replaces its contents), `restic`, `pg_dump`/`pg_restore`, `curl`, `tar`, `bash`, `dcron` all on `PATH`.
 - Consumes: nothing from earlier tasks.
 - [ ] **Step 1: Write `backup/backup.sh` stub**
 ```bash
 #!/bin/bash
 set -euo pipefail
 exit 0
 ```
 - [ ] **Step 2: Write `backup/entrypoint.sh`**
 ```bash
 #!/bin/bash
 set -euo pipefail
 if ! restic snapshots >/dev/null 2>&1; then
  restic init
 fi
 exec crond -f -l 2
 ```
 - [ ] **Step 3: Write `backup/crontab`**
 ```
 0 3 * * * /backup.sh >> /proc/1/fd/1 2>&1
 0 4 * * 0 restic check >> /proc/1/fd/1 2>&1
 ```
 - [ ] **Step 4: Write `backup/Dockerfile`**
 ```dockerfile
 FROM alpine:3.20
 RUN apk add --no-cache restic postgresql16-client curl tzdata dcron tar bash
 COPY backup/backup.sh /backup.sh
 COPY backup/entrypoint.sh /entrypoint.sh
 COPY backup/crontab /etc/crontabs/root
 RUN chmod +x /backup.sh /entrypoint.sh
 ENTRYPOINT ["/entrypoint.sh"]
 ```
 - [ ] **Step 5: Build the image**
 Run: `cd /home/sam/Documents/DragonsStash && docker build -t dragonsstash-backup:test -f backup/Dockerfile .`
 Expected: build completes with `Successfully tagged dragonsstash-backup:test` (or Buildkit's equivalent final `naming to docker.io/library/dragonsstash-backup:test done`), no errors.
 - [ ] **Step 6: Verify the tools are present**
 Run: `docker run --rm dragonsstash-backup:test restic version && docker run --rm dragonsstash-backup:test pg_dump --version`
 Expected: `restic 0.16.x ...` and `pg_dump (PostgreSQL) 16.x` printed, both commands exit 0.
 - [ ] **Step 7: Verify the entrypoint initializes an empty repo and starts cron**
 ```bash
 mkdir -p /tmp/backup-repo-smoke
 docker run -d --name backup-smoke \
  -e RESTIC_REPOSITORY=/backups/restic-repo -e RESTIC_PASSWORD=smoketest \
  -v /tmp/backup-repo-smoke:/backups \
  dragonsstash-backup:test
 sleep 2
 docker logs backup-smoke
 docker exec backup-smoke restic snapshots
 docker rm -f backup-smoke
 rm -rf /tmp/backup-repo-smoke
 ```
 Expected: `docker logs` shows no errors (restic init ran silently); `restic snapshots` prints an empty snapshot list (repo exists, header row only, no error).
 - [ ] **Step 8: Commit**
 ```bash
 git add backup/Dockerfile backup/entrypoint.sh backup/backup.sh backup/crontab
 git commit -m "Add backup service image (Dockerfile, entrypoint, crontab)"
 ```
 ---
 ### Task 2: `backup.sh` script
 **Files:**
 - Modify: `backup/backup.sh` (replace Task 1's stub with the real script)
 **Interfaces:**
 - Consumes: the image built in Task 1 (`dragonsstash-backup:test`), rebuilt after this change.
 - Produces: `/backup.sh`, invoked by cron in Task 1's `crontab` and manually in Task 6's verification. Reads env vars `POSTGRES_USER`, `PGPASSWORD`, `POSTGRES_DB`, `RESTIC_REPOSITORY`, `RESTIC_PASSWORD`, `KUMA_PUSH_URL`. Assumes network hostname `dragonsstash-db:5432` for Postgres and mounts `/data/tdlib-worker`, `/data/tdlib-bot` (read-only) for TDLib state.
 - [ ] **Step 1: Replace `backup/backup.sh` with the real script**
 ```bash
 #!/bin/bash
 set -euo pipefail
 report_failure() {
  curl -fsS "$KUMA_PUSH_URL" --get \
    --data-urlencode "status=down" \
    --data-urlencode "msg=$BASH_COMMAND failed" || true
 }
 trap report_failure ERR
 DUMP_FILE=/tmp/dragonsstash.dump
 TAR_FILE=/tmp/tdlib.tar.gz
 pg_dump -h dragonsstash-db -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Fc -f "$DUMP_FILE"
 tar czf "$TAR_FILE" -C /data tdlib-worker tdlib-bot
 restic backup "$DUMP_FILE" "$TAR_FILE"
 restic forget --keep-daily 14 --prune
 rm -f "$DUMP_FILE" "$TAR_FILE"
 curl -fsS "$KUMA_PUSH_URL" --get \
  --data-urlencode "status=up" \
  --data-urlencode "msg=OK"
 ```
 - [ ] **Step 2: Rebuild the image**
 Run: `docker build -t dragonsstash-backup:test -f backup/Dockerfile .`
 Expected: build succeeds.
 - [ ] **Step 3: Stand up a scratch Postgres aliased as `dragonsstash-db`**
 ```bash
 docker network create backup-test-net 2>/dev/null || true
 docker run -d --name test-pg --network backup-test-net --network-alias dragonsstash-db \
  -e POSTGRES_USER=dragons -e POSTGRES_PASSWORD=stash -e POSTGRES_DB=dragonsstash \
  postgres:16-alpine
 sleep 5
 docker exec test-pg pg_isready -U dragons -d dragonsstash
 ```
 Expected: `accepting connections`.
 - [ ] **Step 4: Stand up a mock Kuma push endpoint**
 ```bash
 mkdir -p /tmp/mock-kuma-root && touch /tmp/mock-kuma-root/push
 docker run -d --name mock-kuma --network backup-test-net \
  -v /tmp/mock-kuma-root:/srv -w /srv python:3-alpine \
  python3 -m http.server 8000
 sleep 1
 ```
 - [ ] **Step 5: Prepare fake TDLib state and a local restic repo dir**
 ```bash
 mkdir -p /tmp/backup-test/tdlib-worker /tmp/backup-test/tdlib-bot /tmp/backup-test/repo
 echo "fake-session" > /tmp/backup-test/tdlib-worker/state.bin
 echo "fake-session" > /tmp/backup-test/tdlib-bot/state.bin
 ```
 - [ ] **Step 6: Initialize the test restic repo and run `backup.sh` (success path)**
 ```bash
 docker run --rm --network backup-test-net \
  -e RESTIC_REPOSITORY=/backups/restic-repo -e RESTIC_PASSWORD=testpassword \
  -v /tmp/backup-test/repo:/backups \
  --entrypoint restic dragonsstash-backup:test init
 docker run --rm --network backup-test-net \
  -e POSTGRES_USER=dragons -e POSTGRES_PASSWORD=stash -e PGPASSWORD=stash -e POSTGRES_DB=dragonsstash \
  -e RESTIC_REPOSITORY=/backups/restic-repo -e RESTIC_PASSWORD=testpassword \
  -e KUMA_PUSH_URL=http://mock-kuma:8000/push \
  -v /tmp/backup-test/tdlib-worker:/data/tdlib-worker:ro \
  -v /tmp/backup-test/tdlib-bot:/data/tdlib-bot:ro \
  -v /tmp/backup-test/repo:/backups \
  --entrypoint /backup.sh dragonsstash-backup:test
 echo "exit code: $?"
 ```
 Expected: exit code `0`, restic prints a line like `snapshot xxxxxxxx saved`, no error output.
 - [ ] **Step 7: Verify the snapshot landed and contains both files**
 ```bash
 docker run --rm -v /tmp/backup-test/repo:/backups \
  -e RESTIC_REPOSITORY=/backups/restic-repo -e RESTIC_PASSWORD=testpassword \
  --entrypoint restic dragonsstash-backup:test snapshots
 docker run --rm -v /tmp/backup-test/repo:/backups \
  -e RESTIC_REPOSITORY=/backups/restic-repo -e RESTIC_PASSWORD=testpassword \
  --entrypoint restic dragonsstash-backup:test ls latest
 ```
 Expected: `snapshots` shows exactly one entry; `ls latest` lists `/tmp/dragonsstash.dump` and `/tmp/tdlib.tar.gz`.
 - [ ] **Step 8: Verify the failure path reports to Kuma**
 ```bash
 docker run --rm --network backup-test-net \
  -e POSTGRES_USER=dragons -e POSTGRES_PASSWORD=wrongpass -e PGPASSWORD=wrongpass -e POSTGRES_DB=dragonsstash \
  -e RESTIC_REPOSITORY=/backups/restic-repo -e RESTIC_PASSWORD=testpassword \
  -e KUMA_PUSH_URL=http://mock-kuma:8000/push \
  -v /tmp/backup-test/tdlib-worker:/data/tdlib-worker:ro \
  -v /tmp/backup-test/tdlib-bot:/data/tdlib-bot:ro \
  -v /tmp/backup-test/repo:/backups \
  --entrypoint /backup.sh dragonsstash-backup:test
 echo "exit code: $?"
 docker logs mock-kuma | tail -5
 ```
 Expected: exit code nonzero (pg_dump auth failure trips `set -e`); `docker logs mock-kuma` shows a GET request line containing `status=down`.
 - [ ] **Step 9: Clean up test resources**
 ```bash
 docker rm -f test-pg mock-kuma
 docker network rm backup-test-net
 rm -rf /tmp/backup-test /tmp/mock-kuma-root
 ```
 - [ ] **Step 10: Commit**
 ```bash
 git add backup/backup.sh
 git commit -m "Implement backup.sh: pg_dump + tdlib tar + restic backup/forget + Kuma reporting"
 ```
 ---
 ### Task 3: Wire the `backup` service into the repo's `docker-compose.yml`
 **Files:**
 - Modify: `docker-compose.yml`
 - Modify: `.env.example`
 **Interfaces:**
 - Consumes: `backup/Dockerfile` (Task 1), `backup/backup.sh` (Task 2).
 - Produces: a `backup` compose service buildable via `docker compose build backup`, and a `nas_backups` named volume other tasks (5) will mirror into the production compose file.
 - [ ] **Step 1: Add the `backup` service to `docker-compose.yml`**
 Insert after the existing `bot` service (before `db`):
 ```yaml
  backup:
    build:
      context: .
      dockerfile: backup/Dockerfile
    pull_policy: never
    environment:
      - POSTGRES_USER=${POSTGRES_USER:-dragons}
      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-stash}
      - PGPASSWORD=${POSTGRES_PASSWORD:-stash}
      - POSTGRES_DB=${POSTGRES_DB:-dragonsstash}
      - RESTIC_REPOSITORY=/backups/restic-repo
      - RESTIC_PASSWORD=${RESTIC_PASSWORD:?Set RESTIC_PASSWORD in .env}
      - KUMA_PUSH_URL=${KUMA_PUSH_URL:?Set KUMA_PUSH_URL in .env}
      - TZ=${TZ:-Etc/UTC}
    volumes:
      - tdlib_state:/data/tdlib-worker:ro
      - tdlib_bot_state:/data/tdlib-bot:ro
      - nas_backups:/backups
    depends_on:
      db:
        condition: service_healthy
    restart: unless-stopped
    deploy:
      resources:
        limits:
          memory: 256M
    networks:
      - backend
 ```
 - [ ] **Step 2: Add the `nas_backups` volume to the `volumes:` block**
 ```yaml
  nas_backups:
    driver_opts:
      type: nfs
      o: "addr=${NAS_HOST},rw,nfsvers=4,soft,timeo=100"
      device: ":${NAS_EXPORT_PATH}"
 ```
 - [ ] **Step 3: Document the new env vars in `.env.example`**
 Append:
 ```
 # Backup (NAS via NFS + restic)
 NAS_HOST=""              # Synology NAS IP or hostname reachable from this host
 NAS_EXPORT_PATH=""       # NFS export path, e.g. /volume1/dragonsstash-backups
 RESTIC_PASSWORD=""       # generate with: openssl rand -base64 32
 KUMA_PUSH_URL=""         # Uptime Kuma Push monitor URL (create the monitor first)
 TZ="Etc/UTC"
 ```
 - [ ] **Step 4: Validate the compose file parses**
 Run (with dummy values so the `:?` guards don't fail parsing — `AUTH_SECRET` is required by the existing `app` service, not by this change, but `config` validates the whole file):
 ```bash
 RESTIC_PASSWORD=dummy KUMA_PUSH_URL=http://dummy NAS_HOST=dummy NAS_EXPORT_PATH=/dummy AUTH_SECRET=dummy \
  docker compose -f docker-compose.yml config --quiet
 ```
 Expected: no output, exit code 0 (a syntax/interpolation error would print to stderr and exit nonzero).
 - [ ] **Step 5: Validate the service actually builds through Compose**
 Run:
 ```bash
 RESTIC_PASSWORD=dummy KUMA_PUSH_URL=http://dummy NAS_HOST=dummy NAS_EXPORT_PATH=/dummy AUTH_SECRET=dummy \
  docker compose -f docker-compose.yml build backup
 ```
 Expected: build succeeds (reuses Task 1's image layers).
 - [ ] **Step 6: Commit**
 ```bash
 git add docker-compose.yml .env.example
 git commit -m "Add backup service and nas_backups volume to docker-compose.yml"
 ```
 ---
 ### Task 4: CI — build and push the backup image
 **Files:**
 - Modify: `.drone.yml`
 **Interfaces:**
 - Consumes: `backup/Dockerfile` (Task 1).
 - Produces: `git.samagsteribbe.nl/admin/dragonsstash-backup:latest` (and `:<short-sha>`), pushed on every push to `main`. Task 5's production compose file references this image tag.
 - [ ] **Step 1: Add a `build-backup` step, mirroring `build-worker`/`build-bot`**
 Insert after the existing `build-bot` step in `.drone.yml`:
 ```yaml
  - name: build-backup
    image: plugins/docker
    depends_on: [clone]
    settings:
      repo: git.samagsteribbe.nl/admin/dragonsstash-backup
      registry: git.samagsteribbe.nl
      dockerfile: backup/Dockerfile
      tags:
        - latest
        - "${DRONE_COMMIT_SHA:0:8}"
      username:
        from_secret: gitea_username
      password:
        from_secret: gitea_password
 ```
 - [ ] **Step 2: Add `build-backup` to the `deploy` step's `depends_on`**
 Change:
 ```yaml
  - name: deploy
    image: alpine
    depends_on: [build-app, build-worker, build-bot]
 ```
 to:
 ```yaml
  - name: deploy
    image: alpine
    depends_on: [build-app, build-worker, build-bot, build-backup]
 ```
 - [ ] **Step 3: Validate YAML syntax**
 Run: `python3 -c "import yaml; yaml.safe_load(open('.drone.yml')); print('OK')"`
 Expected: `OK` printed, no exception.
 - [ ] **Step 4: Commit**
 ```bash
 git add .drone.yml
 git commit -m "Add build-backup CI step, include it in deploy dependencies"
 ```
 ---
 ### Task 5: Wire production (`/opt/stacks/DragonsStash`) — requires NAS + Kuma details from the user
 **Do not start this task until the user has supplied `NAS_HOST` and `NAS_EXPORT_PATH` for the Synology NFS share, and has created an Uptime Kuma Push monitor (name it `dragonsstash-backup`, ~26h expected heartbeat interval) and shared its push URL. If any of these are missing, stop and ask — do not substitute placeholder values here, since this file drives the real deployment.**
 **Files:**
 - Modify: `/opt/stacks/DragonsStash/docker-compose.yml`
 - Modify: `/opt/stacks/DragonsStash/.env`
 **Interfaces:**
 - Consumes: `git.samagsteribbe.nl/admin/dragonsstash-backup:latest` (published by Task 4's CI step once merged/pushed), the real `NAS_HOST`/`NAS_EXPORT_PATH`/`KUMA_PUSH_URL` values gathered above.
 - Produces: a running `dragonsstash-backup` container on the production host, verified in Task 6.
 - [ ] **Step 1: Generate `RESTIC_PASSWORD` and add all four new vars to `/opt/stacks/DragonsStash/.env`**
 ```bash
 cd /opt/stacks/DragonsStash
 printf '\n# Backup (NAS via NFS + restic)\nNAS_HOST="<value from user>"\nNAS_EXPORT_PATH="<value from user>"\nRESTIC_PASSWORD="%s"\nKUMA_PUSH_URL="<value from user>"\nTZ="Etc/UTC"\n' "$(openssl rand -base64 32)" >> .env
 ```
 Replace the two `<value from user>` placeholders with the real NAS details and Kuma push URL before saving — this step cannot be completed with the literal placeholder text left in place.
 - [ ] **Step 2: Add the `backup` service to `/opt/stacks/DragonsStash/docker-compose.yml`**
 Insert after the existing `bot` service (before `db`):
 ```yaml
  backup:
    image: git.samagsteribbe.nl/admin/dragonsstash-backup:latest
    container_name: dragonsstash-backup
    restart: unless-stopped
    environment:
      - POSTGRES_USER=${POSTGRES_USER:-dragons}
      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-stash}
      - PGPASSWORD=${POSTGRES_PASSWORD:-stash}
      - POSTGRES_DB=${POSTGRES_DB:-dragonsstash}
      - RESTIC_REPOSITORY=/backups/restic-repo
      - RESTIC_PASSWORD=${RESTIC_PASSWORD:?Set RESTIC_PASSWORD in .env}
      - KUMA_PUSH_URL=${KUMA_PUSH_URL:?Set KUMA_PUSH_URL in .env}
      - TZ=${TZ:-Etc/UTC}
    volumes:
      - tdlib_state:/data/tdlib-worker:ro
      - tdlib_bot_state:/data/tdlib-bot:ro
      - nas_backups:/backups
    depends_on:
      db:
        condition: service_healthy
    deploy:
      resources:
        limits:
          memory: 256M
    networks:
      - internal
 ```
 - [ ] **Step 3: Add the `nas_backups` volume**
 ```yaml
  nas_backups:
    driver_opts:
      type: nfs
      o: "addr=${NAS_HOST},rw,nfsvers=4,soft,timeo=100"
      device: ":${NAS_EXPORT_PATH}"
 ```
 - [ ] **Step 4: Validate the production compose file parses with the real `.env`**
 Run: `cd /opt/stacks/DragonsStash && docker compose config --quiet`
 Expected: no output, exit code 0.
 - [ ] **Step 5: Commit is not applicable here** — `/opt/stacks/DragonsStash` is a deployed copy, not the git repo (confirm with `git -C /opt/stacks/DragonsStash status` — expect "not a git repository"). Skip committing; Task 6 deploys these file changes directly.
 ---
 ### Task 6: Deploy and verify
 **Files:** none (operational task)
 **Interfaces:**
 - Consumes: everything from Tasks 1–5.
 - Produces: a running, verified backup on the real NAS, and one completed restore drill.
 - [ ] **Step 1: Confirm with the user before pushing/deploying**
 Pushing to `main` triggers Drone CI to build all four images and deploy to the production host via SSH (`docker compose pull && docker compose up -d`). Confirm the user wants this to happen now before proceeding — this affects the live stack.
 - [ ] **Step 2: Push to `main`**
 ```bash
 cd /home/sam/Documents/DragonsStash
 git push origin main
 ```
 Expected: Drone pipeline runs `build-app`, `build-worker`, `build-bot`, `build-backup`, then `deploy`, all green. Check via the Drone UI or `drone build info admin/DragonsStash <build-number>` if the `drone` CLI is configured.
 - [ ] **Step 3: Confirm the container is up on the production host**
 ```bash
 ssh sam@192.168.68.68 "docker ps --filter name=dragonsstash-backup --format '{{.Names}}\t{{.Status}}'"
 ```
 Expected: `dragonsstash-backup   Up ...`.
 - [ ] **Step 4: Trigger one manual backup run and confirm a snapshot lands on the NAS**
 ```bash
 ssh sam@192.168.68.68 "docker exec dragonsstash-backup /backup.sh"
 ssh sam@192.168.68.68 "docker exec dragonsstash-backup restic snapshots"
 ```
 Expected: `backup.sh` exits 0; `restic snapshots` lists exactly one entry.
 - [ ] **Step 5: Confirm the Uptime Kuma push monitor shows green**
 Open the Uptime Kuma dashboard and check the `dragonsstash-backup` monitor's status is up with a recent heartbeat.
 - [ ] **Step 6: Restore drill — prove the backup is actually restorable**
 ```bash
 ssh sam@192.168.68.68 "docker exec dragonsstash-backup restic restore latest --target /tmp/restore-drill"
 ssh sam@192.168.68.68 "docker exec dragonsstash-backup ls -la /tmp/restore-drill/tmp"
 ```
 Expected: `/tmp/restore-drill/tmp/dragonsstash.dump` and `/tmp/restore-drill/tmp/tdlib.tar.gz` both present with nonzero size. Then, on a scratch Postgres (not the live `dragonsstash-db`), confirm the dump restores cleanly:
 ```bash
 ssh sam@192.168.68.68 "docker run -d --rm --name restore-drill-pg --network dragonsstash_internal \
  -e POSTGRES_USER=drill -e POSTGRES_PASSWORD=drill -e POSTGRES_DB=drill postgres:16-alpine"
 ssh sam@192.168.68.68 "docker cp dragonsstash-backup:/tmp/restore-drill/tmp/dragonsstash.dump /tmp/dragonsstash.dump"
 ssh sam@192.168.68.68 "docker cp /tmp/dragonsstash.dump restore-drill-pg:/tmp/dragonsstash.dump"
 ssh sam@192.168.68.68 "docker exec -e PGPASSWORD=drill restore-drill-pg pg_restore -U drill -d drill --clean --if-exists /tmp/dragonsstash.dump"
 ssh sam@192.168.68.68 "docker exec -e PGPASSWORD=drill restore-drill-pg psql -U drill -d drill -c '\\dt' | head -20"
 ssh sam@192.168.68.68 "docker rm -f restore-drill-pg"
 ```
 Expected: `pg_restore` completes without fatal errors; `\dt` lists the app's tables (e.g. `Package`, `User`, `TelegramLink`).
 - [ ] **Step 7: Clean up drill artifacts**
 ```bash
 ssh sam@192.168.68.68 "docker exec dragonsstash-backup rm -rf /tmp/restore-drill"
 ssh sam@192.168.68.68 "rm -f /tmp/dragonsstash.dump"
 ```
@@ -0,0 +1,184 @@
 # Worker Improvements Design
 **Date:** 2026-05-02  
 **Status:** Approved  
 **Scope:** Dragon's Stash Telegram ingestion worker
 ## Problem Statement
 Three issues to address:
 1. **Double-uploads**: The same archive occasionally appears twice in the destination Telegram channel. Root causes: (a) the worker crashes between `uploadToChannel()` confirming success and `createPackageWithFiles()` writing to the DB — no DB record means `recoverIncompleteUploads()` can't detect the orphaned Telegram message, and the next cycle re-uploads; (b) two accounts scanning the same source channel can both pass the hash dedup check before either creates a DB record, racing to upload the same file.
 2. **Sequential account processing**: Both Telegram accounts are processed one after another via `withTdlibMutex`, even though TDLib fully supports multiple concurrent clients in the same process (each with separate `databaseDirectory` and `filesDirectory`). This halves throughput unnecessarily.
 3. **Premium upload limit not used**: The Premium account can upload up to 4 GB per file, but `MAX_UPLOAD_SIZE` is hardcoded at ~1,950 MB. This causes unnecessary file splitting and expensive repack operations for files that could upload directly.
 ## Solution Overview
 Three targeted changes, no architectural overhaul:
 1. Two-phase DB write + hash advisory lock (fixes double-uploads)
 2. Remove TDLib mutex from the scheduler loop (enables parallel accounts)
 3. Per-account `maxUploadSize` from `getMe().is_premium` (enables 4 GB for Premium)
 ---
 ## Section 1: Double-Upload Fix
 ### 1a. Two-Phase DB Write
 **Current flow:**
 ```
 uploadToChannel() → preview download → metadata extraction → createPackageWithFiles()
 ```
 If the worker crashes anywhere between upload confirmation and `createPackageWithFiles()`, no DB record exists. `recoverIncompleteUploads()` only checks packages with an existing `destMessageId` in the DB — it cannot find an orphaned Telegram message with no corresponding row.
 **New flow:**
 ```
 uploadToChannel()
  → createPackageStub()          ← minimal record, destMessageId set immediately
  → preview download
  → metadata extraction
  → updatePackageWithMetadata()  ← adds file list, preview, creator, tags
 ```
 `createPackageStub()` writes: `contentHash`, `fileName`, `fileSize`, `archiveType`, `sourceChannelId`, `sourceMessageId`, `destChannelId`, `destMessageId`, `isMultipart`, `partCount`, `ingestionRunId`. File list and preview are left empty.
 If the worker crashes after the stub is written:
 - `recoverIncompleteUploads()` finds the record (has `destMessageId`), verifies the Telegram message exists, keeps it.
 - Next cycle: `packageExistsByHash()` returns true → skips re-upload.
 - The stub has `fileCount = 0` and no file listing. The UI shows "metadata pending" rather than failing silently.
 Stubs with `fileCount = 0` are valid deliverable packages (the bot can still send the file). Backfilling metadata on stubs is out of scope for this change — the crash case is rare and the stub is functional.
 ### 1b. Hash Advisory Lock
 **The race (two accounts, shared source channel):**
 ```
 Worker A: packageExistsByHash(X) → false  (no record yet)
 Worker B: packageExistsByHash(X) → false  (no record yet)
 Worker A: uploads file → destMessageId_A
 Worker B: uploads file → destMessageId_B  ← duplicate Telegram message
 Worker A: createPackageStub() → succeeds  (contentHash @unique satisfied)
 Worker B: createPackageStub() → fails unique constraint on contentHash
 ```
 Result: two Telegram messages, one DB record. Worker B's upload is wasted.
 **Fix:** Before calling `uploadToChannel()`, acquire a PostgreSQL session advisory lock keyed on the content hash:
 ```sql
 SELECT pg_try_advisory_lock(hash_bigint)
 ```
 Where `hash_bigint` is the first 8 bytes of the SHA-256 content hash interpreted as a signed bigint.
 - `pg_try_advisory_lock` is non-blocking. If another worker holds the lock (same file, shared channel), return `false` → treat as duplicate, skip.
 - After acquiring the lock, **re-run `packageExistsByHash()`** before uploading. This catches the case where another worker finished and released the lock between the first check and this one — without the re-check, the current worker would proceed to re-upload.
 - The lock is session-scoped: released automatically on DB session end. No manual cleanup needed on crash.
 - The lock is released explicitly after `createPackageStub()` completes (or on any error path).
 **Implementation location:** New helper `tryAcquireHashLock(contentHash)` / `releaseHashLock(contentHash)` in `worker/src/db/locks.ts`, reusing the existing DB client pattern.
 ---
 ## Section 2: Parallel Account Processing
 ### Current Constraint
 `withTdlibMutex` in `scheduler.ts` serializes all TDLib operations across accounts. This was a conservative guard, but TDLib explicitly supports multiple concurrent clients in the same process provided each has its own `databaseDirectory` and `filesDirectory`.
 The codebase already satisfies this requirement:
 ```typescript
 // worker/src/tdlib/client.ts
 const dbPath = path.join(config.tdlibStateDir, account.id);
 const client = createClient({
  databaseDirectory: dbPath,
  filesDirectory: path.join(dbPath, "files"),
 });
 ```
 Each account gets `<TDLIB_STATE_DIR>/<account.id>/` — fully isolated.
 ### Change
 Replace the sequential `for` loop in `scheduler.ts` with `Promise.allSettled()`:
 ```typescript
 // Before
 for (const account of accounts) {
  await withTdlibMutex(`ingest:${account.phone}`, () => runWorkerForAccount(account));
 }
 // After
 await Promise.allSettled(accounts.map((account) => runWorkerForAccount(account)));
 ```
 The per-account PostgreSQL advisory lock in `db/locks.ts` already prevents any account from being processed twice simultaneously. `Promise.allSettled()` ensures one account's failure doesn't abort the other.
 The `withTdlibMutex` wrapper can be removed from the ingest path entirely. The auth path (`authenticateAccount`) should also be run in parallel but may remain guarded if TDLib auth flows have ordering dependencies — verify during implementation.
 **No Docker Compose changes needed.** Both accounts run in the same container.
 ### Speed Limit Notifications
 TDLib fires `updateSpeedLimitNotification` when an account's upload or download speed is throttled (non-Premium accounts). Log this event at `warn` level in the client update handler so it's visible in logs without being actionable.
 ---
 ## Section 3: Per-Account Premium Upload Limit
 ### Premium Detection
 After successful authentication, call `getMe()` and read `is_premium: bool` from the returned `user` object. Store this on `TelegramAccount.isPremium` (new boolean field, default `false`, updated on each successful auth).
 ```typescript
 const me = await client.invoke({ _: 'getMe' }) as { is_premium?: boolean };
 await updateAccountPremiumStatus(account.id, me.is_premium ?? false);
 ```
 ### Upload Size Limits
 | Account type | `maxUploadSize` | Effect |
 |---|---|---|
 | Premium | 3,950 MB | Parts ≤ 3.95 GB upload as-is; repack only for parts >3.95 GB (extremely rare) |
 | Non-Premium | 1,950 MB | Current behavior unchanged |
 Pass `maxUploadSize` into `processOneArchiveSet()` as a parameter (currently hardcoded as `MAX_UPLOAD_SIZE` at `worker.ts:1023` and in `archive/split.ts`).
 The `hasOversizedPart` check and `byteLevelSplit` call both use this value, so the repack step is effectively eliminated for Premium accounts in practice — no separate "skip repack" flag needed.
 ### Migration
 ```prisma
 model TelegramAccount {
  // ... existing fields
  isPremium Boolean @default(false)
 }
 ```
 One migration, one new query `updateAccountPremiumStatus(accountId, isPremium)`.
 ---
 ## Files to Change
 | File | Change |
 |---|---|
 | `prisma/schema.prisma` | Add `isPremium Boolean @default(false)` to `TelegramAccount` |
 | `worker/src/db/queries.ts` | Add `updateAccountPremiumStatus()`, `createPackageStub()`, `updatePackageWithMetadata()` |
 | `worker/src/db/locks.ts` | Add `tryAcquireHashLock()`, `releaseHashLock()` |
 | `worker/src/tdlib/client.ts` | Call `getMe()` after auth, return `isPremium` from `createTdlibClient()` |
 | `worker/src/worker.ts` | Two-phase write, hash lock acquire/release, pass `maxUploadSize` per account |
 | `worker/src/archive/split.ts` | Accept `maxPartSize` parameter instead of hardcoded constant |
 | `worker/src/scheduler.ts` | Replace sequential loop with `Promise.allSettled()`, remove `withTdlibMutex` from ingest path |
 ---
 ## What Is Explicitly Out of Scope
 - Backfilling metadata on stub records (rare crash case, functional without it)
 - Download pre-fetching / pipeline parallelism within one account
 - Two separate worker containers (single container is sufficient)
 - Bot or app changes (worker-only)
@@ -0,0 +1,353 @@
 # Channel-Scan Skip Optimization — Design
 **Goal:** stop the worker from re-scanning channels and forum topics that haven't changed since the last scan, especially on restart. Reduce the per-cycle API call count for the Model Printing Emporium channel (1,086 forum topics) from ~1,000+ to ~50.
 **Non-goals:**
 - Replacing polling with event-driven ingestion (`updateNewMessage`). That's a separate, larger design (Phase 2 in the original brainstorm).
 - Surfacing per-channel scan history in the UI (also a separate, observability-only design).
 **Architecture sketch:**
 Add three persisted columns to `AccountChannelMap` and `TopicProgress`, plus one runtime `getChat`/`getForumTopicInfo` lookup before each scan. The new state survives restarts because it's in PostgreSQL; the lookup is a cheap TDLib local-cache call. Failure-retry semantics (`d99a506` + `901f32f`) must be preserved — a channel sitting on retryable `SkippedPackage` rows is never considered idle.
 ---
 ## Problem statement
 ### Today's behavior
 Every ingestion cycle the worker walks every linked source channel for every authenticated account. For each channel/topic it calls TDLib's `searchChatMessages` paginated from `lastProcessedMessageId`. Even when nothing has changed since the previous scan:
 - One `searchChatMessages` call (sometimes paginated) is still made
 - For Model Printing Emporium, that's ~1,086 calls per cycle (one per forum topic)
 - The 1-second `apiDelayMs` between pages multiplies the cost
 - Most calls return zero new messages — the work is wasted
 The cost is most acute right after a restart: the worker boots, runs recovery, then issues 1,000+ effectively-empty calls before any productive work happens.
 ### What we already track
 - `AccountChannelMap.lastProcessedMessageId` — highest processed message ID (per non-forum channel, per account)
 - `TopicProgress.lastProcessedMessageId` — same per forum topic
 - Both are advanced incrementally per archive set (`77aeb4c`)
 - Both are pulled back below failed messages by the `SkippedPackage` retry pass (`901f32f`)
 ### What we don't track and want to add
 - When was the last scan?
 - Did the last scan find any archives, OR is there outstanding retry work?
 - How many cycles in a row have been totally idle?
 These let us skip the scan entirely when nothing has changed.
 ---
 ## High-level approach
 Three guards at the top of the per-channel and per-topic processing loops:
 1. **DB-persistent "skip if recently scanned and truly idle"** — checks `lastScannedAt`, `lastScanFoundArchives`, and a `retryableSkippedCount` query. If all three say "nothing new, nothing failing", skip without any TDLib call.
 2. **Adaptive backoff for cold channels** — `consecutiveEmptyScans` counter. After it crosses a threshold, scan only every Nth cycle. Reset to 0 whenever the channel is "not idle".
 3. **`chat.last_message.id` short-circuit** — if (1) and (2) don't skip but the channel's last server-side message ID matches our watermark, skip the `searchChatMessages` paginated call. This runs after the existing `SkippedPackage` retry pass, which pulls the watermark back below failures, so it correctly forces a scan when retries are pending.
 The retry pass from `901f32f` is preserved untouched — it runs in front of these guards and adjusts the watermark, so retries always happen.
 ---
 ## Schema changes
 ### `AccountChannelMap` (worker/src/db/schema.prisma)
 ```prisma
 model AccountChannelMap {
  // ... existing fields ...
  lastScannedAt          DateTime?
  lastScanFoundArchives  Boolean   @default(false)
  consecutiveEmptyScans  Int       @default(0)
 }
 ```
 ### `TopicProgress`
 ```prisma
 model TopicProgress {
  // ... existing fields ...
  lastScannedAt          DateTime?
  lastScanFoundArchives  Boolean   @default(false)
  consecutiveEmptyScans  Int       @default(0)
 }
 ```
 ### Migration
 ```sql
 -- Both tables get the same three columns. Existing rows get defaults:
 --   lastScannedAt        = NULL  (next scan will populate)
 --   lastScanFoundArchives = false (safe default — will be overwritten by next scan)
 --   consecutiveEmptyScans = 0    (resets backoff for existing channels)
 ALTER TABLE "account_channel_map"
  ADD COLUMN "lastScannedAt" TIMESTAMP(3),
  ADD COLUMN "lastScanFoundArchives" BOOLEAN NOT NULL DEFAULT false,
  ADD COLUMN "consecutiveEmptyScans" INTEGER NOT NULL DEFAULT 0;
 ALTER TABLE "topic_progress"
  ADD COLUMN "lastScannedAt" TIMESTAMP(3),
  ADD COLUMN "lastScanFoundArchives" BOOLEAN NOT NULL DEFAULT false,
  ADD COLUMN "consecutiveEmptyScans" INTEGER NOT NULL DEFAULT 0;
 ```
 NULL `lastScannedAt` means "never scanned" — every channel will be scanned the first cycle after deploy. Subsequent cycles benefit from the new fields.
 ---
 ## Configuration
 Two new env vars in `worker/src/util/config.ts`:
 ```typescript
 /** Window in which a recent successful empty scan lets us skip. Default 5 min. */
 skipRecentScanWindowMs:
  parseInt(process.env.WORKER_SKIP_RECENT_SCAN_WINDOW_MS ?? "300000", 10),
 /** After this many consecutive empty scans, channel enters backoff mode. */
 emptyScanBackoffThreshold:
  parseInt(process.env.WORKER_EMPTY_SCAN_BACKOFF_THRESHOLD ?? "5", 10),
 /** Backoff factor — N means "scan every Nth cycle once in backoff". */
 emptyScanBackoffEveryNth:
  parseInt(process.env.WORKER_EMPTY_SCAN_BACKOFF_EVERY_NTH ?? "5", 10),
 ```
 All three are tunable per deployment without code changes.
 ---
 ## Decision logic per channel / topic
 The skip decision sits at the top of each channel/topic iteration in `runWorkerForAccount`. It runs BEFORE the existing `SkippedPackage` retry pass.
 ```text
 For each channel (or topic):
  1. Query retryableSkippedCount for this scope (already a query we do elsewhere)
  2. If retryableSkippedCount > 0:
       Force scan (don't skip — failures need retry)
       Proceed to existing flow (retry pass → scan)
  3. Else if lastScannedAt is NULL:
       Force scan (we've never touched this)
       Proceed to existing flow
  4. Else if Date.now() - lastScannedAt.getTime() < skipRecentScanWindowMs
          AND lastScanFoundArchives === false:
       Skip — recently scanned and truly idle
  5. Else if consecutiveEmptyScans >= emptyScanBackoffThreshold
          AND (cycleCount % emptyScanBackoffEveryNth !== 0):
       Skip — channel is cold, not its turn to scan
  6. Else:
       Run the existing flow:
         a. SkippedPackage retry pass (901f32f) — may pull watermark back
         b. NEW: getChat (or getForumTopicInfo) — if last_message.id <= watermark, skip
         c. searchChatMessages scan
 ```
 `cycleCount` is the global ingestion-cycle counter from `scheduler.ts`. It already increments per cycle.
 ---
 ## End-of-scan bookkeeping
 After every scan (whether it found archives or not), update the three new fields atomically with the existing watermark write:
 ```typescript
 // "Truly idle" means: nothing new this scan AND nothing failed AND no leftover
 // retryable failures. The retry-pending check is critical — without it, a
 // scan that found no new archives but left SkippedPackage retries pending
 // would be marked idle and incorrectly skipped next cycle.
 const retryablePending = await getRetryableSkippedMessageIds({
  accountId, sourceChannelId, topicId, cap: maxSkipAttempts,
 });
 const trulyIdle =
  scanResult.archives.length === 0
  && minFailedId === null
  && retryablePending.length === 0;
 const newConsecutiveEmpty = trulyIdle
  ? (prev.consecutiveEmptyScans ?? 0) + 1
  : 0;
 await upsertChannelOrTopicScanState({
  // ... existing watermark fields ...
  lastScannedAt: new Date(),
  lastScanFoundArchives: !trulyIdle,
  consecutiveEmptyScans: newConsecutiveEmpty,
 });
 ```
 The `consecutiveEmptyScans` counter resets to 0 the moment *anything* happens — archives found, archives failed, or unresolved retries pending. A channel with a chronically-failing archive (whose attemptCount is still below the cap) keeps the counter at 0 and never enters backoff.
 If a SkippedPackage hits `attemptCount === maxSkipAttempts`, it's no longer "retryable pending" (it's been given up on), so the counter increments correctly. Same for SkippedPackages that get deleted via the UI's "retry" button — the counter behaves correctly without special-casing.
 ---
 ## `getChat` / `getForumTopicInfo` short-circuit
 After the retry pass has finalized the effective watermark, but before `searchChatMessages`:
 ```typescript
 // For non-forum channels:
 const chat = await client.invoke({ _: "getChat", chat_id: Number(channel.telegramId) });
 const channelLastMessageId = chat.last_message?.id;
 if (channelLastMessageId && BigInt(channelLastMessageId) <= effectiveWatermark) {
  // Nothing new server-side — skip the paginated search entirely.
  // Still update lastScannedAt / consecutiveEmptyScans so the recent-scan
  // skip kicks in next cycle.
  await persistScanState({ trulyIdle: true });
  continue;
 }
 // For forum topics:
 const topicInfo = await client.invoke({
  _: "getForumTopicInfo",
  chat_id: Number(channel.telegramId),
  message_thread_id: Number(topic.topicId),
 });
 const topicLastMessageId = topicInfo.info?.last_message_id;
 if (topicLastMessageId && BigInt(topicLastMessageId) <= effectiveWatermark) {
  await persistScanState({ trulyIdle: true });
  continue;
 }
 ```
 `getChat` is served from TDLib's local cache (no network) for chats we've already loaded, which we do up front via `loadChats`. `getForumTopicInfo` is a single round-trip but much cheaper than a paginated `searchChatMessages` call.
 The comparison is `<=` because the watermark is the highest message we've fully processed — if the server's last is the same, we're caught up.
 This step is correct in the failure-retry case because the retry pass runs FIRST: if there were retryable failures, the retry pass pulled the watermark back below them, and `channelLastMessageId > effectiveWatermark` (since the failed message exists in TG), so we don't skip — we scan and re-pick-up the failure.
 ---
 ## Restart behavior
 The improvements compose for restart safety:
 | Scenario | Today | After this change |
 |---|---|---|
 | Restart 5 min after a clean cycle | ~2,000 API calls for MPE | ~10 calls (only retryable + truly-active topics) |
 | Restart 1 hour later (one missed cycle) | ~2,000 API calls | `getChat` per channel + scan only those where `last_message.id > watermark` (≈ 50 for MPE) |
 | Restart after long downtime (12h) | ~2,000 calls + lots of new content | `getChat` per channel, scan everything with new activity |
 The three new columns are in PostgreSQL — they survive container restarts directly. `consecutiveEmptyScans = 47` for a cold topic stays at 47 across restart, so backoff continues to apply.
 ---
 ## Edge cases and their handling
 ### 1. Manual SkippedPackage retry via UI between cycles
 The UI's `retrySkippedPackageAction` lowers the watermark and deletes the SkippedPackage. Next cycle: `retryableSkippedCount === 0` (the row is gone), but the watermark is lower than `chat.last_message.id` (the retried message exists in TG). So step 6 in the decision tree triggers a scan via the `getChat` check. ✓
 ### 2. SkippedPackage hits the attempt cap mid-cycle
 Once `attemptCount === maxSkipAttempts`, the row is no longer in `getRetryableSkippedMessageIds` results. The channel correctly becomes idle-eligible. The capped SkippedPackage stays in the table as "permanently failed (manual retry only)" — that's the existing behavior. ✓
 ### 3. New SkippedPackage is created mid-cycle (e.g., an upload fails)
 At the end of that scan, `retryablePending` includes the new row → `trulyIdle = false` → `lastScanFoundArchives = true` → next cycle does NOT skip. ✓
 ### 4. Channel/topic added after deploy
 New rows in `AccountChannelMap` / `TopicProgress` have `lastScannedAt = NULL`, so step 3 in the decision tree always triggers a scan. After the first scan, the fields are populated normally. ✓
 ### 5. Clock skew / drift
 The `lastScannedAt < 5 min ago` check uses `Date.now() - lastScannedAt.getTime()`. Both are application-side clocks (Node.js + PostgreSQL `NOW()` at write). A few seconds of drift doesn't matter; an hour of clock jump (rare but possible) just means one cycle either skips or re-scans — recoverable.
 ### 6. TDLib `getChat` returns stale data
 TDLib's local cache could theoretically be stale (e.g., the account hasn't received the latest update yet). If `channelLastMessageId` is stale (lower than server reality), we'd skip a scan that should have happened. Mitigation: the next cycle's `getChat` likely has fresh data; the watermark guards correctness (we don't lose data, we just process it one cycle later). Acceptable.
 ### 7. `getForumTopicInfo` rate limit
 Calling it per-topic could add up for channels with 1000+ topics. Mitigation: skip-on-recent-scan (step 4) eliminates the call for most topics; only "stale-but-was-active" topics get the call. Worst case is ~50 calls per cycle for MPE, comfortably under the 30 req/sec global limit.
 ### 8. Channel becomes a forum (or vice versa) between cycles
 Existing code handles this — `isChatForum` is rechecked each cycle and `setChannelForum` updates the DB. The new fields live on the same rows, so no extra handling needed.
 ---
 ## File-level changes
 ### New / modified
 - `prisma/schema.prisma` — add the six new columns
 - `prisma/migrations/<timestamp>_channel_scan_state/migration.sql` — the ALTER TABLE
 - `worker/src/util/config.ts` — three new env vars
 - `worker/src/db/queries.ts` — new helpers:
  - `getChannelScanState(mappingId)` and `getTopicScanState(topicProgressId)`
  - `upsertChannelScanState(...)` and `upsertTopicScanState(...)`
  - Both wrap the existing `updateLastProcessedMessage` / `upsertTopicProgress` so callers don't need to remember to update the new fields too.
 - `worker/src/worker.ts` — top-of-loop skip checks in both the forum and non-forum branches, plus end-of-scan state writes
 - `worker/src/tdlib/chats.ts` — small helper `getChatLastMessageId(client, chatId)` and `getForumTopicLastMessageId(client, chatId, topicId)` wrapping the TDLib calls with the existing `invokeWithTimeout` pattern
 ### Untouched
 - `recovery.ts` — recovery is per-startup and one-shot; not affected
 - `scheduler.ts` — `cycleCount` is already there; just expose it where needed
 - The existing `SkippedPackage` retry pass logic in `runWorkerForAccount` is unchanged
 ---
 ## Testing plan
 The project has no automated tests, so verification is manual via Docker logs after deploy:
 1. **Build cleanly:** `docker compose up -d --build worker` — no migration errors
 2. **First cycle after deploy:** all channels scan (NULL `lastScannedAt`), all fields populated at end of cycle. Log lines confirm normal scan flow.
 3. **Second cycle 5 min later:**
   - Check logs for `"Skipping recently-scanned idle channel"` — should appear for any channel/topic that was empty last cycle
   - Total `searchChatMessages` calls per cycle should drop dramatically (compare to first cycle)
 4. **Failure-retry preservation:**
   - Find a SkippedPackage with `attemptCount < cap`
   - Run a cycle — confirm the channel/topic is NOT skipped (log says it's scanned)
   - Confirm the SkippedPackage gets re-tried
 5. **Backoff:**
   - Pick a cold channel, wait for it to scan 5+ cycles cleanly
   - Confirm `consecutiveEmptyScans` climbs to 5+
   - Confirm subsequent cycles skip it (only scan every 5th)
 6. **`getChat` short-circuit:**
   - Pick an active channel
   - Trigger an immediate cycle (UI button)
   - If `last_message.id <= watermark`, expect log `"Channel caught up via getChat — skipping searchChatMessages"`
 7. **Restart safety:**
   - Push the change, restart worker
   - First cycle after restart should log multiple "Skipping recently-scanned idle channel" lines (because the DB state survived)
   - Total cycle time should be a fraction of a baseline restart
 ---
 ## Risks and mitigations
 | Risk | Mitigation |
 |---|---|
 | Skip incorrectly applied → real failures never retried | Rule 1 (truly-idle includes `retryablePending === 0`) + dedicated test step 4 |
 | `getChat` returns stale data | Next cycle's `getChat` corrects it; watermark guards correctness (no data loss) |
 | `getForumTopicInfo` not available in TDLib 1.8.64 | Verify the method exists in the schema; fall back to scan if it throws |
 | Backoff applies during legitimate activity bursts | Counter resets to 0 the moment any archive is found OR any retry is pending |
 | Migration takes too long on the live DB | Both columns have NOT NULL defaults — Postgres can add them as fast metadata changes (no table rewrite) |
 ---
 ## What's explicitly NOT in this design
 To keep scope tight:
 - **Event-driven ingestion via `updateNewMessage`.** Bigger design, addressed separately. This design is compatible with it — when (D) lands, polling becomes a 4-hour safety net using these same skip rules.
 - **Per-channel scan history UI.** Observability layer; separate design.
 - **Surfacing the new counters in the admin dashboard.** Can come after the worker-side change is verified.
 - **Backfilling `consecutiveEmptyScans` from historical `IngestionRun` data.** Not worth it — it'll converge to the correct value within ~6 cycles.
 ---
 ## Open questions
 None — the failure-retry interaction was the main risk and is handled by Rule 1 + the existing retry pass.
@@ -0,0 +1,170 @@
 # Send All From Creator — Design
 **Date:** 2026-07-04
 **Status:** Approved for planning
 ## Summary
 Add a "Send all from [Creator]" button to the STL packages page that queues every
 sendable package from a given creator for delivery to the user's Telegram. The
 button appears in the packages-tab toolbar only when the list is filtered by a
 single creator (`?creator=<name>`).
 This reuses the existing single-send infrastructure (`BotSendRequest` +
 `pg_notify('bot_send', id)` + the bot's `send-listener`) and closely mirrors the
 existing `sendAllInGroupAction`. No bot, DB schema, or send-listener changes are
 required.
 ## Context
 Relevant existing pieces:
 - **Creator field:** `Package.creator` is a nullable, indexed `String` on the
  `Package` model (`prisma/schema.prisma:476-524`). Filtering by exact creator is
  cheap.
 - **Creator filter:** The packages list already supports `?creator=<name>`
  (`src/app/(app)/stls/page.tsx:22,38`), rendered by the packages column as a
  clickable value (`package-columns.tsx`).
 - **Single send:** `send-to-telegram-button.tsx` → `POST /api/telegram/bot/send`
  creates a `BotSendRequest` (status `PENDING`) and fires
  `pg_notify('bot_send', requestId)`; the button then polls the request to
  completion.
 - **Existing bulk send (group):** `sendAllInGroupAction(groupId)` in
  `src/app/(app)/stls/actions.ts:515-591` fetches the group's packages, filters to
  those with `destChannelId` + `destMessageId`, skips any with an existing
  `PENDING`/`SENDING` request for the same package + telegram link, creates a
  `BotSendRequest` per remaining package, and fires `pg_notify` per package. It is
  wired into the table via `handleSendAllInGroup` (`stl-table.tsx:264-278`) with a
  `confirm()` dialog and a success toast — no polling.
 - **Bot side:** `bot/src/send-listener.ts` processes every `BotSendRequest`
  uniformly; nothing there needs to change.
 ## Requirements
 1. When the packages tab is filtered by a single creator (`?creator=<name>` is
   present), show a toolbar button labeled **"Send all from [Creator]"**.
 2. When no creator filter is active, the button is not rendered.
 3. Clicking the button shows a confirmation dialog before sending (consistent with
   the single-send and group-send flows, since this pushes files to Telegram).
 4. On confirm, queue **all** sendable packages from that creator across all pages —
   not just the current page.
 5. Skip packages that are not yet uploaded (missing `destChannelId` /
   `destMessageId`) and packages that already have a `PENDING`/`SENDING` request for
   the same package + telegram link (dedup).
 6. Report real counts back to the user via toast, e.g.
   "Queued 12 packages from [Creator]" and, when applicable, a skipped count.
 7. No polling — the bulk action queues and returns; sends process in the background
   via the bot listener (same behavior as group send).
 ## Design
 ### Server action — `sendAllFromCreatorAction(creatorName: string)`
 New action in `src/app/(app)/stls/actions.ts`, mirroring `sendAllInGroupAction`:
 1. Auth check (`auth()`), return `Unauthorized` if no session.
 2. Load the user's `TelegramLink`; if none, return
   "No linked Telegram account. Link one in Settings."
 3. Reject an empty/blank `creatorName` (guard against sending the entire "no
   creator" set — see Edge cases).
 4. Fetch sendable packages directly with a `where` filter (no in-memory filtering):
   ```ts
   prisma.package.findMany({
     where: {
       creator: creatorName,
       destChannelId: { not: null },
       destMessageId: { not: null },
     },
     select: { id: true },
   })
   ```
 5. If none, return "No uploaded packages found for this creator."
 6. For each package, skip if an existing `PENDING`/`SENDING` `BotSendRequest`
   exists for that package + telegram link; otherwise create a `BotSendRequest`
   (status `PENDING`) and fire `pg_notify('bot_send', id)` (best-effort, wrapped in
   try/catch — the bot also polls).
 7. `revalidatePath("/stls")`.
 8. Return counts so the UI can show them. This is a small improvement over
   `sendAllInGroupAction`, which currently returns `{ success: true, data: undefined }`.
   Shape:
   ```ts
   { success: true, data: { queued: number, skipped: number } }
   ```
   where `skipped` counts packages that already had a live request. (Packages not
   yet uploaded are excluded by the query and not counted as skipped.)
 ### UI — toolbar button in `stl-table.tsx`
 - Derive the active creator from the URL: `const activeCreator =
  searchParams.get("creator") ?? "";` (parallel to the existing
  `activeTag` at `stl-table.tsx:445`).
 - In the packages-tab toolbar (`stl-table.tsx:478` `flex flex-wrap items-center
  gap-2` row), render the button only when `activeCreator` is non-empty:
  ```tsx
  {activeCreator && (
    <Button variant="outline" size="sm" className="h-9 gap-1.5"
            onClick={handleSendAllFromCreator}>
      <Send className="h-3.5 w-3.5" />
      Send all from {activeCreator}
    </Button>
  )}
  ```
  (Reuse the icon already used by the send action; place near the Upload / Group
  buttons.)
 - Add `handleSendAllFromCreator`, modeled on `handleSendAllInGroup`
  (`stl-table.tsx:264-278`):
  ```ts
  const handleSendAllFromCreator = useCallback(() => {
    if (!confirm(`Send all packages from "${activeCreator}" to your Telegram?`)) return;
    startTransition(async () => {
      const result = await sendAllFromCreatorAction(activeCreator);
      if (result.success) {
        const { queued, skipped } = result.data;
        toast.success(
          `Queued ${queued} package${queued === 1 ? "" : "s"} from ${activeCreator}` +
          (skipped ? ` (${skipped} already queued)` : "")
        );
        router.refresh();
      } else {
        toast.error(result.error);
      }
    });
  }, [activeCreator, router]);
  ```
 - Import `sendAllFromCreatorAction` alongside the existing `sendAllInGroupAction`
  import (`stl-table.tsx:51`).
 ### Data flow
 ```
 User (filtered by creator) → clicks button → confirm dialog
  → sendAllFromCreatorAction(creatorName)
    → for each sendable, un-queued package: create BotSendRequest + pg_notify
  → returns { queued, skipped } → toast + router.refresh()
 (background) bot send-listener consumes each BotSendRequest → delivers to user
 ```
 ## Edge cases
 - **Blank creator:** The action rejects an empty/whitespace `creatorName` so a
  stray `?creator=` never queues every package. The UI already gates the button on
  a non-empty `activeCreator`, so this is defense-in-depth.
 - **No linked Telegram account:** Returns an error toast, queues nothing.
 - **No uploaded packages for creator:** Returns an error toast, queues nothing.
 - **All packages already queued:** `queued: 0`, `skipped: N` → toast reflects it.
 - **pg_notify failure:** Swallowed per-package (best-effort); the bot's periodic
  poll picks the request up. Matches existing behavior.
 ## Out of scope / non-goals
 - No per-row "send all from this creator" action (toolbar-only, per decision).
 - No progress polling / live completion tracking for the bulk send.
 - No changes to the bot, `send-listener`, `BotSendRequest` schema, or API routes.
 - No batching/throttling beyond what the bot listener already does.
 ## Files touched
 - `src/app/(app)/stls/actions.ts` — add `sendAllFromCreatorAction`.
 - `src/app/(app)/stls/_components/stl-table.tsx` — derive `activeCreator`, add
  `handleSendAllFromCreator`, render the toolbar button, import the new action.
@@ -0,0 +1,205 @@
 # NAS Backup for Postgres + TDLib State — Design
 **Date:** 2026-07-23
 **Status:** Approved for planning
 ## Summary
 Add a dedicated `backup` container to the DragonsStash stack that takes daily,
 encrypted, deduplicated backups of the two things that can't be regenerated —
 the Postgres database (inventory/STL metadata, users, everything the app
 manages) and the two TDLib state volumes (Telegram session/auth state for the
 worker and bot) — and ships them to a Synology NAS over NFS. STL archive
 contents themselves are explicitly out of scope: they only live on this host
 temporarily and are not backed up.
 Backups are stored via [restic](https://restic.net/), which provides
 encryption-at-rest, block-level dedup, and retention pruning natively, so no
 custom encryption or pruning scripts need to be written or maintained.
 ## Context
 Current state (as of this design):
 - Production stack runs from `/opt/stacks/DragonsStash/docker-compose.yml` on
  this Dockge-managed host, pulling prebuilt images from
  `git.samagsteribbe.nl`. The `docker-compose.yml` in this repo is the
  build/dev reference and should be kept in sync.
 - Named volumes in use: `postgres_data` (Postgres 16 data directory),
  `tdlib_state` (worker's TDLib session), `tdlib_bot_state` (bot's TDLib
  session), `tmp_zips` and `manual_uploads` (both transient, explicitly out of
  scope here).
 - No backup mechanism, NFS mount, or host cron currently exists anywhere in
  this deployment.
 - The host already runs Uptime Kuma (used here for backup alerting) and Loki
  (container logs are presumably already collected there).
 ## Requirements
 1. Daily backup of the Postgres database and both TDLib state volumes.
 2. Backups stored on a Synology NAS via NFS, not on local disk.
 3. 14-day retention, oldest snapshots pruned automatically.
 4. Backups encrypted at rest (Postgres dumps and TDLib session files both
   contain sensitive material — password hashes, Telegram API secrets, live
   session state).
 5. Postgres backups must be transactionally consistent regardless of live app
   traffic. TDLib state backups are best-effort (see Decisions below) — this
   is an accepted trade-off, not a defect.
 6. Alert (via existing Uptime Kuma) if a backup run fails or doesn't happen.
 7. No new host-level state (no `/etc/fstab` entries, no host crontab) — the
   backup mechanism should be a container, consistent with how everything
   else on this host is deployed and versioned.
 8. No new privileged access — specifically, the backup container must not
   have Docker socket access or any ability to control sibling containers.
 ## Decisions
 - **NFS mounted via Docker's native NFS volume driver** (`driver_opts: type:
  nfs`), not a host-level mount. Keeps all backup-related state inside the
  compose file instead of split across host config.
 - **Restic, not hand-rolled tar+age+find.** Restic already solves encryption,
  dedup, and retention correctly; hand-rolled scripts would be reinventing
  that logic with more room for bugs.
 - **TDLib state is tarred live (best-effort), not paused.** Pausing the
  worker/bot for a clean snapshot would require mounting the Docker socket
  into the backup container so it could stop/start sibling containers — a
  real privilege escalation (a compromised backup container could then
  control any container on the host). The downside of a best-effort tar is
  bounded: worst case, a bad TDLib restore means redoing the Telegram SMS
  auth flow, which is the same outcome as having no backup at all. That
  bounded, low-severity downside doesn't justify the privilege escalation.
 - **Fixed-time cron (`crond`), not a sleep-loop.** A `sleep 86400` loop drifts
  on every container restart; a real crontab entry fires at a fixed time of
  day regardless of restarts, for negligible extra complexity.
 - **Restore is manual, not automated.** A script capable of restoring can
  overwrite live state; that should always require a human deliberately
  running it, not run unattended.
 ## Design
 ### New service: `backup`
 Added to both `/opt/stacks/DragonsStash/docker-compose.yml` (production) and
 this repo's `docker-compose.yml` (dev/build reference).
 - **Image**: custom, `FROM alpine:3.20`, `apk add --no-cache restic
  postgresql16-client curl tzdata dcron tar bash`. No dependency on app
  source — independent Dockerfile, e.g. `backup/Dockerfile`.
 - **Scheduling**: `crond -f` in the foreground as the container's entrypoint,
  with a crontab installed at build time:
  ```
  0 3 * * * /backup.sh >> /proc/1/fd/1 2>&1
  0 4 * * 0 restic check >> /proc/1/fd/1 2>&1
  ```
  (daily dump/backup at 03:00, weekly repo integrity check at 04:00 Sunday).
  `restic init` runs once at container startup (entrypoint, before `crond`
  starts), swallowing the "already initialized" error on subsequent
  container (re)starts.
 - **Network**: `internal` only — reaches `dragonsstash-db:5432` for
  `pg_dump`. No ports exposed.
 - **Volumes**:
  - `tdlib_state:/data/tdlib-worker:ro`
  - `tdlib_bot_state:/data/tdlib-bot:ro`
  - `nas_backups:/backups`, a named volume defined with:
    ```yaml
    nas_backups:
      driver_opts:
        type: nfs
        o: "addr=${NAS_HOST},rw,nfsvers=4,soft,timeo=100"
        device: ":${NAS_EXPORT_PATH}"
    ```
 - **New `.env` entries**: `NAS_HOST`, `NAS_EXPORT_PATH` (NFS share details),
  `RESTIC_PASSWORD` (repo encryption key), `KUMA_PUSH_URL` (Uptime Kuma push
  monitor URL). All four are inputs to gather during implementation, not
  hardcoded.
 - `restart: unless-stopped`, no `privileged`, no Docker socket mount.
 ### `backup.sh`
 ```
 set -euo pipefail
 trap 'curl -fsS "$KUMA_PUSH_URL" --get --data-urlencode "status=down" \
  --data-urlencode "msg=$BASH_COMMAND failed"' ERR
 pg_dump -h dragonsstash-db -U "$POSTGRES_USER" -d "$POSTGRES_DB" \
  -Fc -f /tmp/dragonsstash.dump
 tar czf /tmp/tdlib.tar.gz -C /data tdlib-worker tdlib-bot
 restic backup /tmp/dragonsstash.dump /tmp/tdlib.tar.gz
 restic forget --keep-daily 14 --prune
 rm -f /tmp/dragonsstash.dump /tmp/tdlib.tar.gz
 curl -fsS "$KUMA_PUSH_URL" --get --data-urlencode "status=up" \
  --data-urlencode "msg=OK"
 ```
 `PGPASSWORD` and `RESTIC_REPOSITORY=/backups/restic-repo` are set as
 container environment variables (from `.env`), not inline in the script.
 ### Data flow
 ```
 crond (daily 03:00)
  → pg_dump (consistent snapshot via Postgres MVCC) → /tmp/dragonsstash.dump
  → tar tdlib_state + tdlib_bot_state (best-effort, live) → /tmp/tdlib.tar.gz
  → restic backup (encrypt + dedup) → NFS-mounted repo on Synology NAS
  → restic forget --keep-daily 14 --prune
  → curl Uptime Kuma push monitor (up on success, down + reason on any failure)
 ```
 ### Restore (manual, documented procedure — not scripted/automated)
 ```
 restic -r /backups/restic-repo restore latest --target /tmp/restore
 pg_restore -h dragonsstash-db -U "$POSTGRES_USER" -d "$POSTGRES_DB" \
  --clean --if-exists /tmp/restore/tmp/dragonsstash.dump
 # untar /tmp/restore/tmp/tdlib.tar.gz back into the tdlib_state /
 # tdlib_bot_state volumes (via a throwaway container mounting both)
 ```
 ## Alerting
 - One Uptime Kuma **Push** monitor, created manually in the existing Kuma
  instance, with an expected heartbeat interval of ~26 hours (slack past the
  24h schedule so one slow run doesn't false-positive). Whatever notification
  channels are already configured on that monitor fire automatically — no new
  alerting integration.
 - `backup.sh` pushes `status=up` on success and `status=down` (with the
  failing command in `msg`) on any failure, via the `ERR` trap.
 - Container logs go to stdout, collected the same way every other container's
  logs already are on this host.
 ## Testing
 This repo has no automated test framework (documented convention: manual
 testing). For this infra change:
 - After deploy: manually run `docker exec dragonsstash-backup /backup.sh`
  once, confirm a snapshot appears (`restic snapshots`), confirm the Kuma
  monitor goes green.
 - **Restore drill** (once, during setup): actually restore the dump into a
  scratch Postgres and untar the TDLib archive into scratch volumes, to prove
  the backup is really restorable. Not automated or recurring for now.
 ## Out of scope / non-goals
 - Backing up `tmp_zips` or `manual_uploads` — both transient by design.
 - Automated/scheduled restore testing.
 - Backing up any other stack on this host (this design is DragonsStash-only,
  though the pattern — Docker-native NFS volume + restic — could be reused
  for other stacks later).
 - Pausing worker/bot for a guaranteed-consistent TDLib snapshot (see
  Decisions).
 ## Files touched
 - `docker-compose.yml` (this repo) and
  `/opt/stacks/DragonsStash/docker-compose.yml` (production) — add `backup`
  service, `nas_backups` volume.
 - `backup/Dockerfile` — new.
 - `backup/backup.sh` — new.
 - `backup/crontab` — new.
 - `.env.example` / `.env` — add `NAS_HOST`, `NAS_EXPORT_PATH`,
  `RESTIC_PASSWORD`, `KUMA_PUSH_URL`.
@@ -0,0 +1,271 @@
 # Provenance Backfill on Re-Index — Design
 **Date:** 2026-07-23
 **Status:** Approved for planning
 ## Summary
 Recover the true origin of packages whose recorded "source" is a placeholder —
 specifically the manual-upload and `rebuild.ts`-created records whose
 `sourceChannelId` points at the destination (archive) channel itself. When a
 real source channel is re-indexed and the worker encounters an archive that
 matches such a placeholder package, it backfills the real
 `sourceChannelId` / `sourceMessageId` / `sourceTopicId` / `sourceCaption` /
 `creator` (and, for records that never had one, the file listing and preview)
 onto the existing package — **without downloading the full archive**.
 Matching is a two-stage process: a zero-download candidate lookup by
 `fileName` + total `fileSize`, confirmed by a CRC32 fingerprint read from just
 the archive's central directory via a **ranged (tail) download** of a few
 KB–MB, instead of the multi-GB whole file.
 This is the first of four linked sub-projects. The others (creator
 normalization, provenance display, missing-files) are out of scope here and get
 their own spec → plan cycles. "Missing files" is explicitly deferred until this
 lands.
 ## Context
 Current state (as of this design):
 - `Package` records their origin via `sourceChannelId`, `sourceMessageId`,
  `sourceTopicId`, `sourceCaption`, and (for content dedup) `contentHash` and
  `remoteUniqueId`. `creator` is a free-text string extracted at ingestion.
 - Two ingestion paths create records with **placeholder provenance**, where the
  "source" is the destination channel, not a real origin:
  - **Manual uploads** (`worker/src/manual-upload.ts`) set
    `sourceChannelId = destChannel.id` and `sourceMessageId = destResult.messageId`.
    They *do* populate `PackageFile` (with `crc32`) from a local central-directory
    read at upload time.
  - **`rebuild.ts`** scans the destination channel and creates minimal records
    with `fileCount == 0` and **no** `PackageFile` rows.
 - The main worker upload path (`worker/src/worker.ts` → `uploadToChannel`) sends
  archives to the destination channel with **no caption**, so provenance cannot
  be recovered by re-reading destination messages — it has to come from matching
  against real source channels.
 - The scan/dedup ladder in `processOneArchiveSet` (`worker/src/worker.ts:1521`)
  is entirely **source-channel-scoped**:
  1. `findPackageByRemoteUniqueId(channel.id, …)` — same channel only
  2. `packageExistsBySourceMessage(channel.id, …)` — same channel only
  3. `findRepostedPackage(channel.id, fileName, size)` — same channel only
  4. …then full download → `packageExistsByHash(contentHash)` — global, but only
     *after* downloading the whole archive.
 - Because a placeholder package's `sourceChannelId` is the destination, checks
  1–3 never match it when a real source channel is scanned. Today the worker
  therefore downloads the entire archive, hits check 4, finds it is a duplicate,
  and skips — **wasting the download and leaving the wrong provenance in place.**
 - There is already an in-production precedent for "match an already-known
  archive during scan and enrich its metadata": when `findRepostedPackage`
  matches, the worker backfills richer *topic* context onto the existing package
  via `updatePackageTopicContext` (`worker/src/worker.ts:1608`+). Provenance
  backfill is the same move, widened from same-channel to cross-channel.
 - `remote.unique_id` is **not** a reliable cross-channel key: it identifies a
  stored file object on Telegram's servers, not the content. The same archive
  independently uploaded to two channels gets different `unique_id`s; it only
  matches for forwarded messages (same underlying file object). This is why the
  existing dedup scopes it to a single channel.
 ## Goals / Non-Goals
 **Goals**
 - Attribute true origin to placeholder-provenance packages during normal
  re-index scans, opportunistically (no separate pass).
 - Do it without downloading whole archives (ranged central-directory read only).
 - Be non-destructive: only ever touch packages that currently have placeholder
  provenance; never overwrite a real, non-placeholder source.
 - Be idempotent: re-running a re-index does not re-mutate or duplicate.
 **Non-Goals (separate sub-projects / deferred)**
 - Creator name normalization / canonical creator entity.
 - UI changes to display provenance.
 - Recovering "missing files."
 - Choosing a *preferred* origin when an archive genuinely exists in several
  source channels (first confirmed source wins).
 ## Design
 ### 1. Candidate definition
 > A package is a backfill candidate iff **`sourceChannelId == destChannelId`
 > OR `sourceMessageId == 0`**.
 Two placeholder shapes exist (verified against live data 2026-07-23):
 - **Manual uploads** (`manual-upload.ts`): `sourceChannelId == destChannelId`,
  real `contentHash`, real `sourceMessageId`, has a `PackageFile` listing.
 - **Rebuild records** (`rebuild.ts`): `sourceMessageId == 0n` (deliberate
  "unknown" sentinel), synthetic `contentHash = "rebuild:<destChannelId>:<destMessageId>"`,
  `fileCount == 0`, and `sourceChannelId` set to an **arbitrary fallback source
  channel** (`sourceChannels[0]`) — NOT the destination. (This is the common
  case: e.g. 59,893 records after a destination rebuild.)
 Normal ingestion always sets a real `sourceMessageId` (> 0) and a real source
 channel, so neither marker matches a genuinely-sourced package. Both markers are
 overwritten on backfill (source channel + message become real), so a record
 stops being a candidate once fixed — this is what makes re-scans idempotent.
 **Known limitation:** backfill does NOT rewrite a rebuild record's synthetic
 `"rebuild:"` `contentHash` (the true content hash would require a full download,
 which this feature avoids). That is acceptable — dedup after backfill relies on
 `remoteUniqueId` + name/size within the source channel, not on `contentHash`.
 ### 2. Where it hooks
 A new step is inserted into `processOneArchiveSet` **between check #3
 (`findRepostedPackage`) and the full download**. It runs only after the
 same-channel checks have missed (so genuine same-channel reposts keep their
 existing fast paths).
 Flow for the scanned archive set:
 1. Stage A — **candidate lookup (zero download).** Query for a package where
   `(sourceChannelId == destChannelId OR sourceMessageId == 0)` AND
   `fileName == archiveName` AND `fileSize == totalArchiveSize`.
   (`Package` has `@@index([fileName])`.)
   - No candidate → fall through to normal ingestion unchanged.
 2. Stage B — **fingerprint confirmation (tiny download).** See §3.
 3. On confirmation → **backfill** (see §4) and return `null` (treated as a
   duplicate; no full download, no new package). Increment a `zipsBackfilled`
   counter.
 4. On mismatch / failure / ambiguity → fall through to normal ingestion (see §6).
 ### 3. Fingerprint confirmation
 The confirmation signal is the **multiset of internal CRC32s** of the archive's
 entries (CRC32 of each entry's *uncompressed* data). This value is a property of
 the archive contents and is identical regardless of which channel hosts the
 file.
 - **Candidate side:** for manual uploads, `PackageFile.crc32` is already
  populated from the local central-directory read — zero cost. For **rebuild
  records** (`fileCount == 0`, no `crc32`), there is nothing stored to compare
  against; obtain the candidate's fingerprint with a **second ranged tail read
  of its destination copy** (ZIP/7z — still no full download). RAR candidates
  cannot be tail-read on either side, so they fall back to name+size (see §5).
 - **Scanned-source side:** read via a **ranged (tail) download** of the central
  directory:
  - **ZIP** (incl. multipart): the End-of-Central-Directory record + central
    directory live at the tail of the last part. Download only that tail and
    parse entries. (Reuses the lightweight-listing mechanism that is
    sub-project 4's core.)
  - **7z**: a start header at byte 0 points to an end header at the tail; fetch
    both small pieces.
  - **RAR**: headers are scattered through the file — no cheap tail read. See §5.
 - **Match rule:** confirmed iff the sorted CRC32 multisets are equal **and** file
  counts are equal.
 The comparison logic (CRC32 multiset equality) and the candidate-match predicate
 are implemented as **pure functions** with no TDLib dependency, so they are unit
 testable (see §7).
 ### 4. What gets written
 On a confirmed match, update the candidate package in a single transaction,
 overwriting only placeholder/empty fields:
 | Field | New value | Condition |
 |---|---|---|
 | `sourceChannelId` | scanned `channel.id` | always |
 | `sourceMessageId` | scanned `parts[0].id` | always |
 | `sourceTopicId` | `ctx.sourceTopicId` | always |
 | `sourceCaption` | scanned message caption | always |
 | `remoteUniqueId` | scanned `firstRemoteUniqueId` | always (enables future same-channel dedup via check #1) |
 | `creator` | re-derived from source (topic > filename > channel) | **always** — un-normalized; the later creator-normalization sub-project cleans it up |
 | `fileCount` + `PackageFile[]` | from the central-directory read | only if candidate had none (`fileCount == 0`) — folds in the sub-project 4 outcome for rebuild records |
 | `previewData` / `previewMsgId` | matched preview from scan | only if candidate has none |
 **Left untouched:** `contentHash`, `destChannelId`, `destMessageId`,
 `destMessageIds` — the bytes physically live in the destination channel; that is
 correct and must not change.
 **Transaction safety:** re-check `sourceChannelId == destChannelId` *inside* the
 transaction before writing, so a concurrent worker that already backfilled the
 record causes this one to no-op (mirrors the existing `backfill.ts` guard).
 ### 5. RAR handling
 RAR sources cannot be tail-read, so no cheap CRC fingerprint is available.
 Decision: **backfill RAR matches on `fileName` + `fileSize` alone**, flagged as
 lower-confidence in logs and via a `SystemNotification`, so they can be audited.
 No full download.
 This name+size-only fallback applies to any candidate that cannot produce a
 CRC32 fingerprint cheaply: a RAR **source**, a RAR **candidate**, or a rebuild
 candidate whose destination copy is RAR. ZIP/7z rebuild candidates are still
 confirmed by fingerprint via the second tail read described in §3.
 ### 6. Conflict, mismatch, ambiguity
 - **Fingerprint mismatch** → the scanned archive is genuinely different content
  that merely shares name+size. Fall through to **normal ingestion** (download +
  index) — it is a new package for this source.
 - **Ambiguous candidates** (2+ match name+size and the fingerprint cannot
  disambiguate) → log a `SystemNotification`, backfill nothing, and fall through
  to normal ingestion; the post-download `packageExistsByHash` check still
  dedups it safely.
 - **Same archive in multiple source channels** → the **first re-indexed source
  that confirms wins.** After backfill the package has a real source, so it is no
  longer a candidate; later scans treat it as an ordinary duplicate via checks #1
  (the `remoteUniqueId` we set) or #3.
 ### 7. Idempotency
 Falls out of the candidate definition. Once backfilled:
 - `sourceChannelId` is the real channel → no longer a candidate.
 - A re-scan of that source hits check #1 (`remoteUniqueId`, now set) or check #3
  (`findRepostedPackage`) → normal dedup skip. No re-mutation, no duplicate.
 ### 8. Error handling
 - **Tail-download failure** (network / `FLOOD_WAIT`) → cannot confirm this round.
  Do **not** fall back to a full download and do **not** guess: leave the
  candidate untouched and let the next re-index retry. Wrap the ranged read in
  `withFloodWait` (per the TDLib skill).
 - All new TDLib calls follow the skill's patterns: `withFloodWait`, listener
  attached before the async op, client closed in `finally`.
 ### 9. Visibility
 - Add a `zipsBackfilled` counter to `IngestionRun` activity so a re-index run
  reports "N provenance backfills" instead of silently mutating records.
 - Info log per backfill (candidate id, old vs new source, confidence:
  fingerprint | name+size-RAR).
 - `SystemNotification` for ambiguous-candidate cases.
 ## Testing
 The repo currently has no test framework (`CLAUDE.md`: "testing is manual"). This
 work introduces a **lightweight test harness** for the worker (e.g. `vitest` or
 node's built-in `node:test`) and unit-tests the correctness-critical pure logic:
 - **Unit (automated):**
  - CRC32 multiset fingerprint equality (equal sets, different order, differing
    counts, disjoint sets).
  - Candidate-match predicate (name+size+placeholder true/false cases).
  - Field-merge rules (which fields overwrite, which only fill-if-empty).
 - **Manual integration checklist:**
  1. Re-index a real source channel containing a known manually-uploaded pack →
     verify `sourceChannelId`/`sourceMessageId`/`sourceCaption`/`creator` are
     backfilled and no full download occurs.
  2. A rebuild-created record (`fileCount == 0`) → verify listing + provenance
     are both populated from the single tail read.
  3. A RAR pack → verify name+size backfill with the lower-confidence log/notice.
  4. Re-run the same re-index → verify it is a no-op (idempotent).
  5. A genuine name+size collision (different content) → verify it ingests as a
     new package rather than being mis-attributed.
 ## Open Questions
 None blocking. Preferred-origin selection among multiple real sources is
 deliberately out of scope (first confirmed wins).
 ## Affected Code (indicative, for planning)
 - `worker/src/worker.ts` — `processOneArchiveSet`: new Stage A/B step; new counter.
 - `worker/src/archive/` — ranged central-directory reader (ZIP/7z tail); shared
  with sub-project 4. Pure CRC32-fingerprint compare helper.
 - `worker/src/tdlib/download.ts` — ranged/partial download support (`offset`/`limit`).
 - `worker/src/db/queries.ts` — candidate lookup + transactional backfill update.
 - `prisma/schema.prisma` — `IngestionRun.zipsBackfilled` (and run-counter plumbing).
 - Worker test harness + first unit tests.
@@ -0,0 +1,180 @@
 # Ranged inner-file listing for RAR & 7z — design
 **Date:** 2026-07-27
 **Status:** Approved (design), pending spec review → implementation plan
 ## Problem
 The reindex/provenance-backfill path (`worker/src/provenance-backfill.ts`) can index an
 archive's inner files *without* re-downloading it, by reading the file listing from a small
 ranged read of the copy already in the source/destination channel. This works **only for
 ZIP** today (ZIP keeps its central directory in a tail that `parseZipCentralDirectoryFromTail`
 reads). For **RAR and 7z**, `tryProvenanceBackfill` backfills provenance (creator, source
 channel, `remoteUniqueId`) so the file is skipped on re-scan and never re-downloaded — but it
 leaves the inner listing empty (`fileCount = 0`), because `scannedEntries` is only computed for
 ZIP.
 Scope of the gap (rebuild placeholders with `fileCount = 0`, as of 2026-07-27):
 | Type | Count | Total | Avg | Max | Multipart |
 |---|---|---|---|---|---|
 | 7z | 19,646 | 14 TB | 0.73 GB | 3.9 GB | 0 |
 | RAR | 12,780 | 13 TB | 1.04 GB | 116 GB | 1,016 |
 A "just download the whole archive" fallback for all of these means ~27 TB of re-downloads —
 the exact cost this path exists to avoid.
 ## Goals
 - Index the inner files (names + sizes; CRCs where cheaply available) of RAR and 7z
  placeholders **without** downloading the whole archive in the common case.
 - Reuse the existing, battle-tested CLI listing parsers (`parse7zOutput`,
  `parseUnrarTechnical`) rather than reimplementing filename/size/CRC extraction.
 - Keep cost proportional to **file count**, not archive size (so even the 116 GB RAR is cheap).
 - Guarantee a listing for the rare archives the cheap path can't handle, via a full-download
  fallback that respects the existing max-size guard.
 ## Non-goals
 - No change to ingestion of genuinely-new files (those are downloaded in full to be re-uploaded
  regardless, so a cheap listing does not help there). This feature only affects the
  provenance-backfill / skip path.
 - No new ZIP behaviour — the existing ZIP tail reader stays as-is.
 - Not attempting to list password-encrypted-header archives from ranged reads (no password);
  those take the fallback and, if oversized, are flagged.
 ## Approach (chosen: "harvest header regions → sparse file → native CLI")
 Do the *minimum* binary parsing needed to locate an archive's header bytes, fetch only those
 via ranged reads, write them into a sparse temp file at their true offsets (data regions left
 as unwritten zero holes → ~no disk use), then run the real `7z l` / `unrar lt` and reuse the
 existing parsers. The native tools do the hard parsing (7z's LZMA-encoded headers, RAR's two
 format versions, Unicode names) — we only compute where the headers are.
 Rejected alternatives: full native TS parsers (most custom binary code, highest risk);
 RAR5-quick-open-only (most community RARs lack it → collapses to ~13 TB of RAR downloads).
 ## Components
 New directory `worker/src/archive/ranged/`, one focused module per concern, all returning the
 existing `FileEntry[]` type from `zip-reader.ts`:
 - `sparse-list.ts` — `listFromSparse(parts, runner, parse) → FileEntry[] | null`, where each
  `part` is `{ fileName, size, regions: {offset, bytes}[] }`. For each part it writes a sparse
  temp file (`truncate` to `size`, then write only the header `regions` at their offsets),
  co-locates all parts in one temp dir under their real names, invokes the supplied CLI runner
  (`7z l` / `unrar lt`) on the first part, feeds stdout to the supplied `parse` fn
  (`parse7zOutput` / `parseUnrarTechnical`), and cleans up. Single-part archives are just the
  one-element case. Returns `null` on CLI error / empty parse.
 - `sevenz-ranged.ts` — `readSevenZListingRanged(client, parts) → FileEntry[] | null`.
 - `rar-ranged.ts` — `readRarListingRanged(client, parts) → FileEntry[] | null`.
 - Dispatcher in `provenance-backfill.ts`: `readScannedListingRanged(archiveType, client, parts)`
  replacing the current `if (archiveType === "ZIP")` branch; the destination-copy read in
  `resolveCandidateFingerprintEntries` gets the same dispatch.
 `FileEntry` shape (unchanged): `{ path, fileName, extension, compressedSize, uncompressedSize, crc32 }`.
 ### 7z ranged listing
 7z layout: 32-byte signature header at offset 0 → packed streams → end header (lists files) at
 the end; the signature header stores the end header's location.
 1. Ranged-read `[0, 32)`; validate magic `37 7A BC AF 27 1C`. Read LE `uint64`
   `NextHeaderOffset` (byte 12) and `NextHeaderSize` (byte 20). End header is at absolute offset
   `32 + NextHeaderOffset`, length `NextHeaderSize`.
 2. Ranged-read `[32 + NextHeaderOffset, NextHeaderSize)` — the "next header".
 3. Branch on the next header's first byte (a 7z property id):
   - **`0x01` (kHeader, plain/uncompressed header):** two regions suffice —
     `{0: sigHeader}` and `{32+NextHeaderOffset: endHeader}`.
   - **`0x17` (kEncodedHeader, LZMA-compressed header):** the next header is only a *descriptor*
     whose `PackInfo` points at a packed header stream stored **in the middle** of the file (not
     at EOF). Parse the descriptor's `StreamsInfo → kPackInfo (0x06)` to read `PackPos` and the
     `PackSize`s (7z variable-length "numbers"; sum them). Ranged-read the contiguous packed
     region `[32 + PackPos, Σ PackSize)` and add it as a **third** sparse region. `7z l` then
     decodes the header from that region.
   - **anything else:** return `null` (→ fallback).
 4. `listFromSparse` with the 2 or 3 regions, runner = `7z l`. `parse7zOutput` yields
   names+sizes (`crc32: null`, as today). Return `null` on bad magic / read failure / CLI error.
 **Why the third region is required (spike finding, 2026-07-27):** the original two-region
 (start+end) reconstruction was proven insufficient in a live test — `7z l` rejected it with
 "Cannot open the file as [7z] archive" because these archives use an *encoded* header whose
 compressed bytes live in a packed stream in the file body (a sparse hole), not at EOF. The
 `0x17` branch fetches exactly that packed region. `7z l` still never touches the file-data
 packed streams (it only lists), so those gaps stay sparse. The `read7zNumber` reader (7z's
 base-128-ish variable-length integer with a first-byte length mask) and the minimal
 `kPackInfo` walk are the only new 7z binary parsing; `7z l` still does the actual file listing.
 All 7z placeholders are single-part.
 ### RAR ranged listing
 RAR has no index; walk the block chain, parsing only each block's **size fields** to step
 forward and harvest header bytes.
 1. Read first ~16 bytes; detect **RAR4** (`52 61 72 21 1A 07 00`) vs **RAR5** (`…07 01 00`) and
   the signature length.
 2. From just after the signature, loop:
   - Ranged-read a header chunk (start 8 KB; if parsed `HeaderSize` exceeds it — long filenames
     — re-read exactly).
   - Minimal block-extent parse:
     - RAR5: `CRC32(4)` + vint `HeaderSize` + vint `HeaderType` + vint `HeaderFlags`; if the
       "extra area" flag (`0x0001`) → vint `ExtraAreaSize`; if the "data present" flag
       (`0x0002`) → vint `DataSize`. Next block = `pos + 4 + len(HeaderSize vint) + HeaderSize
       + DataSize`.
     - RAR4: `HEAD_CRC(2)` + `HEAD_TYPE(1)` + `HEAD_FLAGS(2)` + `HEAD_SIZE(2)`; if flag `0x8000`
       → `ADD_SIZE(4)`. Next block = `pos + HEAD_SIZE + ADD_SIZE`.
   - Harvest `[blockOffset, blockOffset + HeaderSize)` into the regions list.
   - Stop at the end-of-archive block or EOF.
 3. `listFromSparse` (headers present, data sparse) → `unrar lt` → `parseUnrarTechnical`. RAR
   headers carry CRC32, so RAR contributes CRCs (fingerprint disambiguation keeps working).
 **Multipart RAR** (1,016): each volume starts with its own signature + headers. Walk **each
 part from its own signature**, reconstruct one sparse temp file per part with correct names
 (`name.part1.rar`, `.part2.rar`, …) co-located in a temp dir, and run `unrar lt` on part 1 —
 `unrar` auto-discovers co-located siblings (per the existing reader's note). The global-offset →
 `(part, offsetInPart)` mapping reuses the multipart size math the ZIP path already uses.
 ## Fallback & integration
 - A ranged reader returning `null` = cheap read failed (bad magic, read error, walk gave up, or
  CLI error on the sparse file) → **full-download fallback**: download the whole archive, run the
  existing `readRarContents` / `read7zContents`, backfill.
 - The fallback is gated by `config.maxZipSizeMB` (the same guard used at ingest). Over the cap →
  no download; write a `SystemNotification` (`INTEGRITY_AUDIT`, WARNING) and leave the listing
  empty for manual review. This ensures nothing pathological (e.g. the 116 GB RAR) is pulled.
 - Downstream is unchanged: `compareFingerprints` already treats null/incomplete CRCs as
  "incomplete" (name-size path), and `backfillProvenance` writes entries when the candidate's
  `fileCount === 0`.
 - Observability: reuse the `zipsBackfilled` counter; add structured logs with
  `confidence: "ranged" | "full-download-fallback"` and a WARN on fallback so miss-rate is
  visible.
 ## Risks & de-risking spike (do before the full build)
 On 3–4 real placeholder archives per format:
 1. Confirm `downloadFileRange` returns correct bytes at **arbitrary (non-tail) offsets** —
   currently only tail-verified in production. Underpins everything; if it fails, stop and
   rethink. (Note: `range-download.ts` flags absolute-offset behaviour as pending live
   verification; tail reads are proven by the 43 ZIP backfills done 2026-07-26.)
 2. Confirm `7z l` and `unrar lt` list correctly from a **sparse reconstructed file** — single
   part first, then multipart RAR (the highest-risk case).
 If multipart-RAR sparse reconstruction proves unreliable in the spike, multipart RAR uses the
 full-download fallback (respecting the size cap → oversized ones flagged, not downloaded).
 ## Testing
 - **Unit (vitest, alongside `central-directory.test.ts`):** 7z signature-header parse; RAR4 &
  RAR5 block-extent walk against committed small fixtures; `sparse-list` writes the correct
  regions. Pure logic, no TDLib.
 - **Live post-deploy:** watch `zipsBackfilled` climb for RAR/7z via the ranged path; spot-check
  a handful of backfilled packages' `package_files` against a real `unrar lt` / `7z l` on a full
  download of the same file; confirm the fallback/flag path fires on a deliberately-broken case.
 ## Rollout
 Local build + deploy (no GitHub push required), per the established recipe: build
 `worker/Dockerfile` locally, recreate the `dragonsstash-worker` container from the local image
 (no `pull`). No new DB migration. The scheduler re-runs hourly and will backfill RAR/7z
 placeholders on subsequent cycles.
@@ -0,0 +1,30 @@
 -- CreateEnum
 CREATE TYPE "ManualUploadStatus" AS ENUM ('PENDING', 'PROCESSING', 'COMPLETED', 'FAILED');
 -- CreateTable
 CREATE TABLE "manual_uploads" (
    "id" TEXT NOT NULL,
    "status" "ManualUploadStatus" NOT NULL DEFAULT 'PENDING',
    "groupName" TEXT,
    "userId" TEXT NOT NULL,
    "errorMessage" TEXT,
    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
    "completedAt" TIMESTAMP(3),
    CONSTRAINT "manual_uploads_pkey" PRIMARY KEY ("id")
 );
 CREATE TABLE "manual_upload_files" (
    "id" TEXT NOT NULL,
    "uploadId" TEXT NOT NULL,
    "fileName" TEXT NOT NULL,
    "filePath" TEXT NOT NULL,
    "fileSize" BIGINT NOT NULL,
    "packageId" TEXT,
    CONSTRAINT "manual_upload_files_pkey" PRIMARY KEY ("id")
 );
 CREATE INDEX "manual_uploads_status_idx" ON "manual_uploads"("status");
 CREATE INDEX "manual_upload_files_uploadId_idx" ON "manual_upload_files"("uploadId");
 ALTER TABLE "manual_uploads" ADD CONSTRAINT "manual_uploads_userId_fkey" FOREIGN KEY ("userId") REFERENCES "User"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
 ALTER TABLE "manual_upload_files" ADD CONSTRAINT "manual_upload_files_uploadId_fkey" FOREIGN KEY ("uploadId") REFERENCES "manual_uploads"("id") ON DELETE CASCADE ON UPDATE CASCADE;
@@ -0,0 +1,2 @@
 -- AlterTable
 ALTER TABLE "telegram_accounts" ADD COLUMN "isPremium" BOOLEAN NOT NULL DEFAULT false;
@@ -0,0 +1,8 @@
 -- AlterTable: track how many times the worker has tried each skipped source message.
 -- Existing rows default to 1 (they represent a single past attempt that the worker
 -- chose to record). Future failures increment via upsertSkippedPackage.
 ALTER TABLE "skipped_packages" ADD COLUMN "attemptCount" INTEGER NOT NULL DEFAULT 1;
 -- AlterEnum: add CHANNEL_ACCESS_LOST so the worker can surface a notification
 -- when a source channel becomes inaccessible (account removed, channel deleted, etc.)
 ALTER TYPE "NotificationType" ADD VALUE 'CHANNEL_ACCESS_LOST';
@@ -0,0 +1,10 @@
 -- AlterTable: capture TDLib's stable per-content identifier for new packages.
 -- Existing rows are NULL; they fall through to the other dedup checks until
 -- they're re-encountered organically.
 ALTER TABLE "packages" ADD COLUMN "remoteUniqueId" TEXT;
 -- CreateIndex: scoped to source channel because we want to dedup
 -- per-channel (the same file appearing in two different channels is still
 -- worth indexing twice — they're different ingestion sources).
 CREATE INDEX "packages_sourceChannelId_remoteUniqueId_idx"
  ON "packages"("sourceChannelId", "remoteUniqueId");
@@ -0,0 +1,11 @@
 -- AlterTable: per-channel scan-state columns
 ALTER TABLE "account_channel_map"
  ADD COLUMN "lastScannedAt" TIMESTAMP(3),
  ADD COLUMN "lastScanFoundArchives" BOOLEAN NOT NULL DEFAULT false,
  ADD COLUMN "consecutiveEmptyScans" INTEGER NOT NULL DEFAULT 0;
 -- AlterTable: per-topic scan-state columns (forum channels)
 ALTER TABLE "topic_progress"
  ADD COLUMN "lastScannedAt" TIMESTAMP(3),
  ADD COLUMN "lastScanFoundArchives" BOOLEAN NOT NULL DEFAULT false,
  ADD COLUMN "consecutiveEmptyScans" INTEGER NOT NULL DEFAULT 0;
@@ -0,0 +1,5 @@
 -- AlterTable: per-topic user-controlled fetch toggle (forum channels)
 -- Additive, safe default (true) so existing rows backfill to "enabled" and
 -- current behaviour is unchanged. Disabling is non-destructive.
 ALTER TABLE "topic_progress"
  ADD COLUMN "fetchEnabled" BOOLEAN NOT NULL DEFAULT true;
@@ -0,0 +1,6 @@
 -- AlterTable: track the forum topic currently being processed on the live run
 -- so the worker status panel can offer a "skip & disable topic" action.
 -- Additive, nullable — no data change for existing rows.
 ALTER TABLE "ingestion_runs"
  ADD COLUMN "currentTopicId" BIGINT,
  ADD COLUMN "currentAccountChannelMapId" TEXT;
@@ -0,0 +1,5 @@
 -- AlterTable: count of packages whose provenance was backfilled during a run
 -- (opportunistic cross-channel provenance backfill). Additive, non-null with a
 -- default of 0 — no data change for existing rows.
 ALTER TABLE "ingestion_runs"
  ADD COLUMN "zipsBackfilled" INTEGER NOT NULL DEFAULT 0;
@@ -39,9 +39,10 @@ model User {
  settings     UserSettings?
  telegramLink TelegramLink?
  kickstarters Kickstarter[]
-  inviteCodes  InviteCode[]   @relation("InviteCreator")
+  inviteCodes    InviteCode[]   @relation("InviteCreator")
-  usedInvite   InviteCode?    @relation("InviteUser", fields: [usedInviteId], references: [id], onDelete: SetNull)
+  usedInvite     InviteCode?    @relation("InviteUser", fields: [usedInviteId], references: [id], onDelete: SetNull)
-  usedInviteId String?
+  usedInviteId   String?
  manualUploads  ManualUpload[]
 }
 model Account {
@@ -405,6 +406,7 @@ model TelegramAccount {
  isActive    Boolean   @default(true)
  authState   AuthState @default(PENDING)
  authCode    String?
  isPremium   Boolean   @default(false)
  lastSeenAt  DateTime?
  createdAt   DateTime  @default(now())
  updatedAt   DateTime  @updatedAt
@@ -448,6 +450,17 @@ model AccountChannelMap {
  channelId              String
  role                   ChannelRole @default(READER)
  lastProcessedMessageId BigInt?
  /// When this channel was last scanned (any reason, including skipped scans
  /// that bumped the timestamp). Used by the recency-skip guard.
  lastScannedAt          DateTime?
  /// True if the last scan found archives OR left retryable SkippedPackages
  /// pending. Tracks "this channel has work I might need to revisit" — not
  /// just "I uploaded something this cycle".
  lastScanFoundArchives  Boolean     @default(false)
  /// Number of consecutive cycles where this channel was trulyIdle (no
  /// archives, no failures, no retryables). Drives the backoff that lets
  /// cold channels skip cycles entirely.
  consecutiveEmptyScans  Int         @default(0)
  createdAt              DateTime    @default(now())
  account        TelegramAccount @relation(fields: [accountId], references: [id], onDelete: Cascade)
@@ -470,6 +483,11 @@ model Package {
  sourceChannelId String
  sourceMessageId BigInt
  sourceTopicId   BigInt?
  /// TDLib's `remote.unique_id` for the FIRST part's file. Stable across
  /// reposts of identical content in the same channel — used as the
  /// strongest pre-download dedup signal (no false positives unlike
  /// fileName + size matching).
  remoteUniqueId  String?
  destChannelId   String?
  destMessageId   BigInt?
  destMessageIds  BigInt[]    @default([])
@@ -501,6 +519,7 @@ model Package {
  @@index([archiveType])
  @@index([creator])
  @@index([packageGroupId])
  @@index([sourceChannelId, remoteUniqueId])
  @@map("packages")
 }
@@ -550,6 +569,7 @@ model IngestionRun {
  zipsFound       Int             @default(0)
  zipsDuplicate   Int             @default(0)
  zipsIngested    Int             @default(0)
  zipsBackfilled  Int             @default(0)
  errorMessage    String?
  // Live activity tracking — written by worker in real-time
@@ -563,6 +583,8 @@ model IngestionRun {
  totalBytes      BigInt?         // Total size of current download
  downloadPercent Int?            // 0-100
  lastActivityAt  DateTime?       // When activity was last updated
  currentTopicId             BigInt?  // Forum topic currently being processed (live "skip topic")
  currentAccountChannelMapId String?  // AccountChannelMap owning the topic being processed
  account  TelegramAccount @relation(fields: [accountId], references: [id])
  packages Package[]
@@ -578,7 +600,20 @@ model TopicProgress {
  accountChannelMapId    String
  topicId                BigInt
  topicName              String?
  /// User-controlled fetch toggle. When false, the worker skips this topic
  /// (no scanning, no fetching/transfer). Defaults true so every existing and
  /// newly-discovered topic is fetched unless explicitly disabled. Disabling
  /// is non-destructive — already-fetched packages are kept.
  fetchEnabled           Boolean   @default(true)
  lastProcessedMessageId BigInt?
  /// When this topic was last scanned (any reason). Used by recency-skip.
  lastScannedAt          DateTime?
  /// True if the last scan found archives OR has retryable SkippedPackages
  /// pending for this topic. See AccountChannelMap doc for details.
  lastScanFoundArchives  Boolean   @default(false)
  /// Number of consecutive cycles where this topic was trulyIdle. Drives
  /// backoff for cold topics.
  consecutiveEmptyScans  Int       @default(0)
  accountChannelMap AccountChannelMap @relation(fields: [accountChannelMapId], references: [id], onDelete: Cascade)
@@ -739,6 +774,12 @@ model SkippedPackage {
  sourceTopicId   BigInt?
  isMultipart     Boolean          @default(false)
  partCount       Int              @default(1)
  /// How many times the worker has tried to process this source message.
  /// The worker auto-retries failures across cycles up to a configurable cap
  /// (WORKER_MAX_SKIP_ATTEMPTS, default 5). After the cap, the watermark is
  /// allowed to advance past the failure so cycles aren't pinned forever;
  /// the user can manually retry via the UI to reset and try again.
  attemptCount    Int              @default(1)
  accountId       String
  account         TelegramAccount  @relation(fields: [accountId], references: [id], onDelete: Cascade)
  createdAt       DateTime         @default(now())
@@ -828,6 +869,7 @@ enum NotificationType {
  DOWNLOAD_FAILED
  GROUPING_CONFLICT
  INTEGRITY_AUDIT
  CHANNEL_ACCESS_LOST
 }
 enum NotificationSeverity {
@@ -865,3 +907,40 @@ model GroupingRule {
  @@index([sourceChannelId])
  @@map("grouping_rules")
 }
 enum ManualUploadStatus {
  PENDING
  PROCESSING
  COMPLETED
  FAILED
 }
 model ManualUpload {
  id              String             @id @default(cuid())
  status          ManualUploadStatus @default(PENDING)
  groupName       String?            // Group name if multiple files
  userId          String
  errorMessage    String?
  createdAt       DateTime           @default(now())
  completedAt     DateTime?
  files           ManualUploadFile[]
  user            User               @relation(fields: [userId], references: [id])
  @@index([status])
  @@map("manual_uploads")
 }
 model ManualUploadFile {
  id              String       @id @default(cuid())
  uploadId        String
  fileName        String
  filePath        String       // Path on shared volume
  fileSize        BigInt
  packageId       String?      // Set after processing
  upload          ManualUpload @relation(fields: [uploadId], references: [id], onDelete: Cascade)
  @@index([uploadId])
  @@map("manual_upload_files")
 }
@@ -1,7 +1,23 @@
 import { redirect } from "next/navigation";
 import { auth } from "@/lib/auth";
 import { prisma } from "@/lib/prisma";
 import { Sidebar } from "@/components/layout/sidebar";
 import { Header } from "@/components/layout/header";
-export default function AppLayout({ children }: { children: React.ReactNode }) {
+export default async function AppLayout({ children }: { children: React.ReactNode }) {
  // Guard against a stale JWT session whose user no longer exists in the
  // database (e.g. after a DB reset). The signed cookie still passes edge
  // middleware, but every downstream query keyed on session.user.id would fail.
  // Send such sessions to /logout, which clears the cookie and returns to login.
  const session = await auth();
  if (session?.user?.id) {
    const user = await prisma.user.findUnique({
      where: { id: session.user.id },
      select: { id: true },
    });
    if (!user) redirect("/logout");
  }
  return (
    <div className="flex h-screen overflow-hidden">
      <div className="hidden lg:block">
@@ -7,6 +7,7 @@ import {
  Check,
  AlertCircle,
  ImageOff,
  Maximize2,
 } from "lucide-react";
 import {
  Dialog,
@@ -20,6 +21,7 @@ import { Button } from "@/components/ui/button";
 import { cn } from "@/lib/utils";
 import { toast } from "sonner";
 import { setPreviewFromExtract } from "../actions";
 import { ImageLightbox } from "./image-lightbox";
 interface ArchiveImage {
  id: string;
@@ -65,6 +67,7 @@ export function ArchivePreviewPicker({
  const [thumbnails, setThumbnails] = useState<Map<string, ThumbnailState>>(new Map());
  const [selectedPath, setSelectedPath] = useState<string | null>(null);
  const [isPending, startTransition] = useTransition();
  const [lightboxSrc, setLightboxSrc] = useState<string | null>(null);
  const pollTimers = useRef<Map<string, ReturnType<typeof setInterval>>>(new Map());
  // Track which paths have already been requested to avoid re-requesting
  const requestedPaths = useRef<Set<string>>(new Set());
@@ -290,71 +293,85 @@ export function ArchivePreviewPicker({
                  const isFailed = thumbState?.status === "failed";
                  return (
-                    <button
+                    <div key={img.id} className="group relative">
-                      key={img.id}
+                      {isLoaded && thumbState?.imageUrl && (
-                      type="button"
+                        <button
-                      className={cn(
+                          type="button"
-                        "relative aspect-square rounded-lg overflow-hidden border-2 transition-all",
+                          className="absolute top-1.5 left-1.5 z-10 flex h-6 w-6 items-center justify-center rounded-md bg-black/60 text-white opacity-0 transition-opacity hover:bg-black/80 group-hover:opacity-100"
-                        "hover:border-primary/50 cursor-pointer group",
+                          onClick={(e) => {
-                        isSelected
+                            e.stopPropagation();
-                          ? "border-primary ring-2 ring-primary/30"
+                            setLightboxSrc(thumbState.imageUrl!);
-                          : "border-border",
+                          }}
-                        isFailed && "opacity-60"
+                          title="Enlarge"
-                      )}
+                        >
-                      onClick={() => {
+                          <Maximize2 className="h-3.5 w-3.5" />
-                        if (isLoaded) {
+                        </button>
                          setSelectedPath(img.path);
                        } else if (isFailed) {
                          // Allow retry on failed
                          requestedPaths.current.delete(img.path);
                          requestThumbnail(img.path);
                        } else if (!thumbState || thumbState.status === "idle") {
                          requestThumbnail(img.path);
                        }
                      }}
                      title={img.path}
                    >
                      {isLoaded && thumbState.imageUrl ? (
                        <img
                          src={thumbState.imageUrl}
                          alt={img.fileName}
                          className="h-full w-full object-cover"
                          loading="lazy"
                        />
                      ) : isLoading ? (
                        <div className="h-full w-full flex items-center justify-center bg-muted">
                          <Loader2 className="h-5 w-5 animate-spin text-muted-foreground" />
                        </div>
                      ) : isFailed ? (
                        <div className="h-full w-full flex flex-col items-center justify-center bg-muted gap-1">
                          <AlertCircle className="h-4 w-4 text-destructive" />
                          <span className="text-[10px] text-destructive px-1 text-center">
                            Click to retry
                          </span>
                        </div>
                      ) : (
                        <div className="h-full w-full flex items-center justify-center bg-muted">
                          <ImageIcon className="h-5 w-5 text-muted-foreground" />
                        </div>
                      )}
                      <button
                        type="button"
                        className={cn(
                          "relative aspect-square w-full rounded-lg overflow-hidden border-2 transition-all",
                          "hover:border-primary/50 cursor-pointer",
                          isSelected
                            ? "border-primary ring-2 ring-primary/30"
                            : "border-border",
                          isFailed && "opacity-60"
                        )}
                        onClick={() => {
                          if (isLoaded) {
                            setSelectedPath(img.path);
                          } else if (isFailed) {
                            // Allow retry on failed
                            requestedPaths.current.delete(img.path);
                            requestThumbnail(img.path);
                          } else if (!thumbState || thumbState.status === "idle") {
                            requestThumbnail(img.path);
                          }
                        }}
                        title={img.path}
                      >
                        {isLoaded && thumbState.imageUrl ? (
                          <img
                            src={thumbState.imageUrl}
                            alt={img.fileName}
                            className="h-full w-full object-cover"
                            loading="lazy"
                          />
                        ) : isLoading ? (
                          <div className="h-full w-full flex items-center justify-center bg-muted">
                            <Loader2 className="h-5 w-5 animate-spin text-muted-foreground" />
                          </div>
                        ) : isFailed ? (
                          <div className="h-full w-full flex flex-col items-center justify-center bg-muted gap-1">
                            <AlertCircle className="h-4 w-4 text-destructive" />
                            <span className="text-[10px] text-destructive px-1 text-center">
                              Click to retry
                            </span>
                          </div>
                        ) : (
                          <div className="h-full w-full flex items-center justify-center bg-muted">
                            <ImageIcon className="h-5 w-5 text-muted-foreground" />
                          </div>
                        )}
-                      {/* Selection checkmark */}
+                        {/* Selection checkmark */}
-                      {isSelected && (
+                        {isSelected && (
-                        <div className="absolute top-1.5 right-1.5 h-5 w-5 rounded-full bg-primary flex items-center justify-center">
+                          <div className="absolute top-1.5 right-1.5 h-5 w-5 rounded-full bg-primary flex items-center justify-center">
-                          <Check className="h-3 w-3 text-primary-foreground" />
+                            <Check className="h-3 w-3 text-primary-foreground" />
-                        </div>
+                          </div>
-                      )}
+                        )}
-                      {/* File info overlay */}
+                        {/* File info overlay */}
-                      <div className="absolute bottom-0 left-0 right-0 bg-black/60 px-1.5 py-1 opacity-0 group-hover:opacity-100 transition-opacity">
+                        <div className="absolute bottom-0 left-0 right-0 bg-black/60 px-1.5 py-1 opacity-0 group-hover:opacity-100 transition-opacity">
-                        <p className="text-[10px] text-white truncate">
+                          <p className="text-[10px] text-white truncate">
-                          {img.fileName}
+                            {img.fileName}
-                        </p>
+                          </p>
-                        <p className="text-[9px] text-white/70">
+                          <p className="text-[9px] text-white/70">
-                          {formatBytes(img.size)}
+                            {formatBytes(img.size)}
-                        </p>
+                          </p>
-                      </div>
+                        </div>
-                    </button>
+                      </button>
                    </div>
                  );
                })}
              </div>
@@ -394,6 +411,13 @@ export function ArchivePreviewPicker({
          </div>
        )}
      </DialogContent>
      <ImageLightbox
        src={lightboxSrc}
        open={!!lightboxSrc}
        onOpenChange={(open) => {
          if (!open) setLightboxSrc(null);
        }}
      />
    </Dialog>
  );
 }
@@ -0,0 +1,82 @@
 "use client";
 import { useState } from "react";
 import { Check, ChevronsUpDown } from "lucide-react";
 import { cn } from "@/lib/utils";
 import { Button } from "@/components/ui/button";
 import {
  Command,
  CommandEmpty,
  CommandGroup,
  CommandInput,
  CommandItem,
  CommandList,
 } from "@/components/ui/command";
 import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover";
 interface CreatorFilterProps {
  creators: string[];
  value: string; // active creator, "" when none
  onChange: (creator: string) => void; // "" clears the filter
 }
 export function CreatorFilter({ creators, value, onChange }: CreatorFilterProps) {
  const [open, setOpen] = useState(false);
  return (
    <Popover open={open} onOpenChange={setOpen}>
      <PopoverTrigger asChild>
        <Button
          variant="outline"
          size="sm"
          role="combobox"
          aria-expanded={open}
          className="h-9 w-[200px] justify-between"
        >
          <span className="truncate">{value || "All Creators"}</span>
          <ChevronsUpDown className="ml-2 h-4 w-4 shrink-0 opacity-50" />
        </Button>
      </PopoverTrigger>
      <PopoverContent className="w-[240px] p-0" align="start">
        <Command>
          <CommandInput placeholder="Search creators..." className="h-9" />
          <CommandList>
            <CommandEmpty>No creators found.</CommandEmpty>
            <CommandGroup>
              <CommandItem
                value="__all__"
                onSelect={() => {
                  onChange("");
                  setOpen(false);
                }}
              >
                <Check
                  className={cn("mr-2 h-4 w-4", value === "" ? "opacity-100" : "opacity-0")}
                />
                All Creators
              </CommandItem>
              {creators.map((creator) => (
                <CommandItem
                  key={creator}
                  value={creator}
                  onSelect={() => {
                    onChange(creator);
                    setOpen(false);
                  }}
                >
                  <Check
                    className={cn(
                      "mr-2 h-4 w-4",
                      value === creator ? "opacity-100" : "opacity-0"
                    )}
                  />
                  <span className="truncate">{creator}</span>
                </CommandItem>
              ))}
            </CommandGroup>
          </CommandList>
        </Command>
      </PopoverContent>
    </Popover>
  );
 }
@@ -0,0 +1,41 @@
 "use client";
 import {
  Dialog,
  DialogContent,
  DialogTitle,
 } from "@/components/ui/dialog";
 interface ImageLightboxProps {
  src: string | null;
  alt?: string;
  open: boolean;
  onOpenChange: (open: boolean) => void;
 }
 /**
 * Full-size, in-page preview viewer. Renders the image at native size capped
 * to the viewport (object-contain). Dismiss via the close button, Esc, or by
 * clicking the overlay.
 */
 export function ImageLightbox({
  src,
  alt = "",
  open,
  onOpenChange,
 }: ImageLightboxProps) {
  return (
    <Dialog open={open} onOpenChange={onOpenChange}>
      <DialogContent className="w-auto max-w-[95vw] border-0 bg-transparent p-0 shadow-none sm:max-w-[90vw]">
        <DialogTitle className="sr-only">Enlarged preview image</DialogTitle>
        {src && (
          <img
            src={src}
            alt={alt}
            className="mx-auto max-h-[88vh] w-auto max-w-full rounded-lg object-contain"
          />
        )}
      </DialogContent>
    </Dialog>
  );
 }
@@ -1,12 +1,14 @@
 "use client";
 import { useState } from "react";
 import { type ColumnDef } from "@tanstack/react-table";
-import { FileArchive, Eye, ChevronRight, Layers, Ungroup, Send, ImagePlus, GitMerge } from "lucide-react";
+import { FileArchive, Eye, ChevronRight, Layers, Ungroup, Send, ImagePlus, GitMerge, Maximize2 } from "lucide-react";
 import { DataTableColumnHeader } from "@/components/shared/data-table-column-header";
 import { Badge } from "@/components/ui/badge";
 import { Button } from "@/components/ui/button";
 import { Checkbox } from "@/components/ui/checkbox";
 import { SendToTelegramButton } from "./send-to-telegram-button";
 import { ImageLightbox } from "./image-lightbox";
 export interface PackageRow {
  id: string;
@@ -84,14 +86,30 @@ export function formatBytes(bytesStr: string): string {
 }
 function PreviewCell({ pkg }: { pkg: PackageRow }) {
  const [lightboxOpen, setLightboxOpen] = useState(false);
  if (pkg.hasPreview) {
    const src = `/api/zips/${pkg.id}/preview`;
    return (
-      <img
+      <>
-        src={`/api/zips/${pkg.id}/preview`}
+        <button
-        alt=""
+          type="button"
-        className="h-9 w-9 rounded-md object-cover bg-muted"
+          className="group/preview relative h-9 w-9 overflow-hidden rounded-md bg-muted"
-        loading="lazy"
+          onClick={() => setLightboxOpen(true)}
-      />
+          title="Click to enlarge"
        >
          <img
            src={src}
            alt=""
            className="h-9 w-9 object-cover"
            loading="lazy"
          />
          <div className="absolute inset-0 flex items-center justify-center rounded-md bg-black/50 opacity-0 transition-opacity group-hover/preview:opacity-100">
            <Maximize2 className="h-3.5 w-3.5 text-white" />
          </div>
        </button>
        <ImageLightbox src={src} open={lightboxOpen} onOpenChange={setLightboxOpen} />
      </>
    );
  }
  return (
@@ -13,6 +13,7 @@ import {
  Upload,
  ImagePlus,
  Images,
  Maximize2,
 } from "lucide-react";
 import {
  Dialog,
@@ -30,6 +31,7 @@ import type { PackageRow } from "./package-columns";
 import { SendToTelegramButton } from "./send-to-telegram-button";
 import { uploadPackagePreview } from "../actions";
 import { ArchivePreviewPicker } from "./archive-preview-picker";
 import { ImageLightbox } from "./image-lightbox";
 interface FileItem {
  id: string;
@@ -264,6 +266,7 @@ export function PackageFilesDrawer({ pkg, open, onOpenChange, highlightTerm }: P
  const [uploading, setUploading] = useState(false);
  const [localPreviewUrl, setLocalPreviewUrl] = useState<string | null>(null);
  const [showPreviewPicker, setShowPreviewPicker] = useState(false);
  const [previewLightboxOpen, setPreviewLightboxOpen] = useState(false);
  const fileInputRef = useRef<HTMLInputElement>(null);
  const handlePreviewUpload = useCallback(
@@ -384,9 +387,8 @@ export function PackageFilesDrawer({ pkg, open, onOpenChange, highlightTerm }: P
              <button
                type="button"
                className="relative group h-20 w-20 shrink-0 rounded-lg overflow-hidden bg-muted"
-                onClick={() => fileInputRef.current?.click()}
+                onClick={() => setPreviewLightboxOpen(true)}
-                disabled={uploading}
+                title="Click to enlarge"
                title="Click to replace preview image"
              >
                <img
                  src={localPreviewUrl ?? `/api/zips/${pkg!.id}/preview`}
@@ -394,11 +396,7 @@ export function PackageFilesDrawer({ pkg, open, onOpenChange, highlightTerm }: P
                  className="h-full w-full object-cover"
                />
                <div className="absolute inset-0 bg-black/50 opacity-0 group-hover:opacity-100 transition-opacity flex items-center justify-center">
-                  {uploading ? (
+                  <Maximize2 className="h-5 w-5 text-white" />
                    <Loader2 className="h-5 w-5 text-white animate-spin" />
                  ) : (
                    <Upload className="h-5 w-5 text-white" />
                  )}
                </div>
              </button>
            ) : (
@@ -582,6 +580,11 @@ export function PackageFilesDrawer({ pkg, open, onOpenChange, highlightTerm }: P
        }}
      />
    )}
    <ImageLightbox
      src={pkg ? (localPreviewUrl ?? `/api/zips/${pkg.id}/preview`) : null}
      open={previewLightboxOpen}
      onOpenChange={setPreviewLightboxOpen}
    />
    </>
  );
 }
@@ -20,6 +20,7 @@ export interface SkippedRow {
  sourceChannel: { id: string; title: string };
  isMultipart: boolean;
  partCount: number;
  attemptCount: number;
  createdAt: string;
 }
@@ -107,9 +108,22 @@ export function getSkippedColumns({
      ),
      accessorFn: (row) => row.sourceChannel.title,
    },
    {
      accessorKey: "attemptCount",
      header: ({ column }) => <DataTableColumnHeader column={column} title="Attempts" />,
      cell: ({ row }) => {
        const count = row.original.attemptCount;
        const variant = count >= 5 ? "destructive" : count > 1 ? "secondary" : "outline";
        return (
          <Badge variant={variant} className="text-[10px]">
            {count}
          </Badge>
        );
      },
    },
    {
      accessorKey: "createdAt",
-      header: ({ column }) => <DataTableColumnHeader column={column} title="Skipped" />,
+      header: ({ column }) => <DataTableColumnHeader column={column} title="Last Skipped" />,
      cell: ({ row }) => (
        <span className="text-sm text-muted-foreground">
          {new Date(row.original.createdAt).toLocaleDateString()}
@@ -3,7 +3,8 @@
 import { useState, useCallback, useTransition, useMemo, useRef } from "react";
 import { useRouter, usePathname, useSearchParams } from "next/navigation";
 import { toast } from "sonner";
-import { Search, Layers } from "lucide-react";
+import { Search, Layers, Upload, Send } from "lucide-react";
 import { UploadDialog } from "./upload-dialog";
 import { useDataTable } from "@/hooks/use-data-table";
 import {
  getPackageColumns,
@@ -15,6 +16,7 @@ import {
 import { PackageFilesDrawer } from "./package-files-drawer";
 import { IngestionStatus } from "./ingestion-status";
 import { SkippedPackagesTab } from "./skipped-packages-tab";
 import { CreatorFilter } from "./creator-filter";
 import { DataTable } from "@/components/shared/data-table";
 import { DataTablePagination } from "@/components/shared/data-table-pagination";
 import { DataTableViewOptions } from "@/components/shared/data-table-view-options";
@@ -48,6 +50,7 @@ import {
  createGroupAction,
  removeFromGroupAction,
  sendAllInGroupAction,
  sendAllFromCreatorAction,
  updateGroupPreviewAction,
  mergeGroupsAction,
 } from "../actions";
@@ -58,6 +61,7 @@ interface StlTableProps {
  totalCount: number;
  ingestionStatus: IngestionAccountStatus[];
  availableTags: string[];
  availableCreators: string[];
  searchTerm: string;
  skippedData: SkippedRow[];
  skippedPageCount: number;
@@ -73,6 +77,7 @@ export function StlTable({
  totalCount,
  ingestionStatus,
  availableTags,
  availableCreators,
  searchTerm,
  skippedData,
  skippedPageCount,
@@ -84,6 +89,7 @@ export function StlTable({
  const router = useRouter();
  const pathname = usePathname();
  const searchParams = useSearchParams();
  const activeCreator = searchParams.get("creator") ?? "";
  const [searchValue, setSearchValue] = useState(searchParams.get("search") ?? "");
  const [viewPkg, setViewPkg] = useState<PackageRow | null>(null);
@@ -106,6 +112,9 @@ export function StlTable({
  // Group merge state
  const [mergeSourceId, setMergeSourceId] = useState<string | null>(null);
  // Upload dialog state
  const [uploadOpen, setUploadOpen] = useState(false);
  const toggleGroup = useCallback((groupId: string) => {
    setExpandedGroups((prev) => {
      const next = new Set(prev);
@@ -203,6 +212,20 @@ export function StlTable({
    [router, pathname, searchParams]
  );
  const updateCreatorFilter = useCallback(
    (value: string) => {
      const params = new URLSearchParams(searchParams.toString());
      if (value) {
        params.set("creator", value);
        params.set("page", "1");
      } else {
        params.delete("creator");
      }
      router.push(`${pathname}?${params.toString()}`, { scroll: false });
    },
    [router, pathname, searchParams]
  );
  const activeTab = searchParams.get("tab") ?? "packages";
  const updateTab = useCallback(
@@ -273,6 +296,23 @@ export function StlTable({
    [router]
  );
  const handleSendAllFromCreator = useCallback(() => {
    if (!confirm(`Send all packages from "${activeCreator}" to your Telegram?`)) return;
    startTransition(async () => {
      const result = await sendAllFromCreatorAction(activeCreator);
      if (result.success) {
        const { queued, skipped } = result.data;
        toast.success(
          `Queued ${queued} package${queued === 1 ? "" : "s"} from ${activeCreator}` +
            (skipped ? ` (${skipped} already queued)` : "")
        );
        router.refresh();
      } else {
        toast.error(result.error);
      }
    });
  }, [activeCreator, router]);
  const handleRemoveFromGroup = useCallback(
    (packageId: string) => {
      startTransition(async () => {
@@ -460,14 +500,9 @@ export function StlTable({
              </Badge>
            )}
          </TabsTrigger>
-          <TabsTrigger value="ungrouped" className="gap-1.5">
+          {/* "Ungrouped" tab hidden: the STL list is now flat and grouping is
-            Ungrouped
+              no longer surfaced here. The tab content below is kept (unreachable)
-            {ungroupedTotalCount > 0 && (
+              to avoid churn; remove it and its data fetch if grouping is dropped. */}
              <Badge variant="secondary" className="h-5 px-1.5 text-[10px]">
                {ungroupedTotalCount}
              </Badge>
            )}
          </TabsTrigger>
        </TabsList>
        <TabsContent value="packages" className="space-y-4">
@@ -496,7 +531,29 @@ export function StlTable({
                </SelectContent>
              </Select>
            )}
            {availableCreators.length > 0 && (
              <CreatorFilter
                creators={availableCreators}
                value={activeCreator}
                onChange={updateCreatorFilter}
              />
            )}
            <DataTableViewOptions table={table} />
            <Button variant="outline" size="sm" className="h-9" onClick={() => setUploadOpen(true)}>
              <Upload className="mr-2 h-4 w-4" />
              Upload Files
            </Button>
            {activeCreator && (
              <Button
                variant="outline"
                size="sm"
                className="h-9 gap-1.5"
                onClick={handleSendAllFromCreator}
              >
                <Send className="h-3.5 w-3.5" />
                Send all from {activeCreator}
              </Button>
            )}
            {selectedPackages.size >= 2 && (
              <Button
                variant="outline"
@@ -587,6 +644,8 @@ export function StlTable({
        </DialogContent>
      </Dialog>
      <UploadDialog open={uploadOpen} onOpenChange={setUploadOpen} />
      {/* Hidden file input for group preview upload (Task 12) */}
      <input
        ref={previewInputRef}
@@ -0,0 +1,243 @@
 "use client";
 import { useState, useRef, useTransition, useEffect } from "react";
 import { Upload, File, X, Loader2, CheckCircle2, AlertCircle } from "lucide-react";
 import { toast } from "sonner";
 import { Button } from "@/components/ui/button";
 import { Input } from "@/components/ui/input";
 import { Label } from "@/components/ui/label";
 import {
  Dialog,
  DialogContent,
  DialogDescription,
  DialogFooter,
  DialogHeader,
  DialogTitle,
 } from "@/components/ui/dialog";
 interface UploadDialogProps {
  open: boolean;
  onOpenChange: (open: boolean) => void;
 }
 function formatSize(bytes: number): string {
  if (bytes >= 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)} GB`;
  if (bytes >= 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(0)} MB`;
  return `${(bytes / 1024).toFixed(0)} KB`;
 }
 type UploadStatus = "idle" | "uploading" | "processing" | "done" | "error";
 export function UploadDialog({ open, onOpenChange }: UploadDialogProps) {
  const [files, setFiles] = useState<File[]>([]);
  const [groupName, setGroupName] = useState("");
  const [status, setStatus] = useState<UploadStatus>("idle");
  const [error, setError] = useState<string | null>(null);
  const [isPending, startTransition] = useTransition();
  const fileInputRef = useRef<HTMLInputElement>(null);
  const pollRef = useRef<ReturnType<typeof setInterval> | null>(null);
  useEffect(() => {
    if (open) {
      setFiles([]);
      setGroupName("");
      setStatus("idle");
      setError(null);
    }
    return () => {
      if (pollRef.current) clearInterval(pollRef.current);
    };
  }, [open]);
  function handleFileChange(e: React.ChangeEvent<HTMLInputElement>) {
    if (e.target.files) {
      setFiles(Array.from(e.target.files));
    }
  }
  function removeFile(index: number) {
    setFiles((prev) => prev.filter((_, i) => i !== index));
  }
  function handleUpload() {
    if (files.length === 0) return;
    startTransition(async () => {
      setStatus("uploading");
      setError(null);
      try {
        const formData = new FormData();
        for (const file of files) {
          formData.append("files", file);
        }
        if (groupName.trim()) {
          formData.append("groupName", groupName.trim());
        }
        const res = await fetch("/api/uploads", {
          method: "POST",
          body: formData,
        });
        const data = await res.json();
        if (!res.ok) {
          setStatus("error");
          setError(data.error ?? "Upload failed");
          return;
        }
        setStatus("processing");
        // Poll for completion
        pollRef.current = setInterval(async () => {
          try {
            const statusRes = await fetch(`/api/uploads/${data.uploadId}`);
            const statusData = await statusRes.json();
            if (statusData.status === "COMPLETED") {
              setStatus("done");
              toast.success(`${files.length} file(s) uploaded and indexed`);
              if (pollRef.current) clearInterval(pollRef.current);
            } else if (statusData.status === "FAILED") {
              setStatus("error");
              setError(statusData.errorMessage ?? "Processing failed");
              if (pollRef.current) clearInterval(pollRef.current);
            }
          } catch {
            // Keep polling
          }
        }, 3000);
        // Stop polling after 10 minutes
        setTimeout(() => {
          if (pollRef.current) {
            clearInterval(pollRef.current);
            pollRef.current = null;
            setStatus((s) => s === "processing" ? "done" : s);
          }
        }, 600_000);
      } catch {
        setStatus("error");
        setError("Network error");
      }
    });
  }
  return (
    <Dialog open={open} onOpenChange={onOpenChange}>
      <DialogContent className="sm:max-w-lg">
        <DialogHeader>
          <DialogTitle>Upload Files</DialogTitle>
          <DialogDescription>
            Upload archive files to be processed and indexed. Multiple files will be automatically grouped.
          </DialogDescription>
        </DialogHeader>
        {status === "idle" && (
          <div className="space-y-4">
            <div
              className="border-2 border-dashed rounded-lg p-8 text-center cursor-pointer hover:border-primary/50 transition-colors"
              onClick={() => fileInputRef.current?.click()}
            >
              <Upload className="h-8 w-8 mx-auto mb-2 text-muted-foreground" />
              <p className="text-sm text-muted-foreground">
                Click to select files or drag & drop
              </p>
              <p className="text-xs text-muted-foreground mt-1">
                ZIP, RAR, 7Z files up to 4GB each
              </p>
              <input
                ref={fileInputRef}
                type="file"
                multiple
                accept=".zip,.rar,.7z,.pdf,.stl"
                onChange={handleFileChange}
                className="hidden"
              />
            </div>
            {files.length > 0 && (
              <div className="space-y-2">
                {files.map((file, i) => (
                  <div key={i} className="flex items-center gap-2 p-2 rounded bg-muted/30">
                    <File className="h-4 w-4 shrink-0 text-muted-foreground" />
                    <span className="text-sm flex-1 truncate">{file.name}</span>
                    <span className="text-xs text-muted-foreground">{formatSize(file.size)}</span>
                    <button onClick={() => removeFile(i)} className="p-0.5 hover:text-destructive">
                      <X className="h-3.5 w-3.5" />
                    </button>
                  </div>
                ))}
              </div>
            )}
            {files.length > 1 && (
              <div>
                <Label htmlFor="groupName" className="text-sm">Group Name (optional)</Label>
                <Input
                  id="groupName"
                  value={groupName}
                  onChange={(e) => setGroupName(e.target.value)}
                  placeholder="Auto-generated from filenames"
                  className="mt-1"
                />
              </div>
            )}
          </div>
        )}
        {(status === "uploading" || status === "processing") && (
          <div className="flex items-center gap-3 p-6 rounded-lg bg-muted/30 border">
            <Loader2 className="h-6 w-6 animate-spin text-primary" />
            <div>
              <p className="text-sm font-medium">
                {status === "uploading" ? "Uploading files..." : "Processing & uploading to Telegram..."}
              </p>
              <p className="text-xs text-muted-foreground mt-0.5">
                {status === "uploading"
                  ? "Sending files to server"
                  : "Hashing, extracting metadata, uploading to destination channel"}
              </p>
            </div>
          </div>
        )}
        {status === "done" && (
          <div className="flex items-center gap-3 p-6 rounded-lg bg-green-500/10 border border-green-500/20">
            <CheckCircle2 className="h-6 w-6 text-green-500" />
            <div>
              <p className="text-sm font-medium text-green-500">Upload complete!</p>
              <p className="text-xs text-muted-foreground">Files have been indexed and uploaded to Telegram.</p>
            </div>
          </div>
        )}
        {status === "error" && (
          <div className="flex items-center gap-3 p-6 rounded-lg bg-destructive/10 border border-destructive/20">
            <AlertCircle className="h-6 w-6 text-destructive" />
            <div>
              <p className="text-sm font-medium text-destructive">Upload failed</p>
              <p className="text-xs text-muted-foreground">{error}</p>
            </div>
          </div>
        )}
        <DialogFooter>
          {status === "idle" && (
            <>
              <Button variant="outline" onClick={() => onOpenChange(false)}>Cancel</Button>
              <Button onClick={handleUpload} disabled={files.length === 0 || isPending}>
                <Upload className="h-4 w-4 mr-1" />
                Upload {files.length > 0 ? `(${files.length})` : ""}
              </Button>
            </>
          )}
          {(status === "done" || status === "error") && (
            <Button variant="outline" onClick={() => onOpenChange(false)}>Close</Button>
          )}
        </DialogFooter>
      </DialogContent>
    </Dialog>
  );
 }
@@ -589,3 +589,82 @@ export async function sendAllInGroupAction(
    return { success: false, error: "Failed to send group packages" };
  }
 }
 export async function sendAllFromCreatorAction(
  creatorName: string
 ): Promise<ActionResult<{ queued: number; skipped: number }>> {
  const session = await auth();
  if (!session?.user?.id) return { success: false, error: "Unauthorized" };
  const creator = creatorName.trim();
  if (!creator) {
    return { success: false, error: "No creator specified" };
  }
  try {
    const telegramLink = await prisma.telegramLink.findUnique({
      where: { userId: session.user.id },
    });
    if (!telegramLink) {
      return { success: false, error: "No linked Telegram account. Link one in Settings." };
    }
    const sendablePackages = await prisma.package.findMany({
      where: {
        creator,
        destChannelId: { not: null },
        destMessageId: { not: null },
      },
      select: { id: true },
    });
    if (sendablePackages.length === 0) {
      return { success: false, error: "No uploaded packages found for this creator" };
    }
    let queued = 0;
    let skipped = 0;
    for (const pkg of sendablePackages) {
      // Only create if no existing PENDING/SENDING request for this package+link combo
      const existing = await prisma.botSendRequest.findFirst({
        where: {
          packageId: pkg.id,
          telegramLinkId: telegramLink.id,
          status: { in: ["PENDING", "SENDING"] },
        },
      });
      if (existing) {
        skipped++;
        continue;
      }
      const sendRequest = await prisma.botSendRequest.create({
        data: {
          packageId: pkg.id,
          telegramLinkId: telegramLink.id,
          requestedByUserId: session.user.id,
          status: "PENDING",
        },
      });
      // Notify the bot via pg_notify
      try {
        await prisma.$queryRawUnsafe(
          `SELECT pg_notify('bot_send', $1)`,
          sendRequest.id
        );
      } catch {
        // Best-effort — the bot also polls periodically
      }
      queued++;
    }
    revalidatePath("/stls");
    return { success: true, data: { queued, skipped } };
  } catch {
    return { success: false, error: "Failed to send creator packages" };
  }
 }
@@ -1,6 +1,6 @@
 import { auth } from "@/lib/auth";
 import { redirect } from "next/navigation";
-import { listDisplayItems, searchPackages, getIngestionStatus, getAllPackageTags, listSkippedPackages, countSkippedPackages, listUngroupedPackages, countUngroupedPackages } from "@/lib/telegram/queries";
+import { listDisplayItems, searchPackages, getIngestionStatus, getAllPackageTags, getAllPackageCreators, listSkippedPackages, countSkippedPackages, listUngroupedPackages, countUngroupedPackages } from "@/lib/telegram/queries";
 import { StlTable } from "./_components/stl-table";
 import type { DisplayItem, PackageListItem } from "@/lib/telegram/types";
@@ -24,7 +24,7 @@ export default async function StlFilesPage({ searchParams }: Props) {
  const tab = (params.tab as string) ?? "packages";
  // Fetch packages, ingestion status, tags, and skipped count in parallel
-  const [result, ingestionStatus, availableTags, skippedCount, ungroupedCount] = await Promise.all([
+  const [result, ingestionStatus, availableTags, availableCreators, skippedCount, ungroupedCount] = await Promise.all([
    search
      ? searchPackages({
          query: search,
@@ -42,6 +42,7 @@ export default async function StlFilesPage({ searchParams }: Props) {
        }),
    getIngestionStatus(),
    getAllPackageTags(),
    getAllPackageCreators(),
    countSkippedPackages(),
    countUngroupedPackages(),
  ]);
@@ -68,6 +69,7 @@ export default async function StlFilesPage({ searchParams }: Props) {
      totalCount={result.pagination.total}
      ingestionStatus={ingestionStatus}
      availableTags={availableTags}
      availableCreators={availableCreators}
      searchTerm={search}
      skippedData={skippedResult?.items ?? []}
      skippedPageCount={skippedResult?.pagination.totalPages ?? 0}
@@ -9,6 +9,7 @@ import {
  ArrowUpFromLine,
  RefreshCcw,
  Tag,
  MessagesSquare,
 } from "lucide-react";
 import { Badge } from "@/components/ui/badge";
 import { Button } from "@/components/ui/button";
@@ -27,6 +28,7 @@ interface ChannelColumnsProps {
  onSetType: (id: string, type: "SOURCE" | "DESTINATION") => void;
  onRescan: (id: string) => void;
  onSetCategory: (id: string, category: string | null) => void;
  onManageTopics: (id: string) => void;
 }
 export function getChannelColumns({
@@ -35,6 +37,7 @@ export function getChannelColumns({
  onSetType,
  onRescan,
  onSetCategory,
  onManageTopics,
 }: ChannelColumnsProps): ColumnDef<ChannelRow, unknown>[] {
  return [
    {
@@ -147,6 +150,14 @@ export function getChannelColumns({
                Rescan Channel
              </DropdownMenuItem>
            )}
            {row.original.type === "SOURCE" && row.original.isForum && (
              <DropdownMenuItem
                onClick={() => onManageTopics(row.original.id)}
              >
                <MessagesSquare className="mr-2 h-3.5 w-3.5" />
                Topics
              </DropdownMenuItem>
            )}
            <DropdownMenuItem
              onClick={() => {
                const cat = prompt("Enter category (e.g. STL, PDF, D&D, Cosplay):", row.original.category ?? "");
@@ -7,6 +7,7 @@ import { getChannelColumns } from "./channel-columns";
 import { DestinationCard } from "./destination-card";
 import { ChannelPickerDialog } from "./channel-picker-dialog";
 import { JoinChannelDialog } from "./join-channel-dialog";
 import { TopicsDrawer } from "./topics-drawer";
 import {
  deleteChannel,
  toggleChannelActive,
@@ -32,6 +33,7 @@ export function ChannelsTab({ channels, globalDestination, accounts }: ChannelsT
  const [rescanId, setRescanId] = useState<string | null>(null);
  const [fetchChannelsAccountId, setFetchChannelsAccountId] = useState<string | null>(null);
  const [joinDialogOpen, setJoinDialogOpen] = useState(false);
  const [topicsChannelId, setTopicsChannelId] = useState<string | null>(null);
  // Find the first authenticated account for "Fetch Channels"
  const authenticatedAccounts = accounts.filter((a) => a.authState === "AUTHENTICATED" && a.isActive);
@@ -60,6 +62,7 @@ export function ChannelsTab({ channels, globalDestination, accounts }: ChannelsT
        else toast.error(result.error);
      });
    },
    onManageTopics: (id) => setTopicsChannelId(id),
  });
  const { table } = useDataTable({
@@ -167,6 +170,17 @@ export function ChannelsTab({ channels, globalDestination, accounts }: ChannelsT
        open={joinDialogOpen}
        onOpenChange={setJoinDialogOpen}
      />
      <TopicsDrawer
        channelId={topicsChannelId}
        channelTitle={
          channels.find((c) => c.id === topicsChannelId)?.title
        }
        open={!!topicsChannelId}
        onOpenChange={(open) => {
          if (!open) setTopicsChannelId(null);
        }}
      />
    </div>
  );
 }
@@ -0,0 +1,182 @@
 "use client";
 import { useState, useEffect, useCallback, useMemo, useTransition } from "react";
 import { toast } from "sonner";
 import { Loader2, Search } from "lucide-react";
 import {
  Dialog,
  DialogContent,
  DialogDescription,
  DialogHeader,
  DialogTitle,
 } from "@/components/ui/dialog";
 import { Input } from "@/components/ui/input";
 import { Switch } from "@/components/ui/switch";
 import { setTopicFetchEnabled } from "../actions";
 interface TopicRow {
  id: string;
  topicId: string;
  topicName: string | null;
  fetchEnabled: boolean;
  lastScannedAt: string | null;
 }
 interface TopicsDrawerProps {
  channelId: string | null;
  channelTitle?: string;
  open: boolean;
  onOpenChange: (open: boolean) => void;
 }
 function topicLabel(t: TopicRow): string {
  return t.topicName ?? `Topic ${t.topicId}`;
 }
 export function TopicsDrawer({
  channelId,
  channelTitle,
  open,
  onOpenChange,
 }: TopicsDrawerProps) {
  const [topics, setTopics] = useState<TopicRow[]>([]);
  const [loading, setLoading] = useState(false);
  const [filter, setFilter] = useState("");
  const [, startTransition] = useTransition();
  const [pendingId, setPendingId] = useState<string | null>(null);
  const fetchTopics = useCallback(async () => {
    if (!channelId) return;
    setLoading(true);
    try {
      const res = await fetch(`/api/telegram/channels/${channelId}/topics`);
      if (res.ok) setTopics(await res.json());
      else toast.error("Failed to load topics");
    } catch {
      toast.error("Failed to load topics");
    }
    setLoading(false);
  }, [channelId]);
  useEffect(() => {
    if (open && channelId) fetchTopics();
  }, [open, channelId, fetchTopics]);
  const handleOpenChange = (next: boolean) => {
    // Reset the filter on close so the next open starts clean (the drawer is
    // opened per-channel, so close-then-open is the normal channel switch).
    if (!next) setFilter("");
    onOpenChange(next);
  };
  const handleToggle = (topic: TopicRow, enabled: boolean) => {
    // Optimistic update
    setTopics((prev) =>
      prev.map((t) => (t.id === topic.id ? { ...t, fetchEnabled: enabled } : t))
    );
    setPendingId(topic.id);
    startTransition(async () => {
      const result = await setTopicFetchEnabled(topic.id, enabled);
      if (!result.success) {
        toast.error(result.error ?? "Failed to update topic");
        // Revert on failure
        setTopics((prev) =>
          prev.map((t) =>
            t.id === topic.id ? { ...t, fetchEnabled: !enabled } : t
          )
        );
      }
      setPendingId(null);
    });
  };
  const filtered = useMemo(() => {
    const q = filter.trim().toLowerCase();
    if (!q) return topics;
    return topics.filter((t) => topicLabel(t).toLowerCase().includes(q));
  }, [topics, filter]);
  return (
    <Dialog open={open} onOpenChange={handleOpenChange}>
      <DialogContent className="sm:max-w-lg max-h-[80vh] flex flex-col gap-0 p-0">
        <DialogHeader className="px-6 pt-6 pb-4 border-b border-border space-y-3">
          <div className="space-y-1">
            <DialogTitle className="truncate pr-8">
              Topics{channelTitle ? `: ${channelTitle}` : ""}
            </DialogTitle>
            <DialogDescription>
              Enabled topics are scanned and their files fetched. Disable a topic
              to stop fetching new files from it — already-fetched files are kept.
            </DialogDescription>
          </div>
          {topics.length > 0 && (
            <div className="relative">
              <Search className="absolute left-2.5 top-2.5 h-4 w-4 text-muted-foreground" />
              <Input
                placeholder="Filter topics..."
                value={filter}
                onChange={(e) => setFilter(e.target.value)}
                className="pl-9 h-9"
              />
            </div>
          )}
        </DialogHeader>
        {/* Native overflow scroll: the Radix ScrollArea viewport does not get a
            bounded height inside this flex-column, max-h, vertically-centred
            dialog, so the list overflowed instead of scrolling. flex-1 +
            min-h-0 + overflow-y-auto is the canonical, touch-friendly fix. */}
        <div className="flex-1 min-h-0 overflow-y-auto">
          <div className="px-6 py-4 space-y-2">
            {loading ? (
              <div className="flex items-center justify-center gap-2 py-12">
                <Loader2 className="h-5 w-5 animate-spin text-muted-foreground" />
                <span className="text-sm text-muted-foreground">
                  Loading topics...
                </span>
              </div>
            ) : topics.length === 0 ? (
              <p className="py-12 text-center text-sm text-muted-foreground">
                No topics discovered yet — they&apos;ll appear here after the next
                scan.
              </p>
            ) : filtered.length === 0 ? (
              <p className="py-12 text-center text-sm text-muted-foreground">
                No topics match &quot;{filter}&quot;.
              </p>
            ) : (
              filtered.map((topic) => (
                <div
                  key={topic.id}
                  className="flex items-center justify-between gap-3 rounded-md border p-3"
                >
                  <div className="min-w-0">
                    <p className="truncate text-sm font-medium">
                      {topicLabel(topic)}
                    </p>
                    <p className="text-xs text-muted-foreground">
                      {topic.fetchEnabled
                        ? "Fetching enabled"
                        : "Fetching disabled"}
                      {topic.lastScannedAt
                        ? ` · last scanned ${new Date(
                            topic.lastScannedAt
                          ).toLocaleDateString()}`
                        : " · not scanned yet"}
                    </p>
                  </div>
                  <Switch
                    checked={topic.fetchEnabled}
                    disabled={pendingId === topic.id}
                    onCheckedChange={(checked) => handleToggle(topic, checked)}
                    aria-label={`Toggle fetching for ${topicLabel(topic)}`}
                  />
                </div>
              ))
            )}
          </div>
        </div>
      </DialogContent>
    </Dialog>
  );
 }
@@ -9,13 +9,14 @@ import {
  Radio,
  AlertTriangle,
  RefreshCw,
  SkipForward,
 } from "lucide-react";
 import { Card, CardContent } from "@/components/ui/card";
 import { Badge } from "@/components/ui/badge";
 import { Button } from "@/components/ui/button";
 import { cn } from "@/lib/utils";
 import { toast } from "sonner";
-import { triggerIngestion } from "../actions";
+import { triggerIngestion, disableActiveTopic } from "../actions";
 import type { IngestionAccountStatus } from "@/lib/telegram/types";
 interface WorkerStatusPanelProps {
@@ -218,6 +219,24 @@ function RunningStatus({
 }: {
  run: NonNullable<IngestionAccountStatus["currentRun"]>;
 }) {
  const [isDisabling, startDisable] = useTransition();
  const handleSkipTopic = () => {
    const acmId = run.currentAccountChannelMapId;
    const topicId = run.currentTopicId;
    if (!acmId || !topicId) return;
    startDisable(async () => {
      const result = await disableActiveTopic(acmId, topicId);
      if (result.success) {
        toast.success(
          "Topic disabled — the current file finishes, then the rest is skipped"
        );
      } else {
        toast.error(result.error ?? "Failed to disable topic");
      }
    });
  };
  return (
    <div className="space-y-2">
      <div className="flex items-center gap-2">
@@ -273,6 +292,27 @@ function RunningStatus({
          </span>
        )}
      </div>
      {/* Skip & disable the topic currently being processed */}
      {run.currentTopicId && run.currentAccountChannelMapId && (
        <div className="pl-6 pt-1">
          <Button
            variant="outline"
            size="sm"
            className="h-7 gap-1.5 text-xs"
            onClick={handleSkipTopic}
            disabled={isDisabling}
            title="Finish the current file, skip the rest of this topic, and don't fetch it in future runs"
          >
            {isDisabling ? (
              <Loader2 className="h-3 w-3 animate-spin" />
            ) : (
              <SkipForward className="h-3 w-3" />
            )}
            Skip &amp; disable this topic
          </Button>
        </div>
      )}
    </div>
  );
 }
@@ -436,6 +436,80 @@ export async function rescanChannel(channelId: string): Promise<ActionResult> {
  }
 }
 // ── Topic fetch toggle ──
 export async function setTopicFetchEnabled(
  topicProgressId: string,
  enabled: boolean
 ): Promise<ActionResult> {
  const admin = await requireAdmin();
  if (!admin.success) return admin;
  const existing = await prisma.topicProgress.findUnique({
    where: { id: topicProgressId },
  });
  if (!existing) return { success: false, error: "Topic not found" };
  try {
    await prisma.topicProgress.update({
      where: { id: topicProgressId },
      data: { fetchEnabled: enabled },
    });
    revalidatePath(REVALIDATE_PATH);
    return { success: true, data: undefined };
  } catch {
    return { success: false, error: "Failed to update topic" };
  }
 }
 /**
 * Disable the topic currently being processed by the worker, identified by its
 * account-channel mapping + Telegram topic id (as exposed on the live run
 * status). Upserts so a disabled row exists even if the worker hasn't persisted
 * the topic yet. The worker honours this live: it finishes the in-flight file
 * then skips the rest of the topic, and future runs skip it entirely.
 */
 export async function disableActiveTopic(
  accountChannelMapId: string,
  topicId: string
 ): Promise<ActionResult> {
  const admin = await requireAdmin();
  if (!admin.success) return admin;
  let topicIdBig: bigint;
  try {
    topicIdBig = BigInt(topicId);
  } catch {
    return { success: false, error: "Invalid topic id" };
  }
  // BigInt("") / BigInt("0") don't throw — reject non-positive ids explicitly
  // since this action is exported and admin-callable outside the UI button.
  if (topicIdBig <= BigInt(0)) {
    return { success: false, error: "Invalid topic id" };
  }
  try {
    await prisma.topicProgress.upsert({
      where: {
        accountChannelMapId_topicId: {
          accountChannelMapId,
          topicId: topicIdBig,
        },
      },
      create: {
        accountChannelMapId,
        topicId: topicIdBig,
        fetchEnabled: false,
      },
      update: { fetchEnabled: false },
    });
    revalidatePath(REVALIDATE_PATH);
    return { success: true, data: undefined };
  } catch {
    return { success: false, error: "Failed to disable topic" };
  }
 }
 // ── Account-Channel link actions ──
 export async function linkChannel(
@@ -3,6 +3,8 @@ import { auth } from "@/lib/auth";
 import {
  markNotificationRead,
  markAllNotificationsRead,
  dismissNotification,
  clearAllNotifications,
 } from "@/data/notification.queries";
 export const dynamic = "force-dynamic";
@@ -15,8 +17,13 @@ export async function POST(request: Request) {
  const body = await request.json().catch(() => ({}));
  const id = body.id as string | undefined;
  const action = (body.action as string) ?? "read";
-  if (id) {
+  if (action === "dismiss" && id) {
    await dismissNotification(id);
  } else if (action === "clear") {
    await clearAllNotifications();
  } else if (id) {
    await markNotificationRead(id);
  } else {
    await markAllNotificationsRead();
@@ -0,0 +1,17 @@
 import { NextResponse } from "next/server";
 import { authenticateApiRequest } from "@/lib/telegram/api-auth";
 import { listChannelTopics } from "@/lib/telegram/admin-queries";
 export const dynamic = "force-dynamic";
 export async function GET(
  request: Request,
  { params }: { params: Promise<{ channelId: string }> }
 ) {
  const authResult = await authenticateApiRequest(request, true);
  if ("error" in authResult) return authResult.error;
  const { channelId } = await params;
  const topics = await listChannelTopics(channelId);
  return NextResponse.json(topics);
 }
@@ -0,0 +1,43 @@
 import { NextResponse } from "next/server";
 import { auth } from "@/lib/auth";
 import { prisma } from "@/lib/prisma";
 export const dynamic = "force-dynamic";
 export async function GET(
  _request: Request,
  { params }: { params: Promise<{ id: string }> }
 ) {
  const session = await auth();
  if (!session?.user?.id) {
    return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
  }
  const { id } = await params;
  const upload = await prisma.manualUpload.findUnique({
    where: { id },
    include: {
      files: {
        select: { id: true, fileName: true, fileSize: true, packageId: true },
      },
    },
  });
  if (!upload || upload.userId !== session.user.id) {
    return NextResponse.json({ error: "Not found" }, { status: 404 });
  }
  return NextResponse.json({
    id: upload.id,
    status: upload.status,
    groupName: upload.groupName,
    errorMessage: upload.errorMessage,
    files: upload.files.map((f) => ({
      ...f,
      fileSize: f.fileSize.toString(),
    })),
    createdAt: upload.createdAt.toISOString(),
    completedAt: upload.completedAt?.toISOString() ?? null,
  });
 }
@@ -0,0 +1,83 @@
 import { NextResponse } from "next/server";
 import { auth } from "@/lib/auth";
 import { prisma } from "@/lib/prisma";
 import { writeFile, mkdir } from "fs/promises";
 import path from "path";
 export const dynamic = "force-dynamic";
 const UPLOAD_DIR = process.env.UPLOAD_DIR ?? "/data/uploads";
 const MAX_FILE_SIZE = 4 * 1024 * 1024 * 1024; // 4GB per file
 export async function POST(request: Request) {
  const session = await auth();
  if (!session?.user?.id) {
    return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
  }
  try {
    const formData = await request.formData();
    const files = formData.getAll("files") as File[];
    const groupName = formData.get("groupName") as string | null;
    if (!files.length) {
      return NextResponse.json({ error: "No files provided" }, { status: 400 });
    }
    // Create the upload record
    const upload = await prisma.manualUpload.create({
      data: {
        userId: session.user.id,
        groupName: groupName || (files.length > 1 ? files[0].name.replace(/\.[^.]+$/, "") : null),
        status: "PENDING",
      },
    });
    // Save files to shared volume
    const uploadDir = path.join(UPLOAD_DIR, upload.id);
    await mkdir(uploadDir, { recursive: true });
    for (const file of files) {
      if (file.size > MAX_FILE_SIZE) {
        return NextResponse.json(
          { error: `File "${file.name}" exceeds 4GB limit` },
          { status: 400 }
        );
      }
      const filePath = path.join(uploadDir, file.name);
      const buffer = Buffer.from(await file.arrayBuffer());
      await writeFile(filePath, buffer);
      await prisma.manualUploadFile.create({
        data: {
          uploadId: upload.id,
          fileName: file.name,
          filePath,
          fileSize: BigInt(file.size),
        },
      });
    }
    // Notify worker
    try {
      await prisma.$queryRawUnsafe(
        `SELECT pg_notify('manual_upload', $1)`,
        upload.id
      );
    } catch {
      // Best-effort
    }
    return NextResponse.json({
      uploadId: upload.id,
      fileCount: files.length,
      status: "PENDING",
    });
  } catch (err) {
    return NextResponse.json(
      { error: err instanceof Error ? err.message : "Upload failed" },
      { status: 500 }
    );
  }
 }
@@ -0,0 +1,9 @@
 import { signOut } from "@/lib/auth";
 // Server-side sign-out that clears the JWT session cookie and redirects to the
 // login page. Used to recover from a stale session whose user no longer exists
 // in the database (e.g. after a DB reset), which a client-only signOut can't
 // reach because the app crashes before rendering the user menu.
 export async function GET() {
  await signOut({ redirectTo: "/login" });
 }
@@ -1,7 +1,7 @@
 "use client";
 import { useState, useEffect, useCallback } from "react";
-import { Bell, AlertTriangle, AlertCircle, Info, CheckCircle2 } from "lucide-react";
+import { Bell, AlertTriangle, AlertCircle, Info, CheckCircle2, X, Trash2 } from "lucide-react";
 import { Button } from "@/components/ui/button";
 import { Badge } from "@/components/ui/badge";
 import {
@@ -94,6 +94,34 @@ export function NotificationBell() {
    }
  }
  async function handleDismiss(id: string) {
    try {
      await fetch("/api/notifications/read", {
        method: "POST",
        headers: { "Content-Type": "application/json" },
        body: JSON.stringify({ id, action: "dismiss" }),
      });
      setNotifications((prev) => prev.filter((n) => n.id !== id));
      setUnreadCount((c) => Math.max(0, c - 1));
    } catch {
      // Ignore
    }
  }
  async function handleClearAll() {
    try {
      await fetch("/api/notifications/read", {
        method: "POST",
        headers: { "Content-Type": "application/json" },
        body: JSON.stringify({ action: "clear" }),
      });
      setNotifications([]);
      setUnreadCount(0);
    } catch {
      // Ignore
    }
  }
  async function handleRepair(notificationId: string) {
    try {
      const res = await fetch("/api/notifications/repair", {
@@ -141,16 +169,29 @@ export function NotificationBell() {
      <PopoverContent className="w-96 p-0" align="end">
        <div className="flex items-center justify-between border-b px-4 py-3">
          <h3 className="text-sm font-semibold">Notifications</h3>
-          {unreadCount > 0 && (
+          <div className="flex items-center gap-1">
-            <Button
+            {unreadCount > 0 && (
-              variant="ghost"
+              <Button
-              size="sm"
+                variant="ghost"
-              className="h-7 text-xs"
+                size="sm"
-              onClick={handleMarkAllRead}
+                className="h-7 text-xs"
-            >
+                onClick={handleMarkAllRead}
-              Mark all read
+              >
-            </Button>
+                Mark all read
-          )}
+              </Button>
            )}
            {notifications.length > 0 && (
              <Button
                variant="ghost"
                size="sm"
                className="h-7 text-xs text-muted-foreground"
                onClick={handleClearAll}
              >
                <Trash2 className="h-3 w-3 mr-1" />
                Clear
              </Button>
            )}
          </div>
        </div>
        <ScrollArea className="max-h-[400px]">
          {notifications.length === 0 ? (
@@ -187,6 +228,13 @@ export function NotificationBell() {
                        {!n.isRead && (
                          <span className="h-2 w-2 rounded-full bg-primary shrink-0" />
                        )}
                        <button
                          className="ml-auto shrink-0 p-0.5 rounded hover:bg-muted text-muted-foreground hover:text-foreground"
                          onClick={(e) => { e.stopPropagation(); handleDismiss(n.id); }}
                          title="Dismiss"
                        >
                          <X className="h-3 w-3" />
                        </button>
                      </div>
                      <p className="text-xs text-muted-foreground line-clamp-2 mt-0.5">
                        {n.message}
@@ -35,3 +35,11 @@ export async function markAllNotificationsRead() {
    data: { isRead: true },
  });
 }
 export async function dismissNotification(id: string) {
  return prisma.systemNotification.delete({ where: { id } });
 }
 export async function clearAllNotifications() {
  return prisma.systemNotification.deleteMany({});
 }
@@ -1,20 +1,37 @@
 import { Prisma } from "@prisma/client";
 import { prisma } from "@/lib/prisma";
 const DEFAULT_SETTINGS = {
  lowStockThreshold: 20,
  currency: "EUR",
  theme: "dark",
  units: "metric",
 } as const;
 export async function getUserSettings(userId: string) {
  let settings = await prisma.userSettings.findUnique({
    where: { userId },
  });
  if (!settings) {
-    settings = await prisma.userSettings.create({
+    try {
-      data: {
+      settings = await prisma.userSettings.create({
-        userId,
+        data: { userId, ...DEFAULT_SETTINGS },
-        lowStockThreshold: 20,
+      });
-        currency: "EUR",
+    } catch (err) {
-        theme: "dark",
+      // The session's user may no longer exist (e.g. a stale JWT cookie after a
-        units: "metric",
+      // database reset). Creating settings then hits a foreign-key violation
-      },
+      // (P2003). Don't crash the Server Component render — return unsaved
-    });
+      // defaults. The (app) layout guard redirects such stale sessions to
      // sign-out, so this fallback is only ever momentarily visible.
      if (
        err instanceof Prisma.PrismaClientKnownRequestError &&
        err.code === "P2003"
      ) {
        return { id: "", userId, ...DEFAULT_SETTINGS };
      }
      throw err;
    }
  }
  return settings;
@@ -41,6 +41,7 @@ export async function listChannels() {
    telegramId: c.telegramId.toString(),
    title: c.title,
    type: c.type,
    isForum: c.isForum,
    isActive: c.isActive,
    category: c.category,
    createdAt: c.createdAt.toISOString(),
@@ -140,3 +141,24 @@ export async function getUnlinkedChannels(accountId: string) {
    telegramId: c.telegramId.toString(),
  }));
 }
 // ── Channel topic queries ──
 export async function listChannelTopics(channelId: string) {
  const rows = await prisma.topicProgress.findMany({
    where: { accountChannelMap: { channelId } },
    orderBy: [{ fetchEnabled: "desc" }, { topicName: "asc" }],
  });
  return rows.map((r) => ({
    id: r.id,
    topicId: r.topicId.toString(),
    topicName: r.topicName,
    fetchEnabled: r.fetchEnabled,
    lastScannedAt: r.lastScannedAt?.toISOString() ?? null,
  }));
 }
 export type ChannelTopicRow = Awaited<
  ReturnType<typeof listChannelTopics>
 >[number];
@@ -9,6 +9,28 @@ import type {
  PackageGroupRow,
 } from "./types";
 /**
 * Returns the subset of the given IDs whose `previewData` column is non-null,
 * WITHOUT transferring the (large) image bytes.
 *
 * List views only need a `hasPreview` boolean per row. Selecting `previewData`
 * directly pulls every JPEG blob (avg ~700 KB, up to 2 MB) into the Node heap
 * just to compare it against null — under concurrent requests this exhausts the
 * container memory limit and crashes the process. This keeps the check in SQL.
 */
 async function fetchPreviewFlags(
  table: "packages" | "package_groups",
  ids: string[]
 ): Promise<Set<string>> {
  if (ids.length === 0) return new Set();
  const placeholders = ids.map((_, i) => `$${i + 1}`).join(", ");
  const rows = await prisma.$queryRawUnsafe<{ id: string }[]>(
    `SELECT id FROM ${table} WHERE "previewData" IS NOT NULL AND id IN (${placeholders})`,
    ...ids
  );
  return new Set(rows.map((r) => r.id));
 }
 export async function listPackages(options: {
  page: number;
  limit: number;
@@ -40,13 +62,17 @@ export async function listPackages(options: {
        indexedAt: true,
        creator: true,
        tags: true,
        previewData: true, // check actual image data, not previewMsgId proxy
        sourceChannel: { select: { id: true, title: true } },
      },
    }),
    prisma.package.count({ where }),
  ]);
  const previewIds = await fetchPreviewFlags(
    "packages",
    items.map((p) => p.id)
  );
  const mapped: PackageListItem[] = items.map((pkg) => ({
    id: pkg.id,
    fileName: pkg.fileName,
@@ -55,7 +81,7 @@ export async function listPackages(options: {
    archiveType: pkg.archiveType,
    fileCount: pkg.fileCount,
    isMultipart: pkg.isMultipart,
-    hasPreview: pkg.previewData !== null,
+    hasPreview: previewIds.has(pkg.id),
    creator: pkg.creator,
    tags: pkg.tags,
    indexedAt: pkg.indexedAt.toISOString(),
@@ -109,31 +135,31 @@ export async function listDisplayItems(options: {
  const sortCol = sortBy === "fileName" ? `"fileName"` : sortBy === "fileSize" ? `"fileSize"` : `"indexedAt"`;
  const sortDir = order === "asc" ? "ASC" : "DESC";
-  // Step 1: Count display items
+  // NOTE: The STL list is intentionally FLAT — every package is its own display
  // row regardless of packageGroupId. Grouping is no longer surfaced in this
  // view (the creator column/filter organizes the list instead). PackageGroup
  // rows and the manual grouping actions still exist in the DB/UI; they just
  // don't drive this list's layout anymore.
  // Step 1: Count display items (one per package)
  const countResult = await prisma.$queryRawUnsafe<[{ count: bigint }]>(
-    `SELECT COUNT(*) AS count FROM (
+    `SELECT COUNT(*) AS count FROM packages p ${whereClause}`,
      SELECT DISTINCT COALESCE(p."packageGroupId", p."id") AS display_id
      FROM packages p
      ${whereClause}
    ) AS display_items`,
    ...params
  );
  const total = Number(countResult[0].count);
-  // Step 2: Get display item IDs for this page
+  // Step 2: Get package IDs for this page
  const limitParam = paramIdx++;
  const offsetParam = paramIdx++;
  const displayRows = await prisma.$queryRawUnsafe<
    { display_id: string; display_type: string }[]
  >(
    `SELECT
-      COALESCE(p."packageGroupId", p."id") AS display_id,
+      p."id" AS display_id,
-      CASE WHEN p."packageGroupId" IS NOT NULL THEN 'group' ELSE 'package' END AS display_type,
+      'package' AS display_type,
-      MAX(p.${sortCol}) AS sort_value
+      p.${sortCol} AS sort_value
    FROM packages p
    ${whereClause}
    GROUP BY COALESCE(p."packageGroupId", p."id"),
             CASE WHEN p."packageGroupId" IS NOT NULL THEN 'group' ELSE 'package' END
    ORDER BY sort_value ${sortDir}
    LIMIT $${limitParam} OFFSET $${offsetParam}`,
    ...params, limit, (page - 1) * limit
@@ -149,7 +175,7 @@ export async function listDisplayItems(options: {
        select: {
          id: true, fileName: true, fileSize: true, contentHash: true,
          archiveType: true, fileCount: true, isMultipart: true,
-          indexedAt: true, creator: true, tags: true, previewData: true,
+          indexedAt: true, creator: true, tags: true,
          sourceChannel: { select: { id: true, title: true } },
        },
      })
@@ -159,13 +185,13 @@ export async function listDisplayItems(options: {
    ? await prisma.packageGroup.findMany({
        where: { id: { in: groupIds } },
        select: {
-          id: true, name: true, previewData: true,
+          id: true, name: true,
          sourceChannel: { select: { id: true, title: true } },
          packages: {
            select: {
              id: true, fileName: true, fileSize: true, contentHash: true,
              archiveType: true, fileCount: true, isMultipart: true,
-              indexedAt: true, creator: true, tags: true, previewData: true,
+              indexedAt: true, creator: true, tags: true,
              sourceChannel: { select: { id: true, title: true } },
            },
            orderBy: { indexedAt: "desc" },
@@ -174,6 +200,16 @@ export async function listDisplayItems(options: {
      })
    : [];
  // Compute hasPreview flags without transferring image bytes (see fetchPreviewFlags)
  const allPackageIds = [
    ...standalonePackages.map((p) => p.id),
    ...groups.flatMap((g) => g.packages.map((p) => p.id)),
  ];
  const [packagePreviewIds, groupPreviewIds] = await Promise.all([
    fetchPreviewFlags("packages", allPackageIds),
    fetchPreviewFlags("package_groups", groups.map((g) => g.id)),
  ]);
  // Build DisplayItem array in the original sort order
  const packageMap = new Map(standalonePackages.map((p) => [p.id, p]));
  const groupMap = new Map(groups.map((g) => [g.id, g]));
@@ -191,7 +227,7 @@ export async function listDisplayItems(options: {
          archiveType: pkg.archiveType,
          fileCount: pkg.fileCount,
          isMultipart: pkg.isMultipart,
-          hasPreview: pkg.previewData !== null,
+          hasPreview: packagePreviewIds.has(pkg.id),
          creator: pkg.creator,
          tags: pkg.tags,
          indexedAt: pkg.indexedAt.toISOString(),
@@ -209,7 +245,7 @@ export async function listDisplayItems(options: {
        data: {
          id: grp.id,
          name: grp.name,
-          hasPreview: grp.previewData !== null,
+          hasPreview: groupPreviewIds.has(grp.id),
          totalFileSize: grp.packages.reduce((sum, p) => sum + p.fileSize, BigInt(0)).toString(),
          totalFileCount: grp.packages.reduce((sum, p) => sum + p.fileCount, 0),
          packageCount: grp.packages.length,
@@ -227,7 +263,7 @@ export async function listDisplayItems(options: {
            archiveType: pkg.archiveType,
            fileCount: pkg.fileCount,
            isMultipart: pkg.isMultipart,
-            hasPreview: pkg.previewData !== null,
+            hasPreview: packagePreviewIds.has(pkg.id),
            creator: pkg.creator,
            tags: pkg.tags,
            indexedAt: pkg.indexedAt.toISOString(),
@@ -440,13 +476,17 @@ export async function searchPackages(options: {
          indexedAt: true,
          creator: true,
          tags: true,
          previewData: true,
          sourceChannel: { select: { id: true, title: true } },
        },
      }),
      Promise.resolve(allIds.length),
    ]);
    const previewIds = await fetchPreviewFlags(
      "packages",
      items.map((p) => p.id)
    );
    const mapped: PackageListItem[] = items.map((pkg) => ({
      id: pkg.id,
      fileName: pkg.fileName,
@@ -455,7 +495,7 @@ export async function searchPackages(options: {
      archiveType: pkg.archiveType,
      fileCount: pkg.fileCount,
      isMultipart: pkg.isMultipart,
-      hasPreview: pkg.previewData !== null,
+      hasPreview: previewIds.has(pkg.id),
      creator: pkg.creator,
      tags: pkg.tags,
      indexedAt: pkg.indexedAt.toISOString(),
@@ -494,6 +534,15 @@ export async function getAllPackageTags(): Promise<string[]> {
  return result.map((r) => r.tag);
 }
 export async function getAllPackageCreators(): Promise<string[]> {
  const result = await prisma.$queryRaw<{ creator: string }[]>`
    SELECT DISTINCT creator FROM packages
    WHERE creator IS NOT NULL AND creator <> ''
    ORDER BY creator
  `;
  return result.map((r) => r.creator);
 }
 export async function getIngestionStatus(): Promise<IngestionAccountStatus[]> {
  const accounts = await prisma.telegramAccount.findMany({
    orderBy: { createdAt: "asc" },
@@ -550,6 +599,8 @@ export async function getIngestionStatus(): Promise<IngestionAccountStatus[]> {
            totalBytes: currentRun.totalBytes?.toString() ?? null,
            downloadPercent: currentRun.downloadPercent,
            lastActivityAt: currentRun.lastActivityAt?.toISOString() ?? null,
            currentTopicId: currentRun.currentTopicId?.toString() ?? null,
            currentAccountChannelMapId: currentRun.currentAccountChannelMapId,
          }
        : null,
    });
@@ -589,6 +640,7 @@ export async function listSkippedPackages(options: {
    sourceMessageId: s.sourceMessageId.toString(),
    isMultipart: s.isMultipart,
    partCount: s.partCount,
    attemptCount: s.attemptCount,
    createdAt: s.createdAt.toISOString(),
  }));
@@ -633,13 +685,17 @@ export async function listUngroupedPackages(options: {
        partCount: true,
        tags: true,
        indexedAt: true,
        previewData: true,
        sourceChannel: { select: { id: true, title: true } },
      },
    }),
    prisma.package.count({ where }),
  ]);
  const previewIds = await fetchPreviewFlags(
    "packages",
    items.map((p) => p.id)
  );
  return {
    items: items.map((p) => ({
      id: p.id,
@@ -653,7 +709,7 @@ export async function listUngroupedPackages(options: {
      partCount: p.partCount,
      tags: p.tags,
      indexedAt: p.indexedAt.toISOString(),
-      hasPreview: !!p.previewData,
+      hasPreview: previewIds.has(p.id),
      sourceChannel: p.sourceChannel,
      matchedFileCount: 0,
      matchedByContent: false,
@@ -55,6 +55,8 @@ export interface SkippedPackageItem {
  sourceMessageId: string;
  isMultipart: boolean;
  partCount: number;
  /** How many times the worker has tried this source message across cycles. */
  attemptCount: number;
  createdAt: string;
 }
@@ -121,5 +123,7 @@ export interface IngestionAccountStatus {
    totalBytes: string | null; // BigInt serialized as string
    downloadPercent: number | null;
    lastActivityAt: string | null;
    currentTopicId: string | null; // BigInt serialized as string
    currentAccountChannelMapId: string | null;
  } | null;
 }
@@ -6,15 +6,17 @@
  "scripts": {
    "build": "tsc",
    "start": "node dist/index.js",
-    "dev": "tsx watch src/index.ts"
+    "dev": "tsx watch src/index.ts",
    "test": "vitest run",
    "test:watch": "vitest"
  },
  "dependencies": {
    "@prisma/adapter-pg": "^7.4.0",
    "@prisma/client": "^7.4.0",
    "pg": "^8.18.0",
    "pino": "^9.6.0",
-    "prebuilt-tdlib": "^0.1008050.0",
+    "prebuilt-tdlib": "^0.1008064.0",
-    "tdl": "^8.0.0",
+    "tdl": "^8.1.0",
    "yauzl": "^3.2.0"
  },
  "devDependencies": {
@@ -23,6 +25,7 @@
    "@types/yauzl": "^2.10.3",
    "prisma": "^7.4.0",
    "tsx": "^4.21.0",
-    "typescript": "^5"
+    "typescript": "^5",
    "vitest": "^3.2.4"
  }
 }
@@ -0,0 +1,69 @@
 import { describe, it, expect } from "vitest";
 import { parseZipCentralDirectoryFromTail } from "./central-directory.js";
 import { crc32 } from "zlib"; // Node 20+ exposes zlib.crc32
 // Build a minimal STORE (no compression) ZIP in-memory with the given files.
 function buildStoreZip(files: { name: string; data: Buffer }[]): Buffer {
  const chunks: Buffer[] = [];
  const central: Buffer[] = [];
  let offset = 0;
  for (const f of files) {
    const crc = crc32(f.data) >>> 0;
    const nameBuf = Buffer.from(f.name, "utf8");
    const local = Buffer.alloc(30);
    local.writeUInt32LE(0x04034b50, 0);
    local.writeUInt16LE(20, 4);              // version needed
    local.writeUInt16LE(0, 6);               // flags
    local.writeUInt16LE(0, 8);               // method = store
    local.writeUInt32LE(crc, 14);
    local.writeUInt32LE(f.data.length, 18);  // compressed
    local.writeUInt32LE(f.data.length, 22);  // uncompressed
    local.writeUInt16LE(nameBuf.length, 26);
    local.writeUInt16LE(0, 28);              // extra len
    const localHeader = Buffer.concat([local, nameBuf, f.data]);
    chunks.push(localHeader);
    const cd = Buffer.alloc(46);
    cd.writeUInt32LE(0x02014b50, 0);
    cd.writeUInt16LE(20, 4); cd.writeUInt16LE(20, 6);
    cd.writeUInt16LE(0, 8); cd.writeUInt16LE(0, 10);
    cd.writeUInt32LE(crc, 16);
    cd.writeUInt32LE(f.data.length, 20);
    cd.writeUInt32LE(f.data.length, 24);
    cd.writeUInt16LE(nameBuf.length, 28);
    cd.writeUInt32LE(offset, 42);            // local header offset
    central.push(Buffer.concat([cd, nameBuf]));
    offset += localHeader.length;
  }
  const cdBuf = Buffer.concat(central);
  const cdOffset = offset;
  const eocd = Buffer.alloc(22);
  eocd.writeUInt32LE(0x06054b50, 0);
  eocd.writeUInt16LE(files.length, 8);
  eocd.writeUInt16LE(files.length, 10);
  eocd.writeUInt32LE(cdBuf.length, 12);
  eocd.writeUInt32LE(cdOffset, 16);
  return Buffer.concat([...chunks, cdBuf, eocd]);
 }
 describe("parseZipCentralDirectoryFromTail", () => {
  it("lists entries with correct names, sizes, and crc32", () => {
    const zip = buildStoreZip([
      { name: "models/dragon.stl", data: Buffer.from("DRAGON") },
      { name: "readme.txt", data: Buffer.from("hello world") },
    ]);
    const entries = parseZipCentralDirectoryFromTail(zip, 0);
    expect(entries.map((e) => e.fileName).sort()).toEqual(["dragon.stl", "readme.txt"]);
    const dragon = entries.find((e) => e.fileName === "dragon.stl")!;
    expect(dragon.path).toBe("models/dragon.stl");
    expect(dragon.uncompressedSize).toBe(6n);
    expect(dragon.crc32).toMatch(/^[0-9a-f]{8}$/);
  });
  it("throws when the central directory begins before the tail window", () => {
    const zip = buildStoreZip([{ name: "a.txt", data: Buffer.alloc(100) }]);
    // Provide only the last 30 bytes but claim they start at offset (len-30):
    const tail = zip.subarray(zip.length - 30);
    expect(() => parseZipCentralDirectoryFromTail(tail, zip.length - 30)).toThrow(RangeError);
  });
 });
@@ -0,0 +1,90 @@
 import path from "path";
 import type { FileEntry } from "./zip-reader.js";
 export const MIN_ZIP_TAIL_BYTES = 65_557;
 const EOCD_SIG = 0x06054b50;
 const CD_SIG = 0x02014b50;
 function extOf(name: string): string | null {
  const e = path.extname(name).replace(/^\./, "").toLowerCase();
  return e === "" ? null : e;
 }
 /** Parse a ZIP central directory from the tail of an archive. */
 export function parseZipCentralDirectoryFromTail(tail: Buffer, tailStart: number): FileEntry[] {
  // 1. Find EOCD by scanning backward for its signature.
  let eocd = -1;
  for (let i = tail.length - 22; i >= 0; i--) {
    if (tail.readUInt32LE(i) === EOCD_SIG) { eocd = i; break; }
  }
  if (eocd < 0) throw new RangeError("EOCD not found in tail");
  let cdSize = tail.readUInt32LE(eocd + 12);
  let cdOffset = tail.readUInt32LE(eocd + 16);
  // ZIP64: sizes/offsets of 0xFFFFFFFF mean "see ZIP64 EOCD".
  if (cdOffset === 0xffffffff || cdSize === 0xffffffff) {
    const locSig = 0x07064b50;
    let loc = -1;
    for (let i = eocd - 20; i >= 0; i--) {
      if (tail.readUInt32LE(i) === locSig) { loc = i; break; }
    }
    if (loc < 0) throw new RangeError("ZIP64 EOCD locator not in tail");
    const z64Abs = Number(tail.readBigUInt64LE(loc + 8)); // absolute offset of ZIP64 EOCD
    const z64 = z64Abs - tailStart;
    if (z64 < 0) throw new RangeError("ZIP64 EOCD before tail window");
    cdSize = Number(tail.readBigUInt64LE(z64 + 40));
    cdOffset = Number(tail.readBigUInt64LE(z64 + 48));
  }
  // 2. Map the absolute central-directory offset into the tail buffer.
  const cdLocal = cdOffset - tailStart;
  if (cdLocal < 0 || cdLocal + cdSize > tail.length) {
    throw new RangeError("Central directory begins before tail window");
  }
  // 3. Walk central-directory headers.
  const entries: FileEntry[] = [];
  let p = cdLocal;
  const end = cdLocal + cdSize;
  while (p + 46 <= end && tail.readUInt32LE(p) === CD_SIG) {
    let crc = tail.readUInt32LE(p + 16) >>> 0;
    let comp = BigInt(tail.readUInt32LE(p + 20));
    let uncomp = BigInt(tail.readUInt32LE(p + 24));
    const nameLen = tail.readUInt16LE(p + 28);
    const extraLen = tail.readUInt16LE(p + 30);
    const commentLen = tail.readUInt16LE(p + 32);
    const name = tail.toString("utf8", p + 46, p + 46 + nameLen);
    // ZIP64 extra field overrides 0xFFFFFFFF sizes.
    if (comp === 0xffffffffn || uncomp === 0xffffffffn) {
      let ep = p + 46 + nameLen;
      const extraEnd = ep + extraLen;
      while (ep + 4 <= extraEnd) {
        const id = tail.readUInt16LE(ep);
        const sz = tail.readUInt16LE(ep + 2);
        if (id === 0x0001) {
          let fp = ep + 4;
          if (uncomp === 0xffffffffn) { uncomp = tail.readBigUInt64LE(fp); fp += 8; }
          if (comp === 0xffffffffn) { comp = tail.readBigUInt64LE(fp); fp += 8; }
        }
        ep += 4 + sz;
      }
    }
    const isDir = name.endsWith("/");
    if (!isDir) {
      entries.push({
        path: name,
        fileName: path.basename(name),
        extension: extOf(name),
        compressedSize: comp,
        uncompressedSize: uncomp,
        crc32: crc !== 0 ? crc.toString(16).padStart(8, "0") : null,
      });
    }
    p += 46 + nameLen + extraLen + commentLen;
  }
  return entries;
 }
@@ -0,0 +1,35 @@
 import { describe, it, expect } from "vitest";
 import { crcFingerprint, fingerprintsMatch } from "./fingerprint.js";
 import type { FileEntry } from "./zip-reader.js";
 const fe = (crc: string | null): FileEntry => ({
  path: "a", fileName: "a", extension: null,
  compressedSize: 0n, uncompressedSize: 0n, crc32: crc,
 });
 describe("crcFingerprint", () => {
  it("sorts crcs and marks complete", () => {
    expect(crcFingerprint([fe("00ff"), fe("00aa")])).toEqual({ crcs: ["00aa", "00ff"], complete: true });
  });
  it("is incomplete when any crc is null", () => {
    expect(crcFingerprint([fe("00aa"), fe(null)]).complete).toBe(false);
  });
  it("is incomplete when empty", () => {
    expect(crcFingerprint([]).complete).toBe(false);
  });
 });
 describe("fingerprintsMatch", () => {
  it("matches identical crc multisets regardless of order", () => {
    expect(fingerprintsMatch([fe("01"), fe("02")], [fe("02"), fe("01")])).toBe(true);
  });
  it("rejects different counts", () => {
    expect(fingerprintsMatch([fe("01")], [fe("01"), fe("02")])).toBe(false);
  });
  it("rejects disjoint sets", () => {
    expect(fingerprintsMatch([fe("01")], [fe("09")])).toBe(false);
  });
  it("rejects when either side is incomplete", () => {
    expect(fingerprintsMatch([fe("01"), fe(null)], [fe("01"), fe("02")])).toBe(false);
  });
 });
@@ -0,0 +1,21 @@
 import type { FileEntry } from "./zip-reader.js";
 export function crcFingerprint(entries: FileEntry[]): { crcs: string[]; complete: boolean } {
  if (entries.length === 0) return { crcs: [], complete: false };
  const crcs: string[] = [];
  let complete = true;
  for (const e of entries) {
    if (e.crc32 == null) { complete = false; continue; }
    crcs.push(e.crc32.toLowerCase());
  }
  crcs.sort();
  return { crcs, complete };
 }
 export function fingerprintsMatch(a: FileEntry[], b: FileEntry[]): boolean {
  const fa = crcFingerprint(a);
  const fb = crcFingerprint(b);
  if (!fa.complete || !fb.complete) return false;
  if (fa.crcs.length !== fb.crcs.length) return false;
  return fa.crcs.every((c, i) => c === fb.crcs[i]);
 }
@@ -0,0 +1,128 @@
 import { execFile } from "child_process";
 import { promisify } from "util";
 import { childLogger } from "../util/logger.js";
 const execFileAsync = promisify(execFile);
 const log = childLogger("integrity");
 export type IntegrityFailureKind = "encrypted" | "corrupt" | "inconclusive";
 export type IntegrityResult =
  | { ok: true }
  | { ok: false; reason: string; kind: IntegrityFailureKind };
 /**
 * Test that the archive can be read end-to-end without errors, BEFORE we
 * spend bandwidth uploading it to the destination channel.
 *
 * Failures are classified so the caller can react appropriately:
 *   - "encrypted"    — password-protected; users can't extract it. Actionable.
 *   - "corrupt"      — genuine CRC / structural error (truncated download, bad
 *                      central directory, CRC mismatch). Actionable.
 *   - "inconclusive" — the test tool itself was killed (OOM) or timed out,
 *                      typically on very large 7z archives in a memory-limited
 *                      container (exit 137 / SIGKILL). This is a TOOL
 *                      LIMITATION, not corruption — callers should NOT raise a
 *                      user-facing alarm for it.
 *
 * Returns { ok: true } if the archive is intact, otherwise
 * { ok: false, reason, kind }.
 *
 * For multipart archives, pass the first part. unzip / unrar / 7z all
 * auto-discover sibling parts.
 *
 * archiveType "DOCUMENT" is a pass-through — there's no container to test.
 */
 export async function testArchiveIntegrity(
  archiveType: "ZIP" | "RAR" | "SEVEN_Z" | "DOCUMENT",
  firstPartPath: string
 ): Promise<IntegrityResult> {
  if (archiveType === "DOCUMENT") {
    return { ok: true };
  }
  try {
    if (archiveType === "ZIP") {
      // -t = test, -qq = very quiet (errors only)
      const { stderr } = await execFileAsync("unzip", ["-tqq", firstPartPath], {
        timeout: 300_000, // 5 min for very large archives
        maxBuffer: 10 * 1024 * 1024,
      });
      if (stderr && stderr.trim()) {
        return { ok: false, kind: "corrupt", reason: `unzip -t reported: ${stderr.slice(0, 500)}` };
      }
      return { ok: true };
    }
    if (archiveType === "RAR") {
      const { stdout, stderr } = await execFileAsync("unrar", ["t", firstPartPath], {
        timeout: 300_000,
        maxBuffer: 10 * 1024 * 1024,
      });
      // unrar uses non-zero exit code on errors, which becomes a throw.
      // If it succeeds, "All OK" is in stdout.
      const combined = `${stdout}\n${stderr}`;
      if (/All OK/i.test(combined)) {
        return { ok: true };
      }
      return { ok: false, kind: "corrupt", reason: `unrar t did not report "All OK": ${combined.slice(-500)}` };
    }
    if (archiveType === "SEVEN_Z") {
      const { stdout, stderr } = await execFileAsync("7z", ["t", firstPartPath], {
        timeout: 300_000,
        maxBuffer: 10 * 1024 * 1024,
      });
      const combined = `${stdout}\n${stderr}`;
      if (/Everything is Ok/i.test(combined)) {
        return { ok: true };
      }
      return { ok: false, kind: "corrupt", reason: `7z t did not report "Everything is Ok": ${combined.slice(-500)}` };
    }
    return { ok: false, kind: "corrupt", reason: `Unknown archive type: ${archiveType}` };
  } catch (err) {
    const msg = err instanceof Error ? err.message : String(err);
    // execFile throws on non-zero exit, on timeout, and when killed by a signal.
    const e = err as {
      stderr?: unknown;
      stdout?: unknown;
      signal?: string | null;
      killed?: boolean;
      code?: number | string | null;
    };
    const stderr = typeof e.stderr === "string" ? e.stderr : "";
    const stdout = typeof e.stdout === "string" ? e.stdout : "";
    const detail = stderr ? `: ${stderr.slice(0, 500)}` : "";
    const haystack = `${msg}\n${stdout}\n${stderr}`;
    // Encrypted archives — users can't extract them, so flag clearly.
    if (/password|encrypted|wrong password|enter password/i.test(haystack)) {
      return { ok: false, kind: "encrypted", reason: `Archive is encrypted (password protected): ${msg}${detail}` };
    }
    // Inconclusive — the test tool was killed or timed out rather than
    // reporting corruption. Common on large 7z in memory-limited containers,
    // where `7z t` gets OOM-killed (SIGKILL / exit 137) mid-decompression.
    // That's a tool limitation, not a corrupt archive.
    const killedBySignal = e.signal === "SIGKILL" || e.signal === "SIGTERM";
    const killedExitCode = e.code === 137 || e.code === 143; // 128 + SIGKILL/SIGTERM
    const timedOut = e.killed === true;
    const maxBufferExceeded = e.code === "ERR_CHILD_PROCESS_STDIO_MAXBUFFER";
    if (killedBySignal || killedExitCode || timedOut || maxBufferExceeded) {
      log.debug(
        { err, archiveType, firstPartPath, signal: e.signal, code: e.code, killed: e.killed },
        "Archive integrity test inconclusive (tool killed or timed out)"
      );
      return {
        ok: false,
        kind: "inconclusive",
        reason: `Integrity test could not complete (tool killed or timed out — likely OOM on a large archive): ${msg}`,
      };
    }
    // Genuine failure: a real non-zero exit with error output.
    log.debug({ err, archiveType, firstPartPath }, "Archive integrity test failed");
    return { ok: false, kind: "corrupt", reason: `Integrity test failed: ${msg}${detail}` };
  }
 }
@@ -11,8 +11,11 @@ export interface TelegramMessage {
  fileSize: bigint;
  date: Date;
  mediaAlbumId?: string;
-  replyToMessageId?: bigint;  // NEW
+  replyToMessageId?: bigint;
-  caption?: string;            // NEW
+  caption?: string;
  /** TDLib's `remote.unique_id` for the file — stable across reposts of
   *  the exact same content. Empty string if the message didn't expose it. */
  remoteUniqueId?: string;
 }
 export interface ArchiveSet {
@@ -0,0 +1,27 @@
 import { describe, it, expect, vi } from "vitest";
 // logLevel is required here too (not just maxZipSizeMB/tempDir) because this
 // mock replaces the config module for the whole test-file graph, including
 // util/logger.ts's module-level `pino({ level: config.logLevel })` call —
 // pino throws at import time if level is undefined.
 vi.mock("../../util/config.js", () => ({ config: { maxZipSizeMB: 1, tempDir: "/tmp", logLevel: "info" } }));
 const created: unknown[] = [];
 vi.mock("../../db/client.js", () => ({
  db: { systemNotification: { create: async (a: unknown) => { created.push(a); } } },
 }));
 import { fullDownloadListing } from "./fallback.js";
 describe("fullDownloadListing", () => {
  it("refuses to download over the size cap and records a notification", async () => {
    const res = await fullDownloadListing({
      client: {} as never,
      parts: [{ fileId: "1", fileSize: 2n * 1024n * 1024n * 1024n, fileName: "big.rar" }],
      archiveType: "RAR",
      totalSize: 2n * 1024n * 1024n * 1024n,
      fileName: "big.rar",
    });
    expect(res).toBeNull();
    expect(created).toHaveLength(1);
  });
 });
@@ -0,0 +1,55 @@
 import { mkdtemp, rm } from "fs/promises";
 import path from "path";
 import type { Client } from "tdl";
 import { config } from "../../util/config.js";
 import { db } from "../../db/client.js";
 import { childLogger } from "../../util/logger.js";
 import { downloadFile } from "../../tdlib/download.js";
 import { read7zContents } from "../sevenz-reader.js";
 import { readRarContents } from "../rar-reader.js";
 import type { FileEntry } from "../zip-reader.js";
 import type { RangedPart } from "./sevenz-ranged.js";
 const log = childLogger("ranged-fallback");
 export async function fullDownloadListing(args: {
  client: Client;
  parts: RangedPart[];
  archiveType: string;
  totalSize: bigint;
  fileName: string;
 }): Promise<FileEntry[] | null> {
  const capBytes = BigInt(config.maxZipSizeMB) * 1024n * 1024n;
  if (args.totalSize > capBytes) {
    await db.systemNotification.create({
      data: {
        type: "INTEGRITY_AUDIT",
        severity: "WARNING",
        title: `Listing skipped (over size cap): ${args.fileName}`,
        message: `Ranged listing failed and the archive (${args.totalSize} bytes) exceeds WORKER_MAX_ZIP_SIZE_MB; not downloaded. Inner files left unindexed.`,
        context: { fileName: args.fileName, archiveType: args.archiveType },
      },
    });
    log.warn({ fileName: args.fileName }, "fallback skipped — over size cap");
    return null;
  }
  const dir = await mkdtemp(path.join(config.tempDir, "fallback-"));
  const paths: string[] = [];
  try {
    for (const p of args.parts) {
      const dest = path.join(dir, p.fileName);
      await downloadFile(args.client, p.fileId, dest, p.fileSize, p.fileName, () => {});
      paths.push(dest);
    }
    const entries =
      args.archiveType === "SEVEN_Z" ? await read7zContents(paths[0])
      : args.archiveType === "RAR" ? await readRarContents(paths[0])
      : [];
    return entries.length > 0 ? entries : null;
  } catch (err) {
    log.warn({ err, fileName: args.fileName }, "full-download fallback failed");
    return null;
  } finally {
    await rm(dir, { recursive: true, force: true });
  }
 }
@@ -0,0 +1,14 @@
 import type { Client } from "tdl";
 import { downloadFileRange } from "../../tdlib/range-download.js";
 export type RangeReader = (
  fileId: string,
  offset: number,
  length: number,
  partSize: bigint,
 ) => Promise<Buffer>;
 export function tdlibRangeReader(client: Client): RangeReader {
  return (fileId, offset, length, partSize) =>
    downloadFileRange(client, fileId, offset, length, partSize);
 }
@@ -0,0 +1,150 @@
 import { describe, it, expect } from "vitest";
 import { readVint, detectRarSignature, parseRar5BlockExtent, parseRar4BlockExtent, walkRarVolume, readRarListingRanged } from "./rar-ranged.js";
 import type { RangeReader } from "./range-reader.js";
 describe("readVint", () => {
  it("reads single-byte and multi-byte values (base-128 LE)", () => {
    expect(readVint(Buffer.from([0x08]), 0)).toEqual({ value: 8, bytes: 1 });
    // 0x80,0x01 => 0 | (1<<7) = 128
    expect(readVint(Buffer.from([0x80, 0x01]), 0)).toEqual({ value: 128, bytes: 2 });
  });
 });
 describe("detectRarSignature", () => {
  it("detects RAR5 and RAR4", () => {
    expect(detectRarSignature(Buffer.from([0x52,0x61,0x72,0x21,0x1a,0x07,0x01,0x00]))).toEqual({ version: 5, sigLen: 8 });
    expect(detectRarSignature(Buffer.from([0x52,0x61,0x72,0x21,0x1a,0x07,0x00]))).toEqual({ version: 4, sigLen: 7 });
    expect(detectRarSignature(Buffer.alloc(8))).toBeNull();
  });
 });
 describe("parseRar5BlockExtent", () => {
  it("computes header+data extent and flags end-of-archive", () => {
    // CRC32(4) | HeaderSize vint=5 | Type vint=2 (file) | Flags vint=2 (data present) | DataSize vint=100 | (pad to headerSize)
    const b = Buffer.concat([
      Buffer.from([0,0,0,0]),      // CRC
      Buffer.from([0x05]),          // HeaderSize = 5 (bytes after this vint)
      Buffer.from([0x02]),          // Type = 2 (file)
      Buffer.from([0x02]),          // Flags = 0x02 -> data present
      Buffer.from([0x64]),          // DataSize = 100
      Buffer.from([0x00, 0x00]),    // padding to fill HeaderSize(5): Type+Flags+DataSize=3, +2 pad =5
    ]);
    const ext = parseRar5BlockExtent(b, 0);
    // headerBytes = 4 (CRC) + 1 (HeaderSize vint) + 5 (HeaderSize) = 10
    expect(ext.headerBytes).toBe(10);
    expect(ext.dataSize).toBe(100);
    expect(ext.isEnd).toBe(false);
    const endBlk = Buffer.from([0,0,0,0, 0x02, 0x05, 0x00]); // HeaderSize=2, Type=5(end), Flags=0
    const e2 = parseRar5BlockExtent(endBlk, 0);
    expect(e2.isEnd).toBe(true);
  });
 });
 describe("parseRar4BlockExtent", () => {
  it("computes extent with ADD_SIZE when flag 0x8000 is set", () => {
    // CRC(2) TYPE(1)=0x74 FLAGS(2)=0x8000 HEAD_SIZE(2)=11 ADD_SIZE(4)=200
    const b = Buffer.alloc(11);
    b.writeUInt8(0x74, 2);
    b.writeUInt16LE(0x8000, 3);
    b.writeUInt16LE(11, 5);
    b.writeUInt32LE(200, 7);
    const ext = parseRar4BlockExtent(b, 0);
    expect(ext.headerBytes).toBe(11);
    expect(ext.dataSize).toBe(200);
    expect(ext.isEnd).toBe(false);
  });
 });
 // Build a synthetic RAR5 volume: signature + main header + 2 file blocks (each
 // with data) + end block. We only need extents to be walkable.
 function buildRar5Volume(): Buffer {
  const sig = Buffer.from([0x52,0x61,0x72,0x21,0x1a,0x07,0x01,0x00]);
  const block = (type: number, flags: number, dataSize: number, pad = 0) => {
    const body = [Buffer.from([type]), Buffer.from([flags])];
    if (flags & 0x0002) body.push(Buffer.from([dataSize])); // DataSize (<=127 for test)
    if (pad) body.push(Buffer.alloc(pad));
    const bodyBuf = Buffer.concat(body);
    const hs = Buffer.from([bodyBuf.length]); // HeaderSize vint (<=127)
    const header = Buffer.concat([Buffer.alloc(4), hs, bodyBuf]); // CRC(4)+HeaderSize+body
    const data = Buffer.alloc(flags & 0x0002 ? dataSize : 0, 0xEE);
    return Buffer.concat([header, data]);
  };
  const main = block(1, 0, 0);       // main archive header, no data
  const f1 = block(2, 0x02, 20);     // file header + 20 bytes data
  const f2 = block(2, 0x02, 30);     // file header + 30 bytes data
  const end = block(5, 0, 0);        // end of archive
  return Buffer.concat([sig, main, f1, f2, end]);
 }
 describe("walkRarVolume", () => {
  it("harvests every block header and stops at end-of-archive", async () => {
    const vol = buildRar5Volume();
    const read: RangeReader = async (_id, offset, length) => vol.subarray(offset, offset + length);
    const regions = await walkRarVolume(read, { fileId: "1", fileSize: BigInt(vol.length), fileName: "a.rar" }, 5, 8);
    expect(regions).not.toBeNull();
    // main + 2 files + end = 4 header regions
    expect(regions!).toHaveLength(4);
    // First region starts right after the 8-byte signature
    expect(regions![0].offset).toBe(8);
  });
  it("returns null when a block claims an absurd header size (corrupt/desynced)", async () => {
    // RAR5 block with HeaderSize vint encoding a value > 8MB.
    // Encode 9_000_000 as RAR vint: bytes little-endian 7-bit groups with continuation bit.
    function encodeVint(n: number): number[] {
      const out: number[] = [];
      while (n >= 0x80) {
        out.push((n & 0x7f) | 0x80);
        n = Math.floor(n / 128);
      }
      out.push(n);
      return out;
    }
    const sig = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x01, 0x00]); // RAR5 signature
    const hsVint = encodeVint(9_000_000);
    // Block = CRC(4) + HeaderSize vint(9MB) + Type(1 byte) + Flags(1 byte)
    const block = Buffer.concat([Buffer.alloc(4), Buffer.from(hsVint), Buffer.from([0x02, 0x00])]);
    const vol = Buffer.concat([sig, block]);
    const size = 20 * 1024 * 1024;
    const read = async (_id: string, offset: number, length: number) => {
      if (offset >= vol.length) return Buffer.alloc(0);
      return vol.subarray(offset, Math.min(offset + length, vol.length));
    };
    const regions = await walkRarVolume(read, { fileId: "1", fileSize: BigInt(size), fileName: "c.rar" }, 5, 8);
    expect(regions).toBeNull();
  });
 });
 describe("readRarListingRanged (single part)", () => {
  it("returns null cleanly when the reconstructed file isn't a real RAR", async () => {
    const vol = buildRar5Volume();
    const read: RangeReader = async (_id, offset, length) => vol.subarray(offset, offset + length);
    const res = await readRarListingRanged([{ fileId: "1", fileSize: BigInt(vol.length), fileName: "a.rar" }], read);
    expect(res === null || Array.isArray(res)).toBe(true); // real unrar parse covered live
  });
 });
 describe("readRarListingRanged (multipart)", () => {
  it("walks each volume from its own signature and reconstructs all parts", async () => {
    const vol = buildRar5Volume(); // reuse from Task 6 test
    // Two volumes with identical structure; each RangeReader read is scoped by fileId.
    const byId: Record<string, Buffer> = { p1: vol, p2: vol };
    const reads: Record<string, number> = { p1: 0, p2: 0 };
    const read: RangeReader = async (fileId, offset, length) => {
      reads[fileId]++;
      return byId[fileId].subarray(offset, offset + length);
    };
    const res = await readRarListingRanged(
      [
        { fileId: "p1", fileSize: BigInt(vol.length), fileName: "x.part1.rar" },
        { fileId: "p2", fileSize: BigInt(vol.length), fileName: "x.part2.rar" },
      ],
      read,
    );
    // Both volumes were walked (each read at least its signature + blocks).
    expect(reads.p1).toBeGreaterThan(0);
    expect(reads.p2).toBeGreaterThan(0);
    expect(res === null || Array.isArray(res)).toBe(true);
  });
 });
@@ -0,0 +1,110 @@
 const RAR4_SIG = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x00]);
 const RAR5_SIG = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x01, 0x00]);
 export function readVint(buf: Buffer, pos: number): { value: number; bytes: number } {
  let value = 0, shift = 0, bytes = 0;
  while (pos + bytes < buf.length) {
    const b = buf[pos + bytes];
    value += (b & 0x7f) * Math.pow(2, shift); // Math.pow keeps >32-bit sizes exact up to 2^53
    bytes++;
    if ((b & 0x80) === 0) return { value, bytes };
    shift += 7;
    if (shift > 63) break;
  }
  throw new RangeError("incomplete RAR vint");
 }
 export function detectRarSignature(buf: Buffer): { version: 4 | 5; sigLen: number } | null {
  if (buf.length >= 8 && buf.subarray(0, 8).equals(RAR5_SIG)) return { version: 5, sigLen: 8 };
  if (buf.length >= 7 && buf.subarray(0, 7).equals(RAR4_SIG)) return { version: 4, sigLen: 7 };
  return null;
 }
 export interface BlockExtent { headerBytes: number; dataSize: number; isEnd: boolean }
 // RAR5: CRC32(4) | HeaderSize(vint) | HeaderType(vint) | HeaderFlags(vint)
 //       [ExtraAreaSize(vint) if flags&0x0001] [DataSize(vint) if flags&0x0002] ...
 export function parseRar5BlockExtent(buf: Buffer, pos: number): BlockExtent {
  let p = pos + 4; // skip CRC32
  const hs = readVint(buf, p); p += hs.bytes;
  const headerBytes = 4 + hs.bytes + hs.value; // CRC + HeaderSize-vint + HeaderSize
  const type = readVint(buf, p); p += type.bytes;
  const flags = readVint(buf, p); p += flags.bytes;
  if (flags.value & 0x0001) { const ea = readVint(buf, p); p += ea.bytes; } // extra area size (skip)
  let dataSize = 0;
  if (flags.value & 0x0002) { const ds = readVint(buf, p); p += ds.bytes; dataSize = ds.value; }
  return { headerBytes, dataSize, isEnd: type.value === 5 };
 }
 // RAR4: HEAD_CRC(2) | HEAD_TYPE(1) | HEAD_FLAGS(2) | HEAD_SIZE(2) [ADD_SIZE(4) if flags&0x8000]
 export function parseRar4BlockExtent(buf: Buffer, pos: number): BlockExtent {
  const type = buf.readUInt8(pos + 2);
  const flags = buf.readUInt16LE(pos + 3);
  const headSize = buf.readUInt16LE(pos + 5);
  const dataSize = (flags & 0x8000) ? buf.readUInt32LE(pos + 7) : 0;
  return { headerBytes: headSize, dataSize, isEnd: type === 0x7b };
 }
 import type { FileEntry } from "../zip-reader.js";
 import type { RangeReader } from "./range-reader.js";
 import type { RangedPart } from "./sevenz-ranged.js";
 import { listFromSparse, type SparsePart } from "./sparse-list.js";
 import { readRarContents } from "../rar-reader.js";
 import { childLogger } from "../../util/logger.js";
 const rlog = childLogger("rar-ranged");
 const MAX_RAR_BLOCKS = 50000;
 const MAX_RAR_HEADER_BYTES = 8 * 1024 * 1024; // 8 MB — real RAR block headers are far smaller; guards against a corrupt/desynced HeaderSize
 const HEADER_CHUNK = 8192;
 export async function walkRarVolume(
  read: RangeReader,
  part: RangedPart,
  version: 4 | 5,
  sigLen: number,
 ): Promise<{ offset: number; bytes: Buffer }[] | null> {
  const size = Number(part.fileSize);
  const regions: { offset: number; bytes: Buffer }[] = [];
  let pos = sigLen;
  let blocks = 0;
  try {
    while (pos < size) {
      if (++blocks > MAX_RAR_BLOCKS) return null;
      const chunkLen = Math.min(HEADER_CHUNK, size - pos);
      let chunk = await read(part.fileId, pos, chunkLen, part.fileSize);
      const ext = version === 5 ? parseRar5BlockExtent(chunk, 0) : parseRar4BlockExtent(chunk, 0);
      if (ext.headerBytes > MAX_RAR_HEADER_BYTES) return null;
      // Ensure we have the full header bytes to harvest (long filenames).
      let headerBuf = chunk;
      if (ext.headerBytes > chunk.length) {
        headerBuf = await read(part.fileId, pos, Math.min(ext.headerBytes, size - pos), part.fileSize);
      }
      regions.push({ offset: pos, bytes: headerBuf.subarray(0, Math.min(ext.headerBytes, size - pos)) });
      if (ext.isEnd) break;
      const advance = ext.headerBytes + ext.dataSize;
      if (advance <= 0) return null;
      if (pos + advance > size) break; // data clamped at the volume boundary (multipart continuation)
      pos += advance;
    }
    return regions;
  } catch (err) {
    rlog.warn({ err, fileId: part.fileId }, "RAR volume walk failed");
    return null;
  }
 }
 export async function readRarListingRanged(
  parts: RangedPart[],
  read: RangeReader,
 ): Promise<FileEntry[] | null> {
  const sparseParts: SparsePart[] = [];
  for (const part of parts) {
    const head = await read(part.fileId, 0, 16, part.fileSize);
    const sig = detectRarSignature(head);
    if (!sig) return null;
    const regions = await walkRarVolume(read, part, sig.version, sig.sigLen);
    if (!regions) return null;
    sparseParts.push({ fileName: part.fileName, size: Number(part.fileSize), regions });
  }
  return listFromSparse(sparseParts, readRarContents);
 }
@@ -0,0 +1,133 @@
 import { describe, it, expect } from "vitest";
 import { parseSevenZSignatureHeader } from "./sevenz-ranged.js";
 const MAGIC = Buffer.from([0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c]);
 function buildSignatureHeader(nextOffset: bigint, nextSize: bigint): Buffer {
  const buf = Buffer.alloc(32);
  MAGIC.copy(buf, 0);
  buf.writeUInt8(0, 6); buf.writeUInt8(4, 7);  // version 0.4
  buf.writeUInt32LE(0, 8);                      // StartHeaderCRC (unused here)
  buf.writeBigUInt64LE(nextOffset, 12);
  buf.writeBigUInt64LE(nextSize, 20);
  buf.writeUInt32LE(0, 28);                     // NextHeaderCRC (unused here)
  return buf;
 }
 describe("parseSevenZSignatureHeader", () => {
  it("reads NextHeaderOffset and NextHeaderSize", () => {
    const buf = buildSignatureHeader(1_000_000n, 4096n);
    expect(parseSevenZSignatureHeader(buf)).toEqual({ nextHeaderOffset: 1_000_000, nextHeaderSize: 4096 });
  });
  it("returns null on bad magic", () => {
    expect(parseSevenZSignatureHeader(Buffer.alloc(32))).toBeNull();
  });
  it("returns null when shorter than 32 bytes", () => {
    expect(parseSevenZSignatureHeader(MAGIC)).toBeNull();
  });
 });
 import { readSevenZListingRanged } from "./sevenz-ranged.js";
 import type { RangeReader } from "./range-reader.js";
 describe("readSevenZListingRanged", () => {
  it("reads the signature + end-header regions and reconstructs for 7z l", async () => {
    const size = 5_000_000;
    const endHeaderOffset = 4_900_000; // absolute
    const nextHeaderOffset = endHeaderOffset - 32;
    const sig = Buffer.alloc(32);
    Buffer.from([0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c]).copy(sig, 0);
    sig.writeBigUInt64LE(BigInt(nextHeaderOffset), 12);
    sig.writeBigUInt64LE(100n, 20);
    const reads: { offset: number; length: number }[] = [];
    const read: RangeReader = async (_id, offset, length) => {
      reads.push({ offset, length });
      if (offset === 0) return sig.subarray(0, length);
      return Buffer.alloc(length, 0xAB); // stand-in end-header bytes
    };
    // Inject a fake lister via the module boundary: readSevenZListingRanged
    // calls listFromSparse(parts, read7zContents). We assert the ranged reads
    // it issued; the sparse file + real 7z is covered by live verification.
    const entries = await readSevenZListingRanged(
      [{ fileId: "1", fileSize: BigInt(size), fileName: "a.7z" }],
      read,
    );
    // entries may be null here because the stand-in bytes aren't a real 7z;
    // the contract under test is the ranged-read offsets:
    expect(reads[0]).toEqual({ offset: 0, length: 32 });
    expect(reads[1]).toEqual({ offset: endHeaderOffset, length: 100 });
    expect(entries === null || Array.isArray(entries)).toBe(true);
  });
 });
 import { read7zNumber, locate7zEncodedHeaderPack } from "./sevenz-ranged.js";
 describe("read7zNumber", () => {
  it("reads a single-byte number", () => {
    expect(read7zNumber(Buffer.from([0x2a]), 0)).toEqual({ value: 42, next: 1 });
  });
  it("reads a two-byte number (first-byte length mask + LE trailing byte)", () => {
    // 1000 = 0x03E8: low byte 0xE8, high nibble 0x03 -> first = 0x80|0x03 = 0x83, trailing 0xE8
    expect(read7zNumber(Buffer.from([0x83, 0xe8]), 0)).toEqual({ value: 1000, next: 2 });
    // 500 = 0x01F4 -> first 0x81, trailing 0xF4
    expect(read7zNumber(Buffer.from([0x81, 0xf4]), 0)).toEqual({ value: 500, next: 2 });
  });
  it("throws when pos starts past the buffer end (short read)", () => {
    expect(() => read7zNumber(Buffer.from([0x2a]), 5)).toThrow(RangeError);
    expect(() => read7zNumber(Buffer.alloc(0), 0)).toThrow(RangeError);
  });
 });
 describe("locate7zEncodedHeaderPack", () => {
  it("parses PackPos and summed PackSize from an encoded header", () => {
    // kEncodedHeader, kPackInfo, PackPos=1000([0x83,0xe8]), NumStreams=1([0x01]),
    // kSize, PackSize=500([0x81,0xf4])
    const enc = Buffer.from([0x17, 0x06, 0x83, 0xe8, 0x01, 0x09, 0x81, 0xf4]);
    expect(locate7zEncodedHeaderPack(enc)).toEqual({ packPos: 1000, packSize: 500 });
  });
  it("returns null for a plain (kHeader 0x01) header", () => {
    expect(locate7zEncodedHeaderPack(Buffer.from([0x01, 0x04]))).toBeNull();
  });
  it("sums multiple pack streams", () => {
    // PackPos=0([0x00]), NumStreams=2([0x02]), kSize, sizes 10([0x0a]) + 20([0x14])
    const enc = Buffer.from([0x17, 0x06, 0x00, 0x02, 0x09, 0x0a, 0x14]);
    expect(locate7zEncodedHeaderPack(enc)).toEqual({ packPos: 0, packSize: 30 });
  });
 });
 describe("readSevenZListingRanged (encoded header)", () => {
  it("fetches the mid-file packed-header region as a third read", async () => {
    const size = 5_000_000;
    const nextHeaderOffset = 4_000_000;      // relative to end of 32-byte sig header
    const endStart = 32 + nextHeaderOffset;  // absolute
    const packPos = 1000;                    // relative to end of sig header
    const packStart = 32 + packPos;          // absolute
    const packSize = 500;
    const sig = Buffer.alloc(32);
    Buffer.from([0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c]).copy(sig, 0);
    sig.writeBigUInt64LE(BigInt(nextHeaderOffset), 12);
    sig.writeBigUInt64LE(8n, 20); // NextHeaderSize = 8 (the encoded-header descriptor below)
    const encHeader = Buffer.from([0x17, 0x06, 0x83, 0xe8, 0x01, 0x09, 0x81, 0xf4]);
    const reads: { offset: number; length: number }[] = [];
    const read = async (_id: string, offset: number, length: number) => {
      reads.push({ offset, length });
      if (offset === 0) return sig.subarray(0, length);
      if (offset === endStart) return encHeader.subarray(0, length);
      return Buffer.alloc(length, 0xcd); // stand-in packed-header bytes
    };
    const entries = await readSevenZListingRanged(
      [{ fileId: "1", fileSize: BigInt(size), fileName: "a.7z" }],
      read,
    );
    expect(reads[0]).toEqual({ offset: 0, length: 32 });
    expect(reads[1]).toEqual({ offset: endStart, length: 8 });
    expect(reads[2]).toEqual({ offset: packStart, length: packSize });
    expect(entries === null || Array.isArray(entries)).toBe(true);
  });
 });
@@ -0,0 +1,112 @@
 import type { FileEntry } from "../zip-reader.js";
 import { read7zContents } from "../sevenz-reader.js";
 import { listFromSparse } from "./sparse-list.js";
 import type { RangeReader } from "./range-reader.js";
 import { childLogger } from "../../util/logger.js";
 const log = childLogger("sevenz-ranged");
 const SEVENZ_MAGIC = Buffer.from([0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c]);
 const K_HEADER = 0x01;
 const K_ENCODED_HEADER = 0x17;
 const K_PACK_INFO = 0x06;
 const K_SIZE = 0x09;
 /** Read a 7z variable-length number: first byte is a length mask, followed by
 *  little-endian bytes. Math.pow keeps values exact above 2^31. */
 export function read7zNumber(buf: Buffer, pos: number): { value: number; next: number } {
  if (pos >= buf.length) throw new RangeError("7z number reads past buffer end");
  const first = buf[pos];
  let mask = 0x80;
  let value = 0;
  let p = pos + 1;
  for (let i = 0; i < 8; i++) {
    if ((first & mask) === 0) {
      value += (first & (mask - 1)) * Math.pow(2, 8 * i);
      return { value, next: p };
    }
    if (p >= buf.length) throw new RangeError("7z number overruns buffer");
    value += buf[p] * Math.pow(2, 8 * i);
    p++;
    mask >>= 1;
  }
  return { value, next: p };
 }
 /** For an encoded (kEncodedHeader) 7z next-header, return the absolute-ish
 *  location of the packed header stream(s): PackPos (relative to end of the
 *  32-byte signature header) and the summed PackSize. Null if not encoded or
 *  the StreamsInfo doesn't start with PackInfo as expected. */
 export function locate7zEncodedHeaderPack(
  nextHeader: Buffer,
 ): { packPos: number; packSize: number } | null {
  let p = 0;
  if (nextHeader[p] !== K_ENCODED_HEADER) return null;
  p++;
  if (nextHeader[p] !== K_PACK_INFO) return null;
  p++;
  const packPos = read7zNumber(nextHeader, p); p = packPos.next;
  const numStreams = read7zNumber(nextHeader, p); p = numStreams.next;
  if (nextHeader[p] !== K_SIZE) return null;
  p++;
  let total = 0;
  for (let i = 0; i < numStreams.value; i++) {
    const s = read7zNumber(nextHeader, p); p = s.next;
    total += s.value;
  }
  return { packPos: packPos.value, packSize: total };
 }
 export function parseSevenZSignatureHeader(
  buf: Buffer,
 ): { nextHeaderOffset: number; nextHeaderSize: number } | null {
  if (buf.length < 32) return null;
  if (!buf.subarray(0, 6).equals(SEVENZ_MAGIC)) return null;
  return {
    nextHeaderOffset: Number(buf.readBigUInt64LE(12)),
    nextHeaderSize: Number(buf.readBigUInt64LE(20)),
  };
 }
 export interface RangedPart { fileId: string; fileSize: bigint; fileName: string }
 export async function readSevenZListingRanged(
  parts: RangedPart[],
  read: RangeReader,
 ): Promise<FileEntry[] | null> {
  const part = parts[0];
  if (!part) return null;
  const size = Number(part.fileSize);
  try {
    const sig = await read(part.fileId, 0, 32, part.fileSize);
    const parsed = parseSevenZSignatureHeader(sig);
    if (!parsed) return null;
    const endStart = 32 + parsed.nextHeaderOffset;
    if (endStart < 0 || endStart + parsed.nextHeaderSize > size) return null;
    const endHeader = await read(part.fileId, endStart, parsed.nextHeaderSize, part.fileSize);
    const regions = [
      { offset: 0, bytes: sig },
      { offset: endStart, bytes: endHeader },
    ];
    const headerType = endHeader[0];
    if (headerType === K_ENCODED_HEADER) {
      // Compressed header: its packed bytes live mid-file, not at EOF. Fetch them.
      const pack = locate7zEncodedHeaderPack(endHeader);
      if (!pack) return null;
      const packStart = 32 + pack.packPos;
      if (packStart < 0 || packStart + pack.packSize > size) return null;
      const packBytes = await read(part.fileId, packStart, pack.packSize, part.fileSize);
      regions.push({ offset: packStart, bytes: packBytes });
    } else if (headerType !== K_HEADER) {
      return null; // unknown next-header type
    }
    return listFromSparse([{ fileName: part.fileName, size, regions }], read7zContents);
  } catch (err) {
    log.warn({ err, fileId: part.fileId }, "ranged 7z listing failed");
    return null;
  }
 }
@@ -0,0 +1,41 @@
 import { describe, it, expect } from "vitest";
 import { open } from "fs/promises";
 import { listFromSparse } from "./sparse-list.js";
 describe("listFromSparse", () => {
  it("writes each region at its offset into a sparse file and passes the path to the lister", async () => {
    const size = 1_000_000;
    const regions = [
      { offset: 0, bytes: Buffer.from("HEAD") },
      { offset: size - 4, bytes: Buffer.from("TAIL") },
    ];
    let seenPath = "";
    const entries = await listFromSparse(
      [{ fileName: "sample.7z", size, regions }],
      async (firstPartPath) => {
        seenPath = firstPartPath;
        const fh = await open(firstPartPath, "r");
        try {
          const head = Buffer.alloc(4); await fh.read(head, 0, 4, 0);
          const tail = Buffer.alloc(4); await fh.read(tail, 0, 4, size - 4);
          const hole = Buffer.alloc(4); await fh.read(hole, 0, 4, 500_000);
          expect(head.toString()).toBe("HEAD");
          expect(tail.toString()).toBe("TAIL");
          expect(hole.equals(Buffer.alloc(4))).toBe(true); // gap is zero
        } finally { await fh.close(); }
        return [{ path: "a/b.stl", fileName: "b.stl", extension: "stl", compressedSize: 1n, uncompressedSize: 1n, crc32: null }];
      },
    );
    expect(seenPath.endsWith("sample.7z")).toBe(true);
    expect(entries).not.toBeNull();
    expect(entries!).toHaveLength(1);
  });
  it("returns null when the lister yields no entries", async () => {
    const res = await listFromSparse(
      [{ fileName: "x.7z", size: 100, regions: [{ offset: 0, bytes: Buffer.from("A") }] }],
      async () => [],
    );
    expect(res).toBeNull();
  });
 });
@@ -0,0 +1,52 @@
 import { mkdtemp, open, rm } from "fs/promises";
 import path from "path";
 import { config } from "../../util/config.js";
 import { childLogger } from "../../util/logger.js";
 import type { FileEntry } from "../zip-reader.js";
 const log = childLogger("sparse-list");
 export interface SparsePart {
  fileName: string;
  size: number;
  regions: { offset: number; bytes: Buffer }[];
 }
 export type SparseLister = (firstPartPath: string) => Promise<FileEntry[]>;
 /**
 * Reconstruct archive header bytes into sparse temp files (data areas left as
 * zero holes), run `lister` on the first part, return its entries.
 * Returns null on any error or when the lister finds nothing.
 */
 export async function listFromSparse(
  parts: SparsePart[],
  lister: SparseLister,
 ): Promise<FileEntry[] | null> {
  if (parts.length === 0) return null;
  const dir = await mkdtemp(path.join(config.tempDir, "ranged-"));
  try {
    let firstPath = "";
    for (let i = 0; i < parts.length; i++) {
      const p = parts[i];
      const filePath = path.join(dir, p.fileName);
      if (i === 0) firstPath = filePath;
      const fh = await open(filePath, "w");
      try {
        await fh.truncate(p.size); // create the sparse hole
        for (const r of p.regions) {
          await fh.write(r.bytes, 0, r.bytes.length, r.offset);
        }
      } finally {
        await fh.close();
      }
    }
    const entries = await lister(firstPath);
    return entries.length > 0 ? entries : null;
  } catch (err) {
    log.warn({ err }, "sparse listing failed");
    return null;
  } finally {
    await rm(dir, { recursive: true, force: true });
  }
 }
@@ -8,83 +8,122 @@ const execFileAsync = promisify(execFile);
 const log = childLogger("rar-reader");
 /**
- * Parse output of `unrar l -v <file>` to extract file metadata.
+ * Parse output of `unrar lt <file>` to extract file metadata.
- * unrar automatically discovers sibling parts when they're co-located.
+ *
 * `lt` (list technical) emits one block per archived file with key:value
 * lines — far more reliable than the column-based default `l -v` output,
 * which has changed format twice across unrar versions.
 *
 * unrar automatically discovers sibling multipart files when they're
 * co-located (e.g. *.part1.rar + *.part2.rar in the same directory).
 *
 * Returns [] on any failure (best-effort: ingestion still succeeds with
 * an empty file list rather than failing the whole archive).
 */
 export async function readRarContents(
  firstPartPath: string
 ): Promise<FileEntry[]> {
  try {
-    const { stdout } = await execFileAsync("unrar", ["l", "-v", firstPartPath], {
+    const { stdout } = await execFileAsync("unrar", ["lt", firstPartPath], {
-      timeout: 30000,
+      timeout: 60_000,
-      maxBuffer: 10 * 1024 * 1024, // 10MB for very large archives
+      maxBuffer: 50 * 1024 * 1024, // 50MB for archives with very many files
    });
-    return parseUnrarOutput(stdout);
+    const entries = parseUnrarTechnical(stdout);
    if (entries.length === 0) {
      // Log a sample of the output so we can diagnose format changes
      log.warn(
        { file: firstPartPath, sample: stdout.slice(0, 500) },
        "unrar lt returned no parseable entries"
      );
    }
    return entries;
  } catch (err) {
    log.warn({ err, file: firstPartPath }, "Failed to read RAR contents");
-    return []; // Fallback: return empty on error
+    return [];
  }
 }
 /**
- * Parse the tabular output of `unrar l -v`.
+ * Parse `unrar lt` output: header followed by per-file key:value blocks
 * separated by blank lines.
 *
- * Example output format:
+ * Example block:
 *  Archive: test.rar
 *  Details: RAR 5
 *
- *   Attributes      Size     Packed Ratio   Date   Time   CRC-32  Name
+ *         Name: folder/file.stl
- *  ----------- ---------  --------- ----- -------- ----- --------  ----
+ *         Type: File
- *   ...A....      12345      10234  83%  2024-01-15 10:30 DEADBEEF  folder/file.stl
+ *         Size: 12345
- *  ----------- ---------  --------- ----- -------- ----- --------  ----
+ *  Packed size: 10234
 *        Ratio: 83%
 *        mtime: 2024-01-15 10:30:00,000000000
 *   Attributes: ..A....
 *        CRC32: DEADBEEF
 *      Host OS: Windows
 *  Compression: RAR 5.0(v50) -m3 -md=32M
 */
-function parseUnrarOutput(output: string): FileEntry[] {
+function parseUnrarTechnical(output: string): FileEntry[] {
  const entries: FileEntry[] = [];
-  const lines = output.split("\n");
+  // Split into blocks on blank lines, then on each block read key:value pairs.
  const blocks = output.split(/\r?\n\s*\r?\n/);
-  let inFileList = false;
+  for (const block of blocks) {
-  let separatorCount = 0;
+    const fields = parseBlock(block);
    if (!fields) continue;
-  for (const line of lines) {
+    // Only File entries (skip Directory, and anything missing the basics)
-    const trimmed = line.trim();
+    if (fields.type && fields.type.toLowerCase() !== "file") continue;
    if (!fields.name || fields.size === undefined) continue;
-    // Detect separator lines (------- pattern)
+    const filePath = fields.name;
-    if (/^-{5,}/.test(trimmed)) {
+    const ext = path.extname(filePath).toLowerCase();
-      separatorCount++;
+    entries.push({
-      if (separatorCount === 1) {
+      path: filePath,
-        inFileList = true;
+      fileName: path.basename(filePath),
-      } else if (separatorCount >= 2) {
+      extension: ext ? ext.slice(1) : null,
-        inFileList = false;
+      uncompressedSize: BigInt(fields.size),
-      }
+      compressedSize: fields.packedSize !== undefined
-      continue;
+        ? BigInt(fields.packedSize)
-    }
+        : BigInt(fields.size),
-
+      crc32: fields.crc32 ? fields.crc32.toLowerCase() : null,
-    if (!inFileList) continue;
+    });
    // Parse file entry line
    // Format: Attributes Size Packed Ratio Date Time CRC Name
    const match = trimmed.match(
      /^\S+\s+(\d+)\s+(\d+)\s+\d+%\s+\S+\s+\S+\s+([0-9A-Fa-f]+)\s+(.+)$/
    );
    if (match) {
      const [, uncompressedStr, compressedStr, crc32, filePath] = match;
      // Skip directory entries (typically end with / or have size 0 with dir attributes)
      if (filePath.endsWith("/") || filePath.endsWith("\\")) continue;
      const ext = path.extname(filePath).toLowerCase();
      entries.push({
        path: filePath,
        fileName: path.basename(filePath),
        extension: ext ? ext.slice(1) : null,
        compressedSize: BigInt(compressedStr),
        uncompressedSize: BigInt(uncompressedStr),
        crc32: crc32.toLowerCase(),
      });
    }
  }
  return entries;
 }
 interface BlockFields {
  name?: string;
  type?: string;
  size?: string;
  packedSize?: string;
  crc32?: string;
 }
 function parseBlock(block: string): BlockFields | null {
  // Skip the archive-header block (contains "Archive:" / "Details:" lines
  // and lacks a Name field).
  if (!/^\s*Name:/m.test(block)) return null;
  const fields: BlockFields = {};
  const lines = block.split(/\r?\n/);
  for (const line of lines) {
    // Match "  key: value" with arbitrary leading whitespace and a multi-word
    // key (e.g. "Packed size", "Host OS").
    const m = line.match(/^\s*([A-Za-z][A-Za-z0-9 ]*?)\s*:\s*(.*)$/);
    if (!m) continue;
    const key = m[1].trim().toLowerCase();
    const value = m[2].trim();
    if (key === "name") fields.name = value;
    else if (key === "type") fields.type = value;
    else if (key === "size") fields.size = value;
    else if (key === "packed size") fields.packedSize = value;
    else if (key === "crc32" || key === "blake2sp" || key === "checksum") {
      // unrar may report BLAKE2sp for newer archives instead of CRC32.
      // Either way we just store it as a hex string in our crc32 field.
      fields.crc32 = value;
    }
  }
  return fields;
 }
@@ -0,0 +1,58 @@
 import type { FileEntry } from "./zip-reader.js";
 /**
 * Mapping from file extensions to slicer tags. Each tag groups a family of
 * extensions that mean the same thing for end users — "this archive contains
 * files I can open in <slicer>".
 *
 * Extensions are matched case-insensitively without the leading dot.
 */
 const SLICER_EXTENSION_MAP: Record<string, string> = {
  // Lychee Slicer
  lys: "lychee",
  lyt: "lychee",
  lyc: "lychee",
  // ChituBox / Anycubic / Phrozen / Elegoo (resin printers)
  chitubox: "chitubox",
  ctb: "chitubox",
  cbddlp: "chitubox",
  // Anycubic Photon family
  photon: "anycubic",
  pwmo: "anycubic",
  pwmx: "anycubic",
  pwmb: "anycubic",
  pwma: "anycubic",
  pws: "anycubic",
  pwsq: "anycubic",
  phz: "anycubic",
  // Bambu / Prusa
  "3mf": "bambu",
  bgcode: "bambu",
  // FDM gcode (generic)
  gcode: "fdm",
  // Mango / generic resin formats sometimes seen in releases
  mfp: "mango",
  mfpv: "mango",
  osla: "mango",
 };
 /**
 * Derive a deduplicated list of slicer tags from an archive's file listing.
 * Returns an empty array if no recognised slicer-specific files are present
 * (e.g., the archive is just STLs without pre-supports).
 */
 export function extractSlicerTags(entries: FileEntry[]): string[] {
  const tags = new Set<string>();
  for (const entry of entries) {
    if (!entry.extension) continue;
    const ext = entry.extension.toLowerCase();
    const tag = SLICER_EXTENSION_MAP[ext];
    if (tag) tags.add(tag);
  }
  return [...tags].sort();
 }
@@ -18,20 +18,22 @@ const log = childLogger("split");
 const MAX_PART_SIZE = BigInt(config.maxPartSizeMB) * 1024n * 1024n;
 /**
- * Split a file into ≤2GB parts using byte-level splitting.
+ * Split a file into parts using byte-level splitting.
- * Returns paths to the split parts. If the file is already ≤2GB, returns the original path.
+ * Returns paths to the split parts. If the file fits in one part, returns the original path.
 * Pass maxPartSize to override the global default (e.g., 3950 MiB for Premium accounts).
 */
-export async function byteLevelSplit(filePath: string): Promise<string[]> {
+export async function byteLevelSplit(filePath: string, maxPartSize?: bigint): Promise<string[]> {
  const effectiveMax = maxPartSize ?? MAX_PART_SIZE;
  const stats = await stat(filePath);
  const fileSize = BigInt(stats.size);
-  if (fileSize <= MAX_PART_SIZE) {
+  if (fileSize <= effectiveMax) {
    return [filePath];
  }
  const dir = path.dirname(filePath);
  const baseName = path.basename(filePath);
-  const partSize = Number(MAX_PART_SIZE);
+  const partSize = Number(effectiveMax);
  const totalParts = Math.ceil(Number(fileSize) / partSize);
  const parts: string[] = [];
@@ -38,7 +38,9 @@ export async function runIntegrityAudit(): Promise<{ checked: number; issues: nu
  for (const pkg of multipartPackages) {
    const actualParts = pkg.destMessageIds.length;
-    if (actualParts > 0 && actualParts !== pkg.partCount) {
+    // Only flag when we have >1 stored IDs but count doesn't match.
    // Packages with exactly 1 ID are legacy (backfilled from single destMessageId) — not actionable.
    if (actualParts > 1 && actualParts !== pkg.partCount) {
      issues++;
      // Check if we already have a notification for this
@@ -0,0 +1,343 @@
 import path from "path";
 import { mkdir, rm } from "fs/promises";
 import { db } from "./db/client.js";
 import { config } from "./util/config.js";
 import { childLogger } from "./util/logger.js";
 import { withTdlibMutex } from "./util/mutex.js";
 import { createTdlibClient, closeTdlibClient } from "./tdlib/client.js";
 import { downloadFile } from "./tdlib/download.js";
 import { getActiveAccounts } from "./db/queries.js";
 import { readZipCentralDirectory } from "./archive/zip-reader.js";
 import { readRarContents } from "./archive/rar-reader.js";
 import { read7zContents } from "./archive/sevenz-reader.js";
 import { extractSlicerTags } from "./archive/slicer-tags.js";
 import type { FileEntry } from "./archive/zip-reader.js";
 const log = childLogger("backfill");
 /**
 * Re-extract file listings for Packages whose fileCount is 0 — usually
 * caused by historical bugs in the archive readers (e.g. the RAR parser
 * that silently returned [] for every archive before 0bdd4ba).
 *
 * For each candidate Package:
 *   1. Download all destMessageIds from the destination channel
 *   2. Run the appropriate reader (ZIP / RAR / 7Z) on the assembled files
 *   3. Insert PackageFile rows + update Package.fileCount
 *   4. Clean up the temp files
 *
 * Triggered via pg_notify "backfill_filelists" with optional payload
 * `{"limit": N, "archiveType": "RAR"}` — both fields optional, defaults
 * are limit=100, archiveType=any.
 */
 export async function processBackfillRequest(payloadJson: string): Promise<void> {
  let limit = 100;
  let archiveTypeFilter: "ZIP" | "RAR" | "SEVEN_Z" | undefined;
  try {
    const parsed = JSON.parse(payloadJson) as { limit?: number; archiveType?: string };
    if (typeof parsed.limit === "number" && parsed.limit > 0) limit = parsed.limit;
    if (parsed.archiveType === "ZIP" || parsed.archiveType === "RAR" || parsed.archiveType === "SEVEN_Z") {
      archiveTypeFilter = parsed.archiveType;
    }
  } catch {
    // Empty / invalid payload — use defaults
  }
  const candidates = await db.package.findMany({
    where: {
      fileCount: 0,
      destChannelId: { not: null },
      destMessageId: { not: null },
      archiveType: archiveTypeFilter
        ? archiveTypeFilter
        : { in: ["ZIP", "RAR", "SEVEN_Z"] },
    },
    select: {
      id: true,
      fileName: true,
      fileSize: true,
      archiveType: true,
      destChannelId: true,
      destMessageId: true,
      destMessageIds: true,
      isMultipart: true,
      partCount: true,
    },
    orderBy: { createdAt: "asc" },
    take: limit,
  });
  if (candidates.length === 0) {
    log.info({ archiveTypeFilter }, "Backfill: no candidates with fileCount=0");
    return;
  }
  log.info(
    { count: candidates.length, archiveTypeFilter },
    "Backfill: starting batch"
  );
  const accounts = await getActiveAccounts();
  if (accounts.length === 0) {
    log.warn("Backfill: no authenticated accounts — aborting");
    return;
  }
  // Prefer the Premium account if available (faster downloads, larger files)
  const account = accounts.find((a) => a.isPremium) ?? accounts[0];
  await withTdlibMutex(account.phone, "backfill", async () => {
    const { client } = await createTdlibClient({ id: account.id, phone: account.phone });
    try {
      // Load chats so TDLib knows about the destination chat
      try {
        await client.invoke({
          _: "getChats",
          chat_list: { _: "chatListMain" },
          limit: 1000,
        });
      } catch {
        // May already be loaded
      }
      let processed = 0;
      let succeeded = 0;
      let failed = 0;
      for (const pkg of candidates) {
        processed++;
        const ctx = { packageId: pkg.id, fileName: pkg.fileName };
        try {
          await processOnePackage(client, pkg, ctx);
          succeeded++;
        } catch (err) {
          failed++;
          log.warn({ err, ...ctx }, "Backfill failed for package");
        }
      }
      log.info(
        { processed, succeeded, failed, archiveTypeFilter },
        "Backfill batch complete"
      );
    } finally {
      await closeTdlibClient(client).catch(() => {});
    }
  });
 }
 interface BackfillPackage {
  id: string;
  fileName: string;
  fileSize: bigint;
  archiveType: "ZIP" | "RAR" | "SEVEN_Z" | "DOCUMENT" | string;
  destChannelId: string | null;
  destMessageId: bigint | null;
  destMessageIds: bigint[];
  isMultipart: boolean;
  partCount: number;
 }
 async function processOnePackage(
  // eslint-disable-next-line @typescript-eslint/no-explicit-any
  client: any,
  pkg: BackfillPackage,
  ctx: { packageId: string; fileName: string }
 ): Promise<void> {
  if (!pkg.destChannelId || !pkg.destMessageId) {
    log.debug(ctx, "Skipping: no destination channel/message");
    return;
  }
  // Look up the destination channel's Telegram ID
  const destChannel = await db.telegramChannel.findUnique({
    where: { id: pkg.destChannelId },
    select: { telegramId: true },
  });
  if (!destChannel) {
    throw new Error("Destination channel not found in DB");
  }
  const chatId = Number(destChannel.telegramId);
  // Resolve which message IDs to download. The Package may carry a
  // single destMessageId or multiple destMessageIds (for multipart).
  const messageIds: bigint[] =
    pkg.destMessageIds.length > 0
      ? pkg.destMessageIds
      : pkg.destMessageId
      ? [pkg.destMessageId]
      : [];
  if (messageIds.length === 0) {
    throw new Error("Package has no destination message IDs");
  }
  const tempDir = path.join(config.tempDir, `backfill_${pkg.id}`);
  await mkdir(tempDir, { recursive: true });
  try {
    const partPaths: string[] = [];
    for (let i = 0; i < messageIds.length; i++) {
      // eslint-disable-next-line @typescript-eslint/no-explicit-any
      const message = (await client.invoke({
        _: "getMessage",
        chat_id: chatId,
        message_id: Number(messageIds[i]),
      })) as unknown as {
        content?: { document?: { file_name?: string; document?: { id: number; size: number } } };
      };
      const doc = message?.content?.document;
      if (!doc?.document?.id) {
        throw new Error(`Destination message ${messageIds[i]} has no document`);
      }
      const fileId = String(doc.document.id);
      const fileName = doc.file_name ?? `${pkg.id}.part${i + 1}`;
      const localPath = path.join(tempDir, fileName);
      await downloadFile(
        client,
        fileId,
        localPath,
        BigInt(doc.document.size),
        fileName
      );
      partPaths.push(localPath);
    }
    // Run the appropriate reader on the assembled file(s)
    let entries: FileEntry[] = [];
    if (pkg.archiveType === "ZIP") {
      entries = await readZipCentralDirectory(partPaths);
    } else if (pkg.archiveType === "RAR") {
      // unrar auto-discovers sibling parts when in the same directory
      entries = await readRarContents(partPaths[0]);
    } else if (pkg.archiveType === "SEVEN_Z") {
      entries = await read7zContents(partPaths[0]);
    } else {
      log.debug({ ...ctx, archiveType: pkg.archiveType }, "Skipping unsupported archive type");
      return;
    }
    if (entries.length === 0) {
      log.warn(ctx, "Reader returned 0 entries — archive may be encrypted or corrupt");
      return;
    }
    // Also derive slicer tags from the file list so the backfilled packages
    // gain the same search/filter context as newly-ingested ones.
    const slicerTags = extractSlicerTags(entries);
    // Write everything in a single transaction so a partial backfill never
    // leaves the Package half-indexed.
    await db.$transaction(async (tx) => {
      // Re-check fileCount inside the transaction: another worker might
      // have backfilled this package between our read and write.
      const current = await tx.package.findUnique({
        where: { id: pkg.id },
        select: { fileCount: true, tags: true },
      });
      if (current && current.fileCount > 0) {
        log.debug({ ...ctx, existingFileCount: current.fileCount }, "Already backfilled by another worker — skipping");
        return;
      }
      await tx.packageFile.deleteMany({ where: { packageId: pkg.id } });
      await tx.packageFile.createMany({
        data: entries.map((e) => ({
          packageId: pkg.id,
          path: e.path,
          fileName: e.fileName,
          extension: e.extension,
          compressedSize: e.compressedSize,
          uncompressedSize: e.uncompressedSize,
          crc32: e.crc32,
        })),
      });
      // Merge slicer tags with whatever's already on the Package (preserve
      // channel category, manual tags, etc.).
      const existingTags = current?.tags ?? [];
      const mergedTags = [...new Set([...existingTags, ...slicerTags])];
      await tx.package.update({
        where: { id: pkg.id },
        data: { fileCount: entries.length, tags: mergedTags },
      });
    });
    log.info({ ...ctx, fileCount: entries.length }, "Backfilled file list");
  } finally {
    await rm(tempDir, { recursive: true, force: true }).catch(() => {});
  }
 }
 /**
 * Cheap pure-DB backfill: walk Packages that already have PackageFile rows
 * but no slicer tags, recompute the tags from their extensions, and merge
 * with the existing tag list. No downloads, no TDLib.
 *
 * Trigger:
 *   SELECT pg_notify('backfill_slicer_tags', '{"limit":1000}');
 */
 export async function processSlicerTagBackfill(payloadJson: string): Promise<void> {
  let limit = 1000;
  try {
    const parsed = JSON.parse(payloadJson) as { limit?: number };
    if (typeof parsed.limit === "number" && parsed.limit > 0) limit = parsed.limit;
  } catch {
    // Use default
  }
  // KNOWN_TAGS = the slicer tags we know how to derive. A Package missing
  // all of these is a candidate for recompute. extractSlicerTags is safe
  // to run on every package (returns [] for archives with no slicer files),
  // but filtering up-front avoids walking the entire DB.
  const KNOWN_TAGS = ["lychee", "chitubox", "anycubic", "bambu", "fdm", "mango"];
  const candidates = await db.package.findMany({
    where: {
      fileCount: { gt: 0 },
      NOT: { tags: { hasSome: KNOWN_TAGS } },
    },
    select: {
      id: true,
      tags: true,
      files: { select: { extension: true } },
    },
    orderBy: { createdAt: "asc" },
    take: limit,
  });
  if (candidates.length === 0) {
    log.info("Slicer tag backfill: no candidates");
    return;
  }
  log.info({ count: candidates.length }, "Slicer tag backfill: starting");
  let updated = 0;
  for (const pkg of candidates) {
    const fileEntries = pkg.files.map((f) => ({
      path: "",
      fileName: "",
      extension: f.extension,
      compressedSize: 0n,
      uncompressedSize: 0n,
      crc32: null as string | null,
    }));
    const slicerTags = extractSlicerTags(fileEntries);
    if (slicerTags.length === 0) continue;
    const merged = [...new Set([...pkg.tags, ...slicerTags])];
    if (merged.length === pkg.tags.length) continue;
    await db.package.update({ where: { id: pkg.id }, data: { tags: merged } });
    updated++;
  }
  log.info({ candidates: candidates.length, updated }, "Slicer tag backfill: done");
 }
@@ -5,7 +5,14 @@ import { config } from "../util/config.js";
 const pool = new pg.Pool({
  connectionString: config.databaseUrl,
-  max: 5,
+  // Pool needs headroom for: 2 account advisory locks (held for entire cycle),
  // up to 2 concurrent hash locks, plus Prisma operations from both accounts.
  // Previously max=5 caused pool exhaustion and indefinite hangs.
  max: 15,
  // Prevent pool.connect() from blocking forever when pool is exhausted.
  // Throws an error after 30s so the operation can fail and retry instead of
  // silently hanging for hours (as happened with the Turnbase.7z stall).
  connectionTimeoutMillis: 30_000,
 });
 const adapter = new PrismaPg(pool);
@@ -79,3 +79,66 @@ export async function releaseLock(accountId: string): Promise<void> {
    client.release();
  }
 }
 /**
 * Derive a lock ID for a content hash. Prefixes with "hash:" so the resulting
 * 32-bit integer does not collide with account advisory lock IDs.
 */
 function contentHashToLockId(contentHash: string): number {
  return hashToLockId(`hash:${contentHash}`);
 }
 /**
 * Acquire a per-content-hash advisory lock before uploading.
 * Prevents two concurrent workers from uploading the same archive
 * when both scan a shared source channel.
 *
 * Returns true if acquired (proceed with upload).
 * Returns false if already held (another worker is handling this archive — skip).
 *
 * MUST be released via releaseHashLock() after createPackageStub() completes,
 * including on all error paths (use try/finally).
 */
 export async function tryAcquireHashLock(contentHash: string): Promise<boolean> {
  const lockId = contentHashToLockId(contentHash);
  const client = await pool.connect();
  try {
    const result = await client.query<{ pg_try_advisory_lock: boolean }>(
      "SELECT pg_try_advisory_lock($1)",
      [lockId]
    );
    const acquired = result.rows[0]?.pg_try_advisory_lock ?? false;
    if (acquired) {
      heldConnections.set(`hash:${contentHash}`, client);
      log.debug({ hash: contentHash.slice(0, 16), lockId }, "Hash lock acquired");
      return true;
    } else {
      client.release();
      log.debug({ hash: contentHash.slice(0, 16), lockId }, "Hash lock held by another worker — skipping");
      return false;
    }
  } catch (err) {
    client.release();
    throw err;
  }
 }
 /**
 * Release the per-content-hash advisory lock.
 * Call after createPackageStub() completes (or on any error path).
 */
 export async function releaseHashLock(contentHash: string): Promise<void> {
  const lockId = contentHashToLockId(contentHash);
  const client = heldConnections.get(`hash:${contentHash}`);
  if (!client) {
    log.warn({ hash: contentHash.slice(0, 16) }, "No held connection for hash lock release");
    return;
  }
  try {
    await client.query("SELECT pg_advisory_unlock($1)", [lockId]);
    log.debug({ hash: contentHash.slice(0, 16) }, "Hash lock released");
  } finally {
    heldConnections.delete(`hash:${contentHash}`);
    client.release();
  }
 }
@@ -1,5 +1,6 @@
 import { db } from "./client.js";
 import type { ArchiveType, FetchStatus } from "@prisma/client";
 import type { FileEntry } from "../archive/zip-reader.js";
 export async function getActiveAccounts() {
  return db.telegramAccount.findMany({
@@ -74,6 +75,108 @@ export async function getUploadedPackageByHash(contentHash: string) {
  });
 }
 export interface CreatePackageStubInput {
  contentHash: string;
  fileName: string;
  fileSize: bigint;
  archiveType: ArchiveType;
  sourceChannelId: string;
  sourceMessageId: bigint;
  sourceTopicId?: bigint | null;
  /** TDLib remote.unique_id of the first part — for future dedup. */
  remoteUniqueId?: string | null;
  destChannelId: string;
  destMessageId: bigint;
  destMessageIds: bigint[];
  isMultipart: boolean;
  partCount: number;
  ingestionRunId: string;
  creator?: string | null;
  tags?: string[];
 }
 /**
 * Write a minimal Package record immediately after Telegram confirms the upload.
 * Call this before preview/metadata extraction so recoverIncompleteUploads() can
 * detect and verify the package if the worker crashes mid-metadata.
 *
 * Follow with updatePackageWithMetadata() once file entries and preview are ready.
 */
 export async function createPackageStub(
  input: CreatePackageStubInput
 ): Promise<{ id: string }> {
  const pkg = await db.package.create({
    data: {
      contentHash: input.contentHash,
      fileName: input.fileName,
      fileSize: input.fileSize,
      archiveType: input.archiveType,
      sourceChannelId: input.sourceChannelId,
      sourceMessageId: input.sourceMessageId,
      sourceTopicId: input.sourceTopicId ?? undefined,
      remoteUniqueId: input.remoteUniqueId ?? undefined,
      destChannelId: input.destChannelId,
      destMessageId: input.destMessageId,
      destMessageIds: input.destMessageIds,
      isMultipart: input.isMultipart,
      partCount: input.partCount,
      fileCount: 0,
      ingestionRunId: input.ingestionRunId,
      creator: input.creator ?? undefined,
      tags: input.tags?.length ? input.tags : undefined,
    },
    select: { id: true },
  });
  try {
    await db.$queryRawUnsafe(
      `SELECT pg_notify('new_package', $1)`,
      JSON.stringify({
        packageId: pkg.id,
        fileName: input.fileName,
        creator: input.creator ?? null,
        tags: input.tags ?? [],
      })
    );
  } catch {
    // Best-effort
  }
  return pkg;
 }
 /**
 * Update a stub Package with file entries and preview after metadata extraction.
 * Called as Phase 2 of the two-phase write after createPackageStub().
 */
 export async function updatePackageWithMetadata(
  packageId: string,
  input: {
    files: {
      path: string;
      fileName: string;
      extension: string | null;
      compressedSize: bigint;
      uncompressedSize: bigint;
      crc32: string | null;
    }[];
    previewData?: Buffer | null;
    previewMsgId?: bigint | null;
  }
 ): Promise<void> {
  await db.package.update({
    where: { id: packageId },
    data: {
      fileCount: input.files.length,
      previewData: input.previewData ? new Uint8Array(input.previewData) : undefined,
      previewMsgId: input.previewMsgId ?? undefined,
      files: {
        create: input.files,
      },
    },
  });
 }
 /**
 * Check if a package already exists for a given source message ID
 * AND was successfully uploaded to the destination (destMessageId is set).
@@ -90,6 +193,73 @@ export async function packageExistsBySourceMessage(
  return pkg !== null;
 }
 /**
 * Strongest pre-download dedup signal: a Package in this channel already
 * has a matching TDLib remote.unique_id. The unique_id is stable across
 * reposts of the exact same file content, so a hit is a guaranteed
 * (lossless) duplicate. No false positives.
 *
 * Falls back to the older findRepostedPackage (name + size) for packages
 * that were ingested before we started capturing remote.unique_id.
 */
 export async function findPackageByRemoteUniqueId(
  sourceChannelId: string,
  remoteUniqueId: string
 ): Promise<{
  id: string;
  destMessageId: bigint | null;
  sourceTopicId: bigint | null;
 } | null> {
  return db.package.findFirst({
    where: {
      sourceChannelId,
      remoteUniqueId,
      destMessageId: { not: null },
    },
    orderBy: { sourceTopicId: { sort: "desc", nulls: "last" } },
    select: { id: true, destMessageId: true, sourceTopicId: true },
  });
 }
 /**
 * Detect a likely repost: same source channel + same fileName + same total
 * fileSize already exists with destMessageId set. Used to skip downloads
 * when the channel admin re-posts the same file under a new message ID
 * (which `packageExistsBySourceMessage` cannot catch because the message ID
 * is different).
 *
 * Returns the existing package's destMessageId for logging/observability,
 * or null if no match. Approximate: same name + same total size is an
 * extremely strong signal that it's the same content, but theoretically
 * two unrelated files could collide. If that ever happens, the new file
 * gets treated as a duplicate and is lost; the user can manually re-link
 * via the UI by removing the existing Package.
 */
 export async function findRepostedPackage(
  sourceChannelId: string,
  fileName: string,
  fileSize: bigint
 ): Promise<{
  id: string;
  destMessageId: bigint | null;
  sourceTopicId: bigint | null;
 } | null> {
  return db.package.findFirst({
    where: {
      sourceChannelId,
      fileName,
      fileSize,
      destMessageId: { not: null },
    },
    // Prefer the existing Package with the most specific (non-NULL)
    // sourceTopicId, so when the user is re-scanning and the file already
    // exists in a specific topic, the audit log shows the most informative
    // match. NULLS LAST in DESC order achieves that for any non-null IDs.
    orderBy: { sourceTopicId: { sort: "desc", nulls: "last" } },
    select: { id: true, destMessageId: true, sourceTopicId: true },
  });
 }
 /**
 * Delete orphaned Package rows that have the same content hash but never
 * completed the upload (destMessageId is null). Called before creating a
@@ -200,10 +370,13 @@ export interface ActivityUpdate {
  downloadedBytes?: bigint | null;
  totalBytes?: bigint | null;
  downloadPercent?: number | null;
  currentTopicId?: bigint | null;
  currentAccountChannelMapId?: string | null;
  messagesScanned?: number;
  zipsFound?: number;
  zipsDuplicate?: number;
  zipsIngested?: number;
  zipsBackfilled?: number;
 }
 export async function updateRunActivity(
@@ -227,6 +400,11 @@ export async function updateRunActivity(
      ...(activity.zipsFound !== undefined && { zipsFound: activity.zipsFound }),
      ...(activity.zipsDuplicate !== undefined && { zipsDuplicate: activity.zipsDuplicate }),
      ...(activity.zipsIngested !== undefined && { zipsIngested: activity.zipsIngested }),
      ...(activity.zipsBackfilled !== undefined && { zipsBackfilled: activity.zipsBackfilled }),
      ...(activity.currentTopicId !== undefined && { currentTopicId: activity.currentTopicId }),
      ...(activity.currentAccountChannelMapId !== undefined && {
        currentAccountChannelMapId: activity.currentAccountChannelMapId,
      }),
    },
  });
 }
@@ -241,6 +419,8 @@ const CLEAR_ACTIVITY = {
  downloadedBytes: null,
  totalBytes: null,
  downloadPercent: null,
  currentTopicId: null,
  currentAccountChannelMapId: null,
  lastActivityAt: new Date(),
 };
@@ -251,6 +431,7 @@ export async function completeIngestionRun(
    zipsFound: number;
    zipsDuplicate: number;
    zipsIngested: number;
    zipsBackfilled: number;
  }
 ) {
  return db.ingestionRun.update({
@@ -286,6 +467,73 @@ export async function updateLastProcessedMessage(
  });
 }
 export interface ScanStateUpdate {
  /** New watermark to persist. Use the same value the caller would have
   *  passed to updateLastProcessedMessage / upsertTopicProgress. */
  lastProcessedMessageId: bigint | null;
  /** True if the scan found archives OR has retryable SkippedPackages
   *  pending. The caller computes this via the trulyIdle formula. */
  lastScanFoundArchives: boolean;
  /** Pre-incremented value of consecutiveEmptyScans. Caller passes:
   *    trulyIdle ? prev + 1 : 0
   *  We do the arithmetic outside the helper so the helper stays a pure
   *  setter — easier to reason about. */
  consecutiveEmptyScans: number;
 }
 /**
 * Atomically update an AccountChannelMap's watermark and scan-state fields.
 * Replaces the older updateLastProcessedMessage for the post-scan write.
 * Sets lastScannedAt = NOW() server-side.
 */
 export async function upsertChannelScanState(
  mappingId: string,
  update: ScanStateUpdate
 ) {
  return db.accountChannelMap.update({
    where: { id: mappingId },
    data: {
      lastProcessedMessageId: update.lastProcessedMessageId ?? undefined,
      lastScannedAt: new Date(),
      lastScanFoundArchives: update.lastScanFoundArchives,
      consecutiveEmptyScans: update.consecutiveEmptyScans,
    },
  });
 }
 /**
 * Atomically upsert a TopicProgress row with the new watermark + scan-state
 * fields. Same semantics as upsertChannelScanState but for forum topics.
 */
 export async function upsertTopicScanState(
  accountChannelMapId: string,
  topicId: bigint,
  topicName: string | null,
  update: ScanStateUpdate
 ) {
  return db.topicProgress.upsert({
    where: {
      accountChannelMapId_topicId: { accountChannelMapId, topicId },
    },
    create: {
      accountChannelMapId,
      topicId,
      topicName,
      lastProcessedMessageId: update.lastProcessedMessageId,
      lastScannedAt: new Date(),
      lastScanFoundArchives: update.lastScanFoundArchives,
      consecutiveEmptyScans: update.consecutiveEmptyScans,
    },
    update: {
      topicName,
      lastProcessedMessageId: update.lastProcessedMessageId ?? undefined,
      lastScannedAt: new Date(),
      lastScanFoundArchives: update.lastScanFoundArchives,
      consecutiveEmptyScans: update.consecutiveEmptyScans,
    },
  });
 }
 export async function markStaleRunsAsFailed() {
  return db.ingestionRun.updateMany({
    where: { status: "RUNNING" },
@@ -308,6 +556,16 @@ export async function updateAccountAuthState(
  });
 }
 export async function updateAccountPremiumStatus(
  accountId: string,
  isPremium: boolean
 ): Promise<void> {
  await db.telegramAccount.update({
    where: { id: accountId },
    data: { isPremium },
  });
 }
 export async function getAccountAuthCode(accountId: string) {
  const account = await db.telegramAccount.findUnique({
    where: { id: accountId },
@@ -413,6 +671,45 @@ export async function upsertTopicProgress(
  });
 }
 /**
 * Ensure a TopicProgress row exists for every discovered topic so the admin
 * UI can list and toggle them — including brand-new topics. Inserts missing
 * rows only (fetchEnabled defaults to true); never overwrites watermarks,
 * scan-state, or the user's fetchEnabled choice on rows that already exist.
 */
 export async function ensureTopicProgressRows(
  mappingId: string,
  topics: { topicId: bigint; name: string | null }[]
 ): Promise<void> {
  if (topics.length === 0) return;
  await db.topicProgress.createMany({
    data: topics.map((t) => ({
      accountChannelMapId: mappingId,
      topicId: t.topicId,
      topicName: t.name,
    })),
    skipDuplicates: true,
  });
 }
 /**
 * Read the CURRENT per-topic fetch flag straight from the DB. Returns true
 * when no row exists yet (default = enabled). Called fresh inside the topic
 * loop so a mid-run toggle is honoured for topics not yet started.
 */
 export async function isTopicFetchEnabled(
  mappingId: string,
  topicId: bigint
 ): Promise<boolean> {
  const row = await db.topicProgress.findUnique({
    where: {
      accountChannelMapId_topicId: { accountChannelMapId: mappingId, topicId },
    },
    select: { fetchEnabled: true },
  });
  return row?.fetchEnabled ?? true;
 }
 // ── Channel fetch requests (DB-mediated communication with web app) ──
 export async function getChannelFetchRequest(requestId: string) {
@@ -516,6 +813,7 @@ export async function upsertSkippedPackage(data: {
      errorMessage: data.errorMessage ?? null,
      fileName: data.fileName,
      fileSize: data.fileSize,
      attemptCount: { increment: 1 },
      createdAt: new Date(),
    },
    create: {
@@ -533,6 +831,26 @@ export async function upsertSkippedPackage(data: {
  });
 }
 /**
 * Return source-message IDs in a channel whose SkippedPackage attemptCount has
 * reached or exceeded the cap — these are treated as "permanently failed for
 * now" so the watermark can advance past them. The user can manually retry via
 * the UI to reset the SkippedPackage record.
 */
 export async function getCappedSkippedMessageIds(
  sourceChannelId: string,
  cap: number
 ): Promise<Set<bigint>> {
  const rows = await db.skippedPackage.findMany({
    where: {
      sourceChannelId,
      attemptCount: { gte: cap },
    },
    select: { sourceMessageId: true },
  });
  return new Set(rows.map((r) => r.sourceMessageId));
 }
 export async function deleteSkippedPackage(
  sourceChannelId: string,
  sourceMessageId: bigint
@@ -542,6 +860,63 @@ export async function deleteSkippedPackage(
  });
 }
 /**
 * Find SkippedPackages for a given account+channel that are still eligible
 * for auto-retry (attemptCount below the cap). Used at the start of a scan
 * to pull the watermark back so we don't strand failed messages forever
 * after the watermark has advanced past them.
 *
 * For non-forum channels, pass `topicId: null` to get rows with NULL topic.
 * For forum channels, pass the topic ID to scope to that topic only.
 */
 export async function getRetryableSkippedMessageIds(args: {
  accountId: string;
  sourceChannelId: string;
  topicId: bigint | null;
  cap: number;
 }): Promise<bigint[]> {
  const rows = await db.skippedPackage.findMany({
    where: {
      accountId: args.accountId,
      sourceChannelId: args.sourceChannelId,
      sourceTopicId: args.topicId,
      attemptCount: { lt: args.cap },
    },
    select: { sourceMessageId: true },
    orderBy: { sourceMessageId: "asc" },
  });
  return rows.map((r) => r.sourceMessageId);
 }
 /**
 * Update a Package's source topic when a more specific topic context is
 * discovered for the same content. Used when findRepostedPackage matches
 * an existing Package whose topic is less specific (e.g., "General") than
 * the topic we just encountered the file in.
 *
 * Also updates the creator if the new topic name is more informative than
 * the existing creator (i.e., the existing creator was derived from a
 * less-specific topic name like "General").
 */
 export async function updatePackageTopicContext(
  packageId: string,
  newTopicId: bigint,
  newTopicName: string | null
 ): Promise<void> {
  await db.package.update({
    where: { id: packageId },
    data: {
      sourceTopicId: newTopicId,
      // Only overwrite creator if the new topic name is meaningful (non-empty,
      // non-General). Keeps explicit creator values from filename or admin
      // input intact.
      ...(newTopicName && newTopicName !== "General"
        ? { creator: newTopicName }
        : {}),
    },
  });
 }
 export async function createOrFindPackageGroup(input: {
  mediaAlbumId: string;
  sourceChannelId: string;
@@ -634,3 +1009,142 @@ export async function createAutoGroup(input: {
  return group.id;
 }
 // ── Provenance backfill ──
 export interface PlaceholderCandidate {
  id: string;
  archiveType: string;
  fileName: string;
  fileCount: number;
  fileSize: bigint;
  destMessageId: bigint | null;
  destMessageIds: bigint[];
  destChannel: { telegramId: bigint } | null;
 }
 /**
 * Find every placeholder Package matching name+size (oldest first). Package
 * has no direct `destChannel` relation (only the scalar `destChannelId`), so
 * each row's destination TelegramChannel telegramId is resolved with a
 * follow-up lookup rather than a Prisma include.
 */
 export async function findPlaceholderCandidates(
  destChannelId: string,
  fileName: string,
  fileSize: bigint,
 ): Promise<PlaceholderCandidate[]> {
  const rows = await db.package.findMany({
    where: {
      fileName,
      fileSize,
      destMessageId: { not: null },
      // Placeholder provenance (spec §1): manual-upload (source == destination)
      // OR rebuild record (sourceMessageId == 0 "unknown" sentinel).
      OR: [
        { sourceChannelId: destChannelId },
        { sourceMessageId: 0n },
      ],
    },
    select: {
      id: true, archiveType: true, fileName: true, fileCount: true, fileSize: true,
      destMessageId: true, destMessageIds: true, destChannelId: true,
    },
    orderBy: { indexedAt: "asc" },
  });
  if (rows.length === 0) return [];
  const destChannelIds = [...new Set(rows.map((r) => r.destChannelId).filter((id): id is string => !!id))];
  const channels = destChannelIds.length
    ? await db.telegramChannel.findMany({
        where: { id: { in: destChannelIds } },
        select: { id: true, telegramId: true },
      })
    : [];
  const telegramIdById = new Map(channels.map((c) => [c.id, c.telegramId]));
  return rows.map((row) => ({
    id: row.id,
    archiveType: row.archiveType,
    fileName: row.fileName,
    fileCount: row.fileCount,
    fileSize: row.fileSize,
    destMessageId: row.destMessageId,
    destMessageIds: row.destMessageIds,
    destChannel: row.destChannelId && telegramIdById.has(row.destChannelId)
      ? { telegramId: telegramIdById.get(row.destChannelId)! }
      : null,
  }));
 }
 export async function findPlaceholderCandidate(
  destChannelId: string,
  fileName: string,
  fileSize: bigint,
 ): Promise<PlaceholderCandidate | null> {
  return (await findPlaceholderCandidates(destChannelId, fileName, fileSize))[0] ?? null;
 }
 export async function getPackageFileCrcs(packageId: string): Promise<(string | null)[]> {
  const rows = await db.packageFile.findMany({
    where: { packageId },
    select: { crc32: true },
  });
  return rows.map((r) => r.crc32);
 }
 export interface BackfillProvenanceInput {
  packageId: string;
  destChannelId: string; // to re-check placeholder status in-txn
  sourceChannelId: string;
  sourceMessageId: bigint;
  sourceTopicId: bigint | null;
  sourceCaption: string | null;
  remoteUniqueId: string | null;
  creator: string | null; // always set (re-derived by caller)
  entries?: FileEntry[]; // set only if candidate had fileCount === 0
  previewData?: Buffer | null; // set only if provided and candidate lacks one
  previewMsgId?: bigint | null;
 }
 export async function backfillProvenance(input: BackfillProvenanceInput): Promise<boolean> {
  return db.$transaction(async (tx) => {
    const current = await tx.package.findUnique({
      where: { id: input.packageId },
      select: { sourceChannelId: true, sourceMessageId: true, previewData: true, fileCount: true },
    });
    // Re-check placeholder status inside the txn (another worker may have won).
    // Placeholder = manual-upload (source==dest) OR rebuild (sourceMessageId==0).
    const stillPlaceholder =
      !!current &&
      (current.sourceChannelId === input.destChannelId || current.sourceMessageId === 0n);
    if (!stillPlaceholder) return false;
    // eslint-disable-next-line @typescript-eslint/no-explicit-any
    const data: any = {
      sourceChannelId: input.sourceChannelId,
      sourceMessageId: input.sourceMessageId,
      sourceTopicId: input.sourceTopicId,
      sourceCaption: input.sourceCaption,
      remoteUniqueId: input.remoteUniqueId,
      creator: input.creator,
    };
    if (input.entries && current.fileCount === 0) {
      await tx.packageFile.deleteMany({ where: { packageId: input.packageId } });
      await tx.packageFile.createMany({
        data: input.entries.map((e) => ({
          packageId: input.packageId,
          path: e.path, fileName: e.fileName, extension: e.extension,
          compressedSize: e.compressedSize, uncompressedSize: e.uncompressedSize, crc32: e.crc32,
        })),
      });
      data.fileCount = input.entries.length;
    }
    if (input.previewData && !current.previewData) {
      data.previewData = input.previewData;
      data.previewMsgId = input.previewMsgId ?? null;
    }
    await tx.package.update({ where: { id: input.packageId }, data });
    return true;
  });
 }
@@ -101,16 +101,14 @@ export async function processExtractRequest(requestId: string): Promise<void> {
  try {
    await mkdir(tempDir, { recursive: true });
-    // Wrap the entire TDLib session in the mutex so no other TDLib
+    const accounts = await getActiveAccounts();
-    // operation can run concurrently (TDLib is single-session).
+    if (accounts.length === 0) {
-    await withTdlibMutex("extract", async () => {
+      throw new Error("No authenticated Telegram accounts available");
-      const accounts = await getActiveAccounts();
+    }
-      if (accounts.length === 0) {
+    const account = accounts[0];
        throw new Error("No authenticated Telegram accounts available");
      }
-      const account = accounts[0];
+    await withTdlibMutex(account.phone, "extract", async () => {
-      const client = await createTdlibClient({ id: account.id, phone: account.phone });
+      const { client } = await createTdlibClient({ id: account.id, phone: account.phone });
      try {
        // Load chat list so TDLib can find the dest channel
@@ -5,6 +5,8 @@ import { withTdlibMutex } from "./util/mutex.js";
 import { processFetchRequest } from "./worker.js";
 import { processExtractRequest } from "./extract-listener.js";
 import { rebuildPackageDatabase } from "./rebuild.js";
 import { processManualUpload } from "./manual-upload.js";
 import { processBackfillRequest, processSlicerTagBackfill } from "./backfill.js";
 import { generateInviteLink, createSupergroup, searchPublicChat } from "./tdlib/chats.js";
 import { createTdlibClient, closeTdlibClient } from "./tdlib/client.js";
 import { triggerImmediateCycle } from "./scheduler.js";
@@ -13,6 +15,7 @@ import {
  getGlobalSetting,
  setGlobalSetting,
  getActiveAccounts,
  getChannelFetchRequest,
  upsertChannel,
  ensureAccountChannelLink,
  updateFetchRequestStatus,
@@ -55,6 +58,9 @@ async function connectListener(): Promise<void> {
    await pgClient.query("LISTEN join_channel");
    await pgClient.query("LISTEN archive_extract");
    await pgClient.query("LISTEN rebuild_packages");
    await pgClient.query("LISTEN manual_upload");
    await pgClient.query("LISTEN backfill_filelists");
    await pgClient.query("LISTEN backfill_slicer_tags");
    pgClient.on("notification", (msg) => {
      if (msg.channel === "channel_fetch" && msg.payload) {
@@ -71,6 +77,12 @@ async function connectListener(): Promise<void> {
        handleArchiveExtract(msg.payload);
      } else if (msg.channel === "rebuild_packages" && msg.payload) {
        handleRebuildPackages(msg.payload);
      } else if (msg.channel === "manual_upload" && msg.payload) {
        handleManualUpload(msg.payload);
      } else if (msg.channel === "backfill_filelists") {
        handleBackfillFilelists(msg.payload ?? "{}");
      } else if (msg.channel === "backfill_slicer_tags") {
        handleBackfillSlicerTags(msg.payload ?? "{}");
      }
    });
@@ -96,7 +108,7 @@ async function connectListener(): Promise<void> {
      }
    });
-    log.info("Fetch listener started (channel_fetch, generate_invite, create_destination, ingestion_trigger, join_channel, archive_extract, rebuild_packages)");
+    log.info("Fetch listener started (channel_fetch, generate_invite, create_destination, ingestion_trigger, join_channel, archive_extract, rebuild_packages, manual_upload, backfill_filelists, backfill_slicer_tags)");
  } catch (err) {
    log.error({ err }, "Failed to start fetch listener — retrying");
    scheduleReconnect();
@@ -129,7 +141,9 @@ let fetchQueue: Promise<void> = Promise.resolve();
 function handleChannelFetch(requestId: string): void {
  fetchQueue = fetchQueue.then(async () => {
    try {
-      await withTdlibMutex("fetch-channels", () =>
+      const request = await getChannelFetchRequest(requestId);
      const key = request?.account?.phone ?? "global";
      await withTdlibMutex(key, "fetch-channels", () =>
        processFetchRequest(requestId)
      );
    } catch (err) {
@@ -143,22 +157,20 @@ function handleChannelFetch(requestId: string): void {
 function handleGenerateInvite(channelId: string): void {
  fetchQueue = fetchQueue.then(async () => {
    try {
-      await withTdlibMutex("generate-invite", async () => {
+      const accounts = await getActiveAccounts();
      if (accounts.length === 0) {
        log.warn("No authenticated accounts to generate invite link");
        return;
      }
      const account = accounts[0];
      await withTdlibMutex(account.phone, "generate-invite", async () => {
        const destChannel = await getGlobalDestinationChannel();
        if (!destChannel || destChannel.id !== channelId) {
          log.warn({ channelId }, "Destination channel mismatch, skipping invite generation");
          return;
        }
-        // Use the first available authenticated account to generate the link
+        const { client } = await createTdlibClient({ id: account.id, phone: account.phone });
        const accounts = await getActiveAccounts();
        if (accounts.length === 0) {
          log.warn("No authenticated accounts to generate invite link");
          return;
        }
        const account = accounts[0];
        const client = await createTdlibClient({ id: account.id, phone: account.phone });
        try {
          const link = await generateInviteLink(client, destChannel.telegramId);
@@ -183,7 +195,13 @@ function handleCreateDestination(payload: string): void {
      const parsed = JSON.parse(payload) as { requestId: string; title: string };
      requestId = parsed.requestId;
-      await withTdlibMutex("create-destination", async () => {
+      const accounts = await getActiveAccounts();
      if (accounts.length === 0) {
        throw new Error("No authenticated accounts available to create the group");
      }
      const account = accounts[0];
      await withTdlibMutex(account.phone, "create-destination", async () => {
        const { db } = await import("./db/client.js");
        // Mark the request as in-progress
@@ -192,14 +210,7 @@ function handleCreateDestination(payload: string): void {
          data: { status: "IN_PROGRESS" },
        });
-        // Use the first available authenticated account
+        const { client } = await createTdlibClient({ id: account.id, phone: account.phone });
        const accounts = await getActiveAccounts();
        if (accounts.length === 0) {
          throw new Error("No authenticated accounts available to create the group");
        }
        const account = accounts[0];
        const client = await createTdlibClient({ id: account.id, phone: account.phone });
        try {
          // Create the supergroup via TDLib
@@ -324,16 +335,16 @@ function handleJoinChannel(payload: string): void {
      const parsed = JSON.parse(payload) as { requestId: string; input: string; accountId: string };
      requestId = parsed.requestId;
-      await withTdlibMutex("join-channel", async () => {
+      const accounts = await getActiveAccounts();
      const account = accounts.find((a) => a.id === parsed.accountId) ?? accounts[0];
      if (!account) {
        throw new Error("No authenticated accounts available");
      }
      await withTdlibMutex(account.phone, "join-channel", async () => {
        await updateFetchRequestStatus(requestId!, "IN_PROGRESS");
-        const accounts = await getActiveAccounts();
+        const { client } = await createTdlibClient({ id: account.id, phone: account.phone });
        const account = accounts.find((a) => a.id === parsed.accountId) ?? accounts[0];
        if (!account) {
          throw new Error("No authenticated accounts available");
        }
        const client = await createTdlibClient({ id: account.id, phone: account.phone });
        try {
          const linkInfo = parseTelegramInput(parsed.input);
@@ -503,7 +514,12 @@ function handleIngestionTrigger(): void {
 function handleRebuildPackages(requestId: string): void {
  fetchQueue = fetchQueue.then(async () => {
    try {
-      await withTdlibMutex("rebuild-packages", () =>
+      const accounts = await getActiveAccounts();
      if (accounts.length === 0) {
        log.warn("No authenticated accounts to rebuild packages");
        return;
      }
      await withTdlibMutex(accounts[0].phone, "rebuild-packages", () =>
        rebuildPackageDatabase(requestId)
      );
    } catch (err) {
@@ -511,3 +527,38 @@ function handleRebuildPackages(requestId: string): void {
    }
  });
 }
 // ── Manual upload handler ──
 function handleManualUpload(uploadId: string): void {
  fetchQueue = fetchQueue
    .then(() => processManualUpload(uploadId))
    .catch((err) => log.error({ err, uploadId }, "Manual upload processing failed"));
 }
 // ── Backfill file-list handler ──
 //
 // Trigger via:
 //   SELECT pg_notify('backfill_filelists', '{"limit":50,"archiveType":"RAR"}');
 //
 // Both fields are optional. archiveType filters to one of ZIP/RAR/SEVEN_Z.
 // Default limit is 100. The handler queues so multiple notifications run
 // sequentially (no concurrent TDLib downloads competing for the mutex).
 function handleBackfillFilelists(payload: string): void {
  fetchQueue = fetchQueue
    .then(() => processBackfillRequest(payload))
    .catch((err) => log.error({ err, payload }, "Backfill request failed"));
 }
 // ── Slicer tag backfill handler ──
 //
 // Trigger:
 //   SELECT pg_notify('backfill_slicer_tags', '{"limit":1000}');
 //
 // Pure-DB pass over Packages that have file lists but no slicer tags.
 // No downloads, no TDLib involvement — fast and safe.
 function handleBackfillSlicerTags(payload: string): void {
  fetchQueue = fetchQueue
    .then(() => processSlicerTagBackfill(payload))
    .catch((err) => log.error({ err, payload }, "Slicer tag backfill failed"));
 }
@@ -27,6 +27,33 @@ async function main(): Promise<void> {
  await cleanupTempDir();
  await markStaleRunsAsFailed();
  // Release any advisory locks orphaned by a previous worker instance.
  // When Docker kills a container, PostgreSQL may keep the session alive
  // (zombie connections), holding advisory locks that block the new worker.
  try {
    const result = await pool.query(`
      SELECT pid, state, left(query, 80) as query, age(clock_timestamp(), state_change) as idle_time
      FROM pg_stat_activity
      WHERE datname = current_database()
        AND pid != pg_backend_pid()
        AND state = 'idle'
        AND query LIKE '%pg_try_advisory_lock%'
        AND state_change < clock_timestamp() - interval '5 minutes'
    `);
    for (const row of result.rows) {
      log.warn(
        { pid: row.pid, idleTime: row.idle_time, query: row.query },
        "Terminating stale advisory lock session from previous worker"
      );
      await pool.query("SELECT pg_terminate_backend($1)", [row.pid]);
    }
    if (result.rows.length > 0) {
      log.info({ terminated: result.rows.length }, "Cleaned up stale advisory lock sessions");
    }
  } catch (err) {
    log.warn({ err }, "Failed to clean up stale advisory locks (non-fatal)");
  }
  // Verify destination messages exist for all "uploaded" packages.
  // Resets any packages whose dest message is missing so they get re-processed.
  await recoverIncompleteUploads();
@@ -0,0 +1,211 @@
 import path from "path";
 import { rm } from "fs/promises";
 import { db } from "./db/client.js";
 import { childLogger } from "./util/logger.js";
 import { config } from "./util/config.js";
 import { hashParts } from "./archive/hash.js";
 import { byteLevelSplit } from "./archive/split.js";
 import { uploadToChannel } from "./upload/channel.js";
 import { createTdlibClient, closeTdlibClient } from "./tdlib/client.js";
 import { readZipCentralDirectory } from "./archive/zip-reader.js";
 import { readRarContents } from "./archive/rar-reader.js";
 import { read7zContents } from "./archive/sevenz-reader.js";
 import { getActiveAccounts } from "./db/queries.js";
 const log = childLogger("manual-upload");
 export async function processManualUpload(uploadId: string): Promise<void> {
  log.info({ uploadId }, "Processing manual upload");
  const upload = await db.manualUpload.findUnique({
    where: { id: uploadId },
    include: { files: true },
  });
  if (!upload || upload.status !== "PENDING") {
    log.warn({ uploadId }, "Manual upload not found or not pending");
    return;
  }
  await db.manualUpload.update({
    where: { id: uploadId },
    data: { status: "PROCESSING" },
  });
  try {
    // Get destination channel
    const destSetting = await db.globalSetting.findUnique({
      where: { key: "destination_channel_id" },
    });
    if (!destSetting) throw new Error("No destination channel configured");
    const destChannel = await db.telegramChannel.findFirst({
      where: { id: destSetting.value, type: "DESTINATION", isActive: true },
    });
    if (!destChannel) throw new Error("Destination channel not found or inactive");
    // Get a TDLib client (use first active account)
    const accounts = await getActiveAccounts();
    const account = accounts[0];
    if (!account) throw new Error("No authenticated Telegram account available");
    const { client } = await createTdlibClient({ id: account.id, phone: account.phone });
    try {
      const packageIds: string[] = [];
      for (const file of upload.files) {
        try {
          const filePath = file.filePath;
          const fileName = file.fileName;
          const fileSize = file.fileSize;
          log.info({ fileName, fileSize: Number(fileSize) }, "Processing file");
          // Determine archive type
          let archiveType: "ZIP" | "RAR" | "SEVEN_Z" | "DOCUMENT" = "DOCUMENT";
          const ext = fileName.toLowerCase();
          if (ext.endsWith(".zip")) archiveType = "ZIP";
          else if (ext.endsWith(".rar")) archiveType = "RAR";
          else if (ext.endsWith(".7z")) archiveType = "SEVEN_Z";
          // Hash the file
          const contentHash = await hashParts([filePath]);
          // Check for duplicates
          const existing = await db.package.findFirst({
            where: { contentHash, destMessageId: { not: null } },
            select: { id: true },
          });
          if (existing) {
            log.info({ fileName, contentHash }, "Duplicate file, skipping upload");
            await db.manualUploadFile.update({
              where: { id: file.id },
              data: { packageId: existing.id },
            });
            packageIds.push(existing.id);
            continue;
          }
          // Read archive metadata
          let entries: {
            path: string;
            fileName: string;
            extension: string | null;
            compressedSize: bigint;
            uncompressedSize: bigint;
            crc32: string | null;
          }[] = [];
          try {
            if (archiveType === "ZIP") entries = await readZipCentralDirectory([filePath]);
            else if (archiveType === "RAR") entries = await readRarContents(filePath);
            else if (archiveType === "SEVEN_Z") entries = await read7zContents(filePath);
          } catch {
            log.debug({ fileName }, "Could not read archive metadata");
          }
          // Split if needed
          const MAX_UPLOAD_SIZE = BigInt(config.maxPartSizeMB) * 1024n * 1024n;
          let uploadPaths = [filePath];
          if (fileSize > MAX_UPLOAD_SIZE) {
            uploadPaths = await byteLevelSplit(filePath);
          }
          // Upload to Telegram
          const destResult = await uploadToChannel(
            client,
            destChannel.telegramId,
            uploadPaths
          );
          // Create package record
          const pkg = await db.package.create({
            data: {
              contentHash,
              fileName,
              fileSize,
              archiveType,
              sourceChannelId: destChannel.id,
              sourceMessageId: destResult.messageId,
              destChannelId: destChannel.id,
              destMessageId: destResult.messageId,
              destMessageIds: destResult.messageIds,
              isMultipart: uploadPaths.length > 1,
              partCount: uploadPaths.length,
              fileCount: entries.length,
              files: entries.length > 0 ? { create: entries } : undefined,
            },
          });
          await db.manualUploadFile.update({
            where: { id: file.id },
            data: { packageId: pkg.id },
          });
          packageIds.push(pkg.id);
          log.info({ fileName, packageId: pkg.id }, "File processed and uploaded");
          // Clean up split files (but not the original)
          if (uploadPaths.length > 1) {
            for (const splitPath of uploadPaths) {
              if (splitPath !== filePath) {
                await rm(splitPath, { force: true }).catch(() => {});
              }
            }
          }
        } catch (fileErr) {
          log.error({ err: fileErr, fileName: file.fileName }, "Failed to process file");
        }
      }
      // Group packages if multiple files
      if (packageIds.length >= 2) {
        const groupName =
          upload.groupName ?? upload.files[0].fileName.replace(/\.[^.]+$/, "");
        const group = await db.packageGroup.create({
          data: {
            name: groupName,
            sourceChannelId: destChannel.id,
            groupingSource: "MANUAL",
          },
        });
        await db.package.updateMany({
          where: { id: { in: packageIds } },
          data: { packageGroupId: group.id },
        });
        log.info(
          { groupId: group.id, groupName, packageCount: packageIds.length },
          "Created group for uploaded files"
        );
      }
      await db.manualUpload.update({
        where: { id: uploadId },
        data: { status: "COMPLETED", completedAt: new Date() },
      });
      log.info(
        { uploadId, fileCount: upload.files.length, packageCount: packageIds.length },
        "Manual upload completed"
      );
    } finally {
      await closeTdlibClient(client);
    }
  } catch (err) {
    const message = err instanceof Error ? err.message : String(err);
    log.error({ err, uploadId }, "Manual upload failed");
    await db.manualUpload.update({
      where: { id: uploadId },
      data: { status: "FAILED", errorMessage: message },
    });
  }
  // Clean up uploaded files
  try {
    const uploadDir = path.join("/data/uploads", uploadId);
    await rm(uploadDir, { recursive: true, force: true });
  } catch {
    // Best-effort cleanup
  }
 }
@@ -0,0 +1,306 @@
 import { db } from "./db/client.js";
 import { childLogger } from "./util/logger.js";
 import { downloadFileRange } from "./tdlib/range-download.js";
 import { invokeWithTimeout } from "./tdlib/download.js";
 import { parseZipCentralDirectoryFromTail, MIN_ZIP_TAIL_BYTES } from "./archive/central-directory.js";
 import { fingerprintsMatch, crcFingerprint } from "./archive/fingerprint.js";
 import {
  findPlaceholderCandidates,
  getPackageFileCrcs,
  backfillProvenance,
  type PlaceholderCandidate,
 } from "./db/queries.js";
 import type { FileEntry } from "./archive/zip-reader.js";
 import { readSevenZListingRanged, type RangedPart } from "./archive/ranged/sevenz-ranged.js";
 import { readRarListingRanged } from "./archive/ranged/rar-ranged.js";
 import { tdlibRangeReader } from "./archive/ranged/range-reader.js";
 import { fullDownloadListing } from "./archive/ranged/fallback.js";
 import type { Client } from "tdl";
 const log = childLogger("provenance-backfill");
 export interface BackfillArgs {
  client: Client;
  destChannelId: string;
  scannedSourceChannelId: string;
  fileName: string;
  fileSize: bigint;
  archiveType: string;
  sourceMessageId: bigint;
  sourceTopicId: bigint | null;
  sourceCaption: string | null;
  remoteUniqueId: string | null;
  creator: string | null;
  scannedParts: RangedPart[];
  previewData?: Buffer | null;
  previewMsgId?: bigint | null;
 }
 /**
 * Read a ZIP central directory from the tail of a (possibly multipart)
 * archive. `parts` is ordered; only the LAST part carries the EOCD record.
 * `fileSize` on each part is that part's own size (NOT the whole-archive
 * total) so the download offset stays within that part's bounds, while
 * `tailStart` passed to the parser is the logical whole-archive offset
 * (preceding parts' sizes + the offset within the last part).
 */
 async function readScannedZipListing(
  client: Client,
  parts: { fileId: string; fileSize: bigint }[],
 ): Promise<FileEntry[] | null> {
  if (parts.length === 0) return null;
  const lastPart = parts[parts.length - 1];
  const precedingSize = parts.slice(0, -1).reduce((sum, p) => sum + Number(p.fileSize), 0);
  const lastSize = Number(lastPart.fileSize);
  for (const tailBytes of [MIN_ZIP_TAIL_BYTES, MIN_ZIP_TAIL_BYTES * 4]) {
    const partOffset = Math.max(0, lastSize - tailBytes);
    const downloadLen = Math.min(tailBytes, lastSize);
    try {
      const buf = await downloadFileRange(client, lastPart.fileId, partOffset, downloadLen, lastPart.fileSize);
      const tailStart = precedingSize + partOffset;
      return parseZipCentralDirectoryFromTail(buf, tailStart);
    } catch (err) {
      if (err instanceof RangeError) continue; // try a larger tail
      log.warn({ err, fileId: lastPart.fileId }, "ranged ZIP listing failed");
      return null;
    }
  }
  return null;
 }
 /**
 * Resolve the destination copy's message(s) into ranged parts (file id +
 * size + name), in order, so a multipart destination copy is reconstructed
 * with correct per-part sizes and names (the last message carries the
 * EOCD-bearing tail part for ZIP; multipart RAR needs correctly-suffixed
 * `.partN.rar` names for `unrar` sibling discovery). Cheap-only: any TDLib
 * failure here degrades the caller to name-size confidence rather than
 * falling back to a full download.
 */
 async function resolveDestParts(
  client: Client,
  destChatTelegramId: bigint,
  destMessageIds: bigint[],
  destMessageId: bigint | null,
  fallbackFileName: string,
 ): Promise<RangedPart[] | null> {
  const messageIds = destMessageIds.length > 0 ? destMessageIds : destMessageId ? [destMessageId] : [];
  if (messageIds.length === 0) return null;
  try {
    const parts: RangedPart[] = [];
    for (const msgId of messageIds) {
      const msg = (await invokeWithTimeout(client, {
        _: "getMessage",
        chat_id: Number(destChatTelegramId),
        message_id: Number(msgId),
      })) as { content?: { document?: { document?: { id: number; size?: number }; file_name?: string } } };
      const doc = msg?.content?.document?.document;
      if (!doc?.id) return null;
      const fileName = msg?.content?.document?.file_name || fallbackFileName;
      parts.push({ fileId: String(doc.id), fileSize: BigInt(doc.size ?? 0), fileName });
    }
    return parts;
  } catch (err) {
    log.warn({ err, destMessageIds: messageIds.map(Number) }, "destination archive part resolution failed");
    return null;
  }
 }
 async function readScannedListingRanged(
  archiveType: string,
  client: Client,
  parts: RangedPart[],
 ): Promise<FileEntry[] | null> {
  const read = tdlibRangeReader(client);
  if (archiveType === "ZIP") return readScannedZipListing(client, parts);
  if (archiveType === "SEVEN_Z") return readSevenZListingRanged(parts, read);
  if (archiveType === "RAR") return readRarListingRanged(parts, read);
  return null;
 }
 /**
 * Build the CRC fingerprint entries for a placeholder candidate: start from
 * its stored PackageFile CRCs, and if those are incomplete (e.g. a rebuild
 * candidate with fileCount === 0), fall back to a fresh ranged read of the
 * candidate's own copy in the destination channel (Task 9).
 */
 async function resolveCandidateFingerprintEntries(
  client: Client,
  candidate: PlaceholderCandidate,
 ): Promise<FileEntry[]> {
  const candidateCrcs = await getPackageFileCrcs(candidate.id);
  let candidateEntries: FileEntry[] = candidateCrcs.map((crc) => ({
    path: "", fileName: "", extension: null, compressedSize: 0n, uncompressedSize: 0n, crc32: crc,
  }));
  const hasDestMessage = candidate.destMessageIds.length > 0 || candidate.destMessageId != null;
  if (!crcFingerprint(candidateEntries).complete && hasDestMessage && candidate.destChannel) {
    const destParts = await resolveDestParts(
      client,
      candidate.destChannel.telegramId,
      candidate.destMessageIds,
      candidate.destMessageId,
      candidate.fileName,
    );
    let destEntries: FileEntry[] | null = null;
    if (destParts) {
      const read = tdlibRangeReader(client);
      destEntries =
        candidate.archiveType === "ZIP" ? await readScannedZipListing(client, destParts)
        : candidate.archiveType === "SEVEN_Z" ? await readSevenZListingRanged(destParts, read)
        : candidate.archiveType === "RAR" ? await readRarListingRanged(destParts, read)
        : null;
    }
    if (destEntries) {
      candidateEntries = destEntries;
    }
  }
  return candidateEntries;
 }
 /**
 * Classify a fingerprint comparison between two entry sets. "incomplete"
 * means at least one side is missing CRCs (e.g. an empty file → CRC32 of
 * zero-length data → null) and the comparison CANNOT be used to confirm or
 * refute a match — callers must fall back to name+size confidence rather
 * than treating this as a mismatch.
 */
 function compareFingerprints(a: FileEntry[], b: FileEntry[]): "match" | "mismatch" | "incomplete" {
  const fa = crcFingerprint(a);
  const fb = crcFingerprint(b);
  if (!fa.complete || !fb.complete) return "incomplete";
  return fingerprintsMatch(a, b) ? "match" : "mismatch";
 }
 export async function tryProvenanceBackfill(
  args: BackfillArgs,
 ): Promise<{ backfilled: boolean; confidence?: "fingerprint" | "name-size" }> {
  const candidates = await findPlaceholderCandidates(args.destChannelId, args.fileName, args.fileSize);
  if (candidates.length === 0) return { backfilled: false };
  let scannedEntries: FileEntry[] | null = await readScannedListingRanged(
    args.archiveType, args.client, args.scannedParts,
  );
  // Cheap ranged read failed — fall back to a size-capped full download so the
  // listing still gets indexed. Only worth it when the candidate lacks a listing.
  if (!scannedEntries && candidates.some((c) => c.fileCount === 0)) {
    const totalSize = args.scannedParts.reduce((s, p) => s + p.fileSize, 0n);
    scannedEntries = await fullDownloadListing({
      client: args.client, parts: args.scannedParts, archiveType: args.archiveType,
      totalSize, fileName: args.fileName,
    });
  }
  let chosen = candidates[0];
  let confidence: "fingerprint" | "name-size" = "name-size";
  if (candidates.length > 1) {
    // Multiple placeholder packages share this name+size. Try to
    // disambiguate by fingerprint (ZIP only); if we can't uniquely resolve
    // it, notify instead of guessing which one is the real match.
    if (args.archiveType === "ZIP" && scannedEntries) {
      const matches: PlaceholderCandidate[] = [];
      // Candidates NOT ruled out as a definite (both-complete) mismatch —
      // used as the name+size fallback pool when the fingerprint can't
      // confirm a match (e.g. incomplete CRCs on either side).
      const nonMismatches: PlaceholderCandidate[] = [];
      for (const c of candidates) {
        const candidateEntries = await resolveCandidateFingerprintEntries(args.client, c);
        const comparison = compareFingerprints(scannedEntries, candidateEntries);
        if (comparison === "match") {
          matches.push(c);
          nonMismatches.push(c);
        } else if (comparison === "incomplete") {
          nonMismatches.push(c);
        }
        // comparison === "mismatch": both sides complete and differ — excluded.
      }
      if (matches.length === 1) {
        chosen = matches[0];
        confidence = "fingerprint";
      } else if (matches.length === 0 && nonMismatches.length === 1) {
        // Fingerprint couldn't confirm (incomplete CRCs), but exactly one
        // candidate wasn't ruled out as a definite mismatch — fall back to
        // name+size confidence rather than treating this as unresolved.
        chosen = nonMismatches[0];
        confidence = "name-size";
      } else {
        await db.systemNotification.create({
          data: {
            type: "INTEGRITY_AUDIT",
            severity: "WARNING",
            title: `Ambiguous provenance match: ${args.fileName}`,
            message: `${candidates.length} placeholder packages share this name+size and the fingerprint did not uniquely disambiguate. No provenance was backfilled.`,
            context: { fileName: args.fileName, candidateIds: candidates.map((c) => c.id) },
          },
        });
        return { backfilled: false };
      }
    } else {
      // Can't disambiguate without a fingerprint — notify, don't guess.
      await db.systemNotification.create({
        data: {
          type: "INTEGRITY_AUDIT",
          severity: "WARNING",
          title: `Ambiguous provenance match: ${args.fileName}`,
          message: `${candidates.length} placeholder packages share this name+size (archive type ${args.archiveType} — no cheap fingerprint). No provenance was backfilled.`,
          context: { fileName: args.fileName, candidateIds: candidates.map((c) => c.id) },
        },
      });
      return { backfilled: false };
    }
  } else if (scannedEntries) {
    const candidateEntries = await resolveCandidateFingerprintEntries(args.client, chosen);
    const comparison = compareFingerprints(scannedEntries, candidateEntries);
    if (comparison === "match") {
      confidence = "fingerprint";
    } else if (comparison === "mismatch") {
      // Both sides' CRCs are complete and differ: NOT the same content
      // despite name+size. Do not backfill.
      log.info({ candidateId: chosen.id, fileName: args.fileName }, "fingerprint mismatch — not backfilling");
      return { backfilled: false };
    }
    // comparison === "incomplete": can't confirm or refute by fingerprint —
    // fall through and backfill on name+size confidence instead.
  }
  const ok = await backfillProvenance({
    packageId: chosen.id,
    destChannelId: args.destChannelId,
    sourceChannelId: args.scannedSourceChannelId,
    sourceMessageId: args.sourceMessageId,
    sourceTopicId: args.sourceTopicId,
    sourceCaption: args.sourceCaption,
    remoteUniqueId: args.remoteUniqueId,
    creator: args.creator,
    entries: chosen.fileCount === 0 && scannedEntries ? scannedEntries : undefined,
    previewData: args.previewData ?? undefined,
    previewMsgId: args.previewMsgId ?? undefined,
  });
  if (!ok) return { backfilled: false };
  if (confidence === "name-size") {
    // Lower-confidence backfill: no CRC fingerprint guard confirmed this
    // match. Record it as an auditable event so name+size-only backfills
    // can be reviewed after the fact.
    await db.systemNotification.create({
      data: {
        type: "INTEGRITY_AUDIT",
        severity: "INFO",
        title: `Provenance backfilled by name+size: ${args.fileName}`,
        message: `Package ${chosen.id} was matched to a scanned source message by file name and size only (no CRC fingerprint confirmation).`,
        context: {
          packageId: chosen.id,
          fileName: args.fileName,
          sourceChannelId: args.scannedSourceChannelId,
        },
      },
    });
  }
  log.info(
    { candidateId: chosen.id, fileName: args.fileName, confidence, source: args.scannedSourceChannelId },
    "provenance backfilled",
  );
  return { backfilled: true, confidence };
 }
@@ -63,7 +63,7 @@ export async function rebuildPackageDatabase(
    }
    const account = accounts[0];
-    const client = await createTdlibClient({
+    const { client } = await createTdlibClient({
      id: account.id,
      phone: account.phone,
    });
@@ -308,14 +308,15 @@ async function scanDestinationChannel(
    }>(client, {
      _: "searchChatMessages",
      chat_id: Number(chatId),
      // No topic context for a flat destination scan. TDLib 1.8.64+ replaced
      // `message_thread_id` / `saved_messages_topic_id` with a single
      // optional `topic_id`; for a flat scan we just omit it.
      query: "",
      from_message_id: currentFromId,
      offset: 0,
      limit: 100,
      filter: { _: "searchMessagesFilterDocument" },
      sender_id: null,
      message_thread_id: 0,
      saved_messages_topic_id: 0,
    });
    if (!result.messages || result.messages.length === 0) break;
@@ -63,7 +63,7 @@ export async function recoverIncompleteUploads(): Promise<void> {
  let client: Client | undefined;
  try {
-    client = await createTdlibClient({ id: account.id, phone: account.phone });
+    ({ client } = await createTdlibClient({ id: account.id, phone: account.phone }));
    // Load the chat list so TDLib can resolve chat IDs
    try {
@@ -78,18 +78,31 @@ export async function recoverIncompleteUploads(): Promise<void> {
    let resetCount = 0;
    let verifiedCount = 0;
    let unknownCount = 0;
    let wrongContentCount = 0;
    // Batch size for getMessages. TDLib accepts up to ~100 IDs per call.
    // Using 100 means 20k packages → ~200 round-trips instead of 20k.
    const BATCH_SIZE = 100;
    for (const [, channelPackages] of byChannel) {
-      for (const pkg of channelPackages) {
+      // Group packages by destChannelId (already done) — within each group,
-        const exists = await verifyMessageExists(
+      // process in batches via getMessages (plural).
      for (let i = 0; i < channelPackages.length; i += BATCH_SIZE) {
        const batch = channelPackages.slice(i, i + BATCH_SIZE);
        const batchResults = await verifyMessagesBatch(
          client,
          destChannel.telegramId,
-          pkg.destMessageId!
+          batch.map((p) => p.destMessageId!)
        );
-        if (exists) {
+        for (let j = 0; j < batch.length; j++) {
-          verifiedCount++;
+          const pkg = batch[j];
-        } else {
+          const result = batchResults[j];
          if (result.state === "exists") {
            verifiedCount++;
          } else if (result.state === "deleted") {
          log.warn(
            {
              packageId: pkg.id,
@@ -100,21 +113,50 @@ export async function recoverIncompleteUploads(): Promise<void> {
          );
          await resetPackageDestination(pkg.id);
          resetCount++;
        } else if (result.state === "wrong-content") {
          // The message exists but isn't a document anymore (got cleared /
          // replaced). Treat as missing so we re-upload.
          log.warn(
            {
              packageId: pkg.id,
              fileName: pkg.fileName,
              destMessageId: Number(pkg.destMessageId),
              contentType: result.contentType,
            },
            "Destination message is not a document, resetting package for re-upload"
          );
          await resetPackageDestination(pkg.id);
          wrongContentCount++;
        } else {
          // Unknown — TDLib couldn't tell us. Don't reset, but DO count this
          // so the summary line shows recovery wasn't 100% successful.
          unknownCount++;
          log.warn(
            {
              packageId: pkg.id,
              fileName: pkg.fileName,
              destMessageId: Number(pkg.destMessageId),
              reason: result.reason.slice(0, 200),
            },
            "Could not verify destination message — will retry on next startup"
          );
        }
        }
      }
    }
-    if (resetCount > 0) {
+    log.info(
-      log.info(
+      {
-        { resetCount, verifiedCount, totalChecked: packages.length },
+        verifiedCount,
-        "Upload recovery complete — packages reset for re-processing"
+        resetCount,
-      );
+        wrongContentCount,
-    } else {
+        unknownCount,
-      log.info(
+        totalChecked: packages.length,
-        { verifiedCount, totalChecked: packages.length },
+      },
-        "Upload recovery complete — all destination messages verified"
+      unknownCount === 0
-      );
+        ? "Upload recovery complete"
-    }
+        : "Upload recovery complete — some packages could not be verified, will retry next startup"
    );
  } catch (err) {
    log.error({ err }, "Upload recovery failed (non-fatal, will retry next startup)");
  } finally {
@@ -124,15 +166,77 @@ export async function recoverIncompleteUploads(): Promise<void> {
  }
 }
 type VerifyResult =
  | { state: "exists" }
  | { state: "deleted" }
  | { state: "wrong-content"; contentType: string }
  | { state: "unknown"; reason: string };
 /**
- * Check whether a message exists in a Telegram chat.
+ * Batch version of verifyMessageExists. Calls TDLib's getMessages (plural)
- * Returns false if the message was deleted or never existed.
+ * with up to ~100 message IDs at once. Returns one VerifyResult per input
 * ID, in input order. Missing messages come back as null in TDLib's response
 * — translated to {state: "deleted"} here.
 *
 * Falls back to per-message verification on any error so that one bad batch
 * doesn't lose all verification for that chunk.
 */
 async function verifyMessagesBatch(
  client: Client,
  chatTelegramId: bigint,
  messageIds: bigint[]
 ): Promise<VerifyResult[]> {
  try {
    // eslint-disable-next-line @typescript-eslint/no-explicit-any
    const result = (await withFloodWait(
      () =>
        client.invoke({
          _: "getMessages",
          chat_id: Number(chatTelegramId),
          message_ids: messageIds.map((id) => Number(id)),
        }),
      "getMessages:verify"
    )) as { messages?: (null | { content?: { _: string } })[] };
    const messages = result.messages ?? [];
    return messageIds.map((_id, i) => {
      const m = messages[i];
      if (!m || !m.content) return { state: "deleted" };
      if (m.content._ !== "messageDocument") {
        return { state: "wrong-content", contentType: String(m.content._) };
      }
      return { state: "exists" };
    });
  } catch (err) {
    // If the whole batch errors out, fall back to per-message verification.
    log.warn(
      { err, batchSize: messageIds.length, chatTelegramId: chatTelegramId.toString() },
      "getMessages batch failed, falling back to per-message verification"
    );
    const out: VerifyResult[] = [];
    for (const id of messageIds) {
      out.push(await verifyMessageExists(client, chatTelegramId, id));
    }
    return out;
  }
 }
 /**
 * Check whether a message exists in a Telegram chat and is the document we
 * uploaded. Returns a discriminated result instead of a bare boolean so the
 * caller can distinguish "definitely gone" (reset) from "couldn't reach TG"
 * (leave alone, try again next startup).
 *
 * Previous version conflated all non-404 errors with "exists", which masked
 * recovery completely when TDLib had a degraded connection — the worker
 * would log "all destination messages verified" even though it had answered
 * questions it couldn't actually answer.
 */
 async function verifyMessageExists(
  client: Client,
  chatTelegramId: bigint,
  messageId: bigint
-): Promise<boolean> {
+): Promise<VerifyResult> {
  try {
    const result = await withFloodWait(
      () =>
@@ -144,44 +248,37 @@ async function verifyMessageExists(
      "getMessage:verify"
    );
    // TDLib returns the message object if it exists.
    // A deleted message may return with content type "messageChatDeleteMessage"
    // or the call may throw. Check that we got a real message with content.
    // eslint-disable-next-line @typescript-eslint/no-explicit-any
    const msg = result as any;
    if (!msg || !msg.content) {
-      return false;
+      return { state: "deleted" };
    }
    // Check that the message has document content (our uploads are documents)
    // A message that exists but has no document content was likely cleared/replaced
    if (msg.content._ !== "messageDocument") {
-      log.debug(
+      return { state: "wrong-content", contentType: String(msg.content._) };
        {
          messageId: Number(messageId),
          contentType: msg.content._,
        },
        "Destination message exists but is not a document"
      );
      return false;
    }
-    return true;
+    return { state: "exists" };
  } catch (err) {
-    // TDLib throws "Message not found" (error code 404) for deleted messages
+    const errMessage = err instanceof Error ? err.message : String(err);
    const message = err instanceof Error ? err.message : String(err);
    const code = (err as { code?: number })?.code;
-    if (code === 404 || message.includes("not found") || message.includes("Not Found")) {
+    // Hard "the message is definitely gone" signals from TDLib:
-      return false;
+    //   - HTTP 404
    //   - "Message not found" / "MESSAGE_ID_INVALID" error strings
    const lower = errMessage.toLowerCase();
    if (
      code === 404 ||
      lower.includes("message not found") ||
      lower.includes("message_id_invalid") ||
      lower.includes("messageidinvalid") ||
      lower.includes("not found")
    ) {
      return { state: "deleted" };
    }
-    // For other errors (network issues, etc.), assume the message exists
+    // Everything else (network, connection, TDLib internal) is genuinely
-    // to avoid incorrectly resetting packages due to transient failures
+    // unknown — do NOT claim "verified".
-    log.warn(
+    return { state: "unknown", reason: errMessage };
      { err, messageId: Number(messageId) },
      "Could not verify message (assuming it exists)"
    );
    return true;
  }
 }
@@ -1,6 +1,6 @@
 import { config } from "./util/config.js";
 import { childLogger } from "./util/logger.js";
-import { withTdlibMutex } from "./util/mutex.js";
+import { withTdlibMutex, forceReleaseMutex } from "./util/mutex.js";
 import { getActiveAccounts, getPendingAccounts } from "./db/queries.js";
 import { runWorkerForAccount, authenticateAccount } from "./worker.js";
 import { runIntegrityAudit } from "./audit.js";
@@ -19,13 +19,19 @@ let activeCyclePromise: Promise<void> | null = null;
 */
 const CYCLE_TIMEOUT_MS = (parseInt(process.env.WORKER_CYCLE_TIMEOUT_MINUTES ?? "240", 10)) * 60 * 1000;
 /** Read-only access to the current cycle counter for code that needs to
 *  apply per-cycle modulo logic (e.g. the cold-channel backoff). */
 export function getCurrentCycle(): number {
  return cycleCount;
 }
 /**
 * Run one ingestion cycle:
 * 1. Authenticate any PENDING accounts (triggers SMS code flow + auto-fetch channels)
 * 2. Process all active AUTHENTICATED accounts for ingestion
 *
- * All TDLib operations are wrapped in the mutex to ensure only one client
+ * Each account's TDLib operations are wrapped in a per-key mutex so different
- * runs at a time (also shared with the fetch listener for on-demand requests).
+ * accounts run concurrently while the same account is still serialized.
 *
 * The cycle has a configurable timeout (WORKER_CYCLE_TIMEOUT_MINUTES, default 4h).
 * Once the timeout elapses, no new accounts will be started but any in-progress
@@ -55,7 +61,7 @@ async function runCycle(): Promise<void> {
          log.warn("Cycle timeout reached during authentication phase, stopping");
          break;
        }
-        await withTdlibMutex(`auth:${account.phone}`, () =>
+        await withTdlibMutex(account.phone, `auth:${account.phone}`, () =>
          authenticateAccount(account)
        );
      }
@@ -71,17 +77,38 @@ async function runCycle(): Promise<void> {
    log.info({ accountCount: accounts.length }, "Processing accounts");
-    for (const account of accounts) {
+    const results = await Promise.allSettled(
-      if (Date.now() - cycleStart > CYCLE_TIMEOUT_MS) {
+      accounts.map((account) => {
-        log.warn(
+        let timer: ReturnType<typeof setTimeout>;
-          { elapsed: Math.round((Date.now() - cycleStart) / 60_000), timeoutMinutes: CYCLE_TIMEOUT_MS / 60_000 },
+        return Promise.race([
-          "Cycle timeout reached, skipping remaining accounts"
+          withTdlibMutex(account.phone, `ingest:${account.phone}`, () =>
            runWorkerForAccount(account)
          ),
          new Promise<never>((_, reject) => {
            timer = setTimeout(
              () => reject(new Error(`Account ${account.phone} ingestion timed out after ${CYCLE_TIMEOUT_MS / 60_000}min`)),
              CYCLE_TIMEOUT_MS
            );
          }),
        ]).finally(() => clearTimeout(timer));
      })
    );
    for (let i = 0; i < results.length; i++) {
      if (results[i].status === "rejected") {
        const reason = (results[i] as PromiseRejectedResult).reason;
        log.error(
          { phone: accounts[i].phone, err: reason },
          "Account ingestion failed"
        );
-        break;
+        // If the cycle timed out, force-release the mutex so the next cycle
        // (or other operations like fetch-channels) can proceed immediately
        // instead of waiting 30 minutes for the mutex timeout.
        const errMsg = reason instanceof Error ? reason.message : String(reason);
        if (errMsg.includes("timed out") || errMsg.includes("mutex wait timeout")) {
          forceReleaseMutex(accounts[i].phone);
        }
      }
      await withTdlibMutex(`ingest:${account.phone}`, () =>
        runWorkerForAccount(account)
      );
    }
    log.info(
@@ -5,6 +5,36 @@ import { withFloodWait } from "../util/retry.js";
 const log = childLogger("chats");
 /**
 * Collect chat folder IDs to widen the loadChats sweep across all folder
 * chat lists. In TDLib 1.8.64+ there's no synchronous getChatFolders call —
 * the folder list arrives via updateChatFolders. We listen for it briefly
 * (200ms) and fall back to an empty list if nothing arrives; chats inside
 * folders are still reachable via chatListMain so this only loses some
 * preemptive cache warming.
 */
 async function collectFolderIds(
  client: Client
 ): Promise<{ _: "chatListFolder"; chat_folder_id: number }[]> {
  return new Promise((resolve) => {
    const ids: number[] = [];
    // eslint-disable-next-line @typescript-eslint/no-explicit-any
    const handler = (update: any) => {
      if (update?._ === "updateChatFolders") {
        const folders = update.chat_folders as { id: number }[] | undefined;
        if (folders) {
          for (const f of folders) ids.push(f.id);
        }
      }
    };
    client.on("update", handler);
    setTimeout(() => {
      client.off("update", handler);
      resolve(ids.map((id) => ({ _: "chatListFolder" as const, chat_folder_id: id })));
    }, 200);
  });
 }
 export interface TelegramChatInfo {
  chatId: bigint;
  title: string;
@@ -37,21 +67,16 @@ export async function getAccountChats(
  // First, load all chats into TDLib's cache using loadChats (the proper API).
  // loadChats returns 404 when all chats have been loaded.
  // Then use getChats to retrieve the IDs for enrichment.
-  // Load from main, archive, AND chat folders to cover all chat types.
+  //
-  const folderLists: { _: "chatListFolder"; chat_folder_id: number }[] = [];
+  // Folder-specific loading (chatListFolder) was removed in TDLib 1.8.64+ —
-  try {
+  // getChatFolders (plural) is no longer a callable method, only the
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  // updateChatFolders event. The chats inside folders are still reachable
-    const folders = (await client.invoke({ _: "getChatFolders" })) as any;
+  // via chatListMain so this isn't a functional regression.
-    if (folders?.chat_folders) {
+  const folderLists: { _: "chatListFolder"; chat_folder_id: number }[] =
-      for (const f of folders.chat_folders) {
+    await collectFolderIds(client);
        folderLists.push({ _: "chatListFolder", chat_folder_id: f.id });
      }
    }
  } catch {
    // getChatFolders may not be available in older TDLib versions
  }
-  const chatLists: Record<string, unknown>[] = [
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
  const chatLists: any[] = [
    { _: "chatListMain" },
    { _: "chatListArchive" },
    ...folderLists,
@@ -282,3 +307,63 @@ export async function searchPublicChat(
 function sleep(ms: number): Promise<void> {
  return new Promise((resolve) => setTimeout(resolve, ms));
 }
 /**
 * Return the chat's server-side last_message.id from TDLib's local cache.
 * Used by the channel-scan-skip guard to short-circuit a paginated
 * searchChatMessages when nothing has changed since our watermark.
 *
 * Returns null when the chat has no last_message (empty channel) or the
 * call fails — callers must treat null as "unknown" and run the scan.
 */
 export async function getChannelLastMessageId(
  client: Client,
  chatId: bigint
 ): Promise<bigint | null> {
  try {
    // eslint-disable-next-line @typescript-eslint/no-explicit-any
    const chat = (await client.invoke({
      _: "getChat",
      chat_id: Number(chatId),
    })) as { last_message?: { id?: number } };
    const id = chat.last_message?.id;
    return id ? BigInt(id) : null;
  } catch (err) {
    log.debug({ err, chatId: chatId.toString() }, "getChannelLastMessageId failed");
    return null;
  }
 }
 /**
 * Return the forum topic's last_message_id from TDLib. Same purpose as
 * getChannelLastMessageId but scoped to a single topic in a forum
 * supergroup. TDLib's `getForumTopic` returns a `forumTopic` whose `info`
 * field contains the last_message_id.
 *
 * Returns null on failure or empty topic — caller treats as "unknown".
 */
 export async function getForumTopicLastMessageId(
  client: Client,
  chatId: bigint,
  topicId: bigint
 ): Promise<bigint | null> {
  try {
    // TDLib 1.8.64 uses `forum_topic_id` (renamed from `message_thread_id`
    // in the request) — consistent with the rest of the forum-topic API
    // surface in this version.
    // eslint-disable-next-line @typescript-eslint/no-explicit-any
    const topic = (await client.invoke({
      _: "getForumTopic",
      chat_id: Number(chatId),
      forum_topic_id: Number(topicId),
    })) as { last_message?: { id?: number }; info?: { last_message_id?: number } };
    const id = topic.last_message?.id ?? topic.info?.last_message_id;
    return id ? BigInt(id) : null;
  } catch (err) {
    log.debug(
      { err, chatId: chatId.toString(), topicId: topicId.toString() },
      "getForumTopicLastMessageId failed"
    );
    return null;
  }
 }
@@ -6,6 +6,7 @@ import { childLogger } from "../util/logger.js";
 import {
  updateAccountAuthState,
  getAccountAuthCode,
  updateAccountPremiumStatus,
 } from "../db/queries.js";
 const log = childLogger("tdlib-client");
@@ -27,7 +28,7 @@ interface AccountConfig {
 */
 export async function createTdlibClient(
  account: AccountConfig
-): Promise<Client> {
+): Promise<{ client: Client; isPremium: boolean }> {
  const dbPath = path.join(config.tdlibStateDir, account.id);
  const client = createClient({
@@ -78,7 +79,30 @@ export async function createTdlibClient(
    await updateAccountAuthState(account.id, "AUTHENTICATED");
    log.info({ accountId: account.id }, "TDLib client authenticated");
-    return client;
+
    let isPremium = false;
    try {
      const me = await client.invoke({ _: "getMe" }) as { is_premium?: boolean };
      isPremium = me.is_premium ?? false;
      await updateAccountPremiumStatus(account.id, isPremium);
      log.info({ accountId: account.id, isPremium }, "Account Premium status detected");
    } catch (err) {
      log.warn({ err, accountId: account.id }, "Could not detect Premium status, defaulting to false");
    }
    client.on("update", (update: unknown) => {
      const u = update as { _?: string; is_upload?: boolean };
      if (u?._ === "updateSpeedLimitNotification") {
        log.warn(
          { accountId: account.id, isUpload: u.is_upload },
          u.is_upload
            ? "Upload speed limited by Telegram (account is not Premium)"
            : "Download speed limited by Telegram (account is not Premium)"
        );
      }
    });
    return { client, isPremium };
  } catch (err) {
    log.error({ err, accountId: account.id }, "TDLib authentication failed");
    await updateAccountAuthState(account.id, "EXPIRED");
@@ -39,7 +39,15 @@ interface TdMessage {
  id: number;
  date: number;
  media_album_id?: string;
  // TDLib 1.8.50 exposed `reply_to_message_id` directly on the message.
  // 1.8.64+ replaced it with a tagged-union `reply_to: MessageReplyTo`.
  // Read both for resilience across versions.
  reply_to_message_id?: number;
  reply_to?: {
    _: string;
    chat_id?: number;
    message_id?: number;
  };
  content: {
    _: string;
    document?: {
@@ -51,6 +59,10 @@ interface TdMessage {
          path?: string;
          is_downloading_completed?: boolean;
        };
        remote?: {
          /** Stable identifier across reposts of the same file content. */
          unique_id?: string;
        };
      };
    };
    photo?: {
@@ -62,6 +74,24 @@ interface TdMessage {
  };
 }
 /**
 * Pick the right "the message I'm replying to" ID across TDLib versions.
 *  - 1.8.50 and earlier expose it directly as `reply_to_message_id`.
 *  - 1.8.64+ expose `reply_to: MessageReplyTo` (tagged union); a reply to
 *    a regular message has `_: "messageReplyToMessage"` with `message_id`.
 *  - Story replies (`_: "messageReplyToStory"`) intentionally return null
 *    here — they aren't useful for our reply-chain grouping.
 */
 function extractReplyToMessageId(msg: TdMessage): bigint | undefined {
  if (msg.reply_to_message_id) {
    return BigInt(msg.reply_to_message_id);
  }
  if (msg.reply_to && msg.reply_to._ === "messageReplyToMessage" && msg.reply_to.message_id) {
    return BigInt(msg.reply_to.message_id);
  }
  return undefined;
 }
 interface TdFile {
  id: number;
  size: number;
@@ -79,6 +109,8 @@ export interface ChannelScanResult {
  archives: TelegramMessage[];
  photos: TelegramPhoto[];
  totalScanned: number;
  /** Highest message ID seen during scan (for watermark, even when no archives found). */
  maxScannedMessageId: bigint | null;
 }
 export type ScanProgressCallback = (messagesScanned: number) => void;
@@ -113,7 +145,10 @@ export async function invokeWithTimeout<T>(
          }
        }, timeoutMs);
-        (client.invoke(request) as Promise<T>)
+        // The tdl 8.1+ types are very strict about the literal `_` field;
        // our generic wrapper passes arbitrary requests, so cast through any.
        // eslint-disable-next-line @typescript-eslint/no-explicit-any
        (client.invoke(request as any) as Promise<T>)
          .then((result) => {
            if (!settled) {
              settled = true;
@@ -158,6 +193,7 @@ export async function getChannelMessages(
  const archives: TelegramMessage[] = [];
  const photos: TelegramPhoto[] = [];
  const boundary = lastProcessedMessageId ? Number(lastProcessedMessageId) : null;
  let maxScannedMessageId: bigint | null = null;
  // Open the chat so TDLib can access it
  try {
@@ -192,18 +228,26 @@ export async function getChannelMessages(
      const result = await invokeWithTimeout<{ messages: TdMessage[]; total_count?: number }>(client, {
        _: "searchChatMessages",
        chat_id: Number(chatId),
        // No topic_id for a flat (non-forum) channel scan. TDLib 1.8.64+
        // dropped the top-level `message_thread_id: 0` we used to pass; the
        // type-narrow now is "omit the field entirely if not in a topic".
        query: "",
        from_message_id: fromMessageId,
        offset: 0,
        limit: Math.min(limit, 100),
        filter,
        message_thread_id: 0,
      });
      if (!result.messages || result.messages.length === 0) break;
      totalScanned += result.messages.length;
      // Track highest message ID (first message in batch = newest, since results are newest-first)
      const batchMaxId = BigInt(result.messages[0].id);
      if (maxScannedMessageId === null || batchMaxId > maxScannedMessageId) {
        maxScannedMessageId = batchMaxId;
      }
      for (const msg of result.messages) {
        // Check for archive documents
        const doc = msg.content?.document;
@@ -217,8 +261,9 @@ export async function getChannelMessages(
            fileSize: BigInt(doc.document.size),
            date: new Date(msg.date * 1000),
            mediaAlbumId: msg.media_album_id && msg.media_album_id !== "0" ? msg.media_album_id : undefined,
-            replyToMessageId: msg.reply_to_message_id ? BigInt(msg.reply_to_message_id) : undefined,
+            replyToMessageId: extractReplyToMessageId(msg),
            caption: msg.content?.caption?.text || undefined,
            remoteUniqueId: doc.document.remote?.unique_id || undefined,
          });
          continue;
        }
@@ -246,6 +291,11 @@ export async function getChannelMessages(
      fromMessageId = result.messages[result.messages.length - 1].id;
      if (result.messages.length < Math.min(limit, 100)) break;
      // Early exit: searchChatMessages returns newest-first. Once the oldest
      // message on this page is at or below the boundary, all remaining pages
      // are even older — no new messages exist, stop scanning immediately.
      if (boundary && fromMessageId <= boundary) break;
      await sleep(config.apiDelayMs);
    }
  }
@@ -266,6 +316,7 @@ export async function getChannelMessages(
    archives: archives.reverse(),
    photos: photos.reverse(),
    totalScanned,
    maxScannedMessageId,
  };
 }
@@ -0,0 +1,57 @@
 import { open } from "fs/promises";
 import type { Client } from "tdl";
 import { childLogger } from "../util/logger.js";
 import { withFloodWait } from "../util/retry.js";
 const log = childLogger("range-download");
 const RANGE_TIMEOUT_MS = 120_000;
 // NOTE (from Task 4 spike): TDLib writes the requested region into
 // file.local.path at its ABSOLUTE file offset; file.local.downloaded_prefix_size
 // counts contiguous bytes from download_offset. We request a 1KB-aligned offset
 // so downloaded_prefix_size covers our whole [offset, offset+limit) window.
 // This absolute-offset assumption is PENDING LIVE VERIFICATION ON DEPLOY —
 // the authenticated TDLib session could not be spiked in this environment.
 export async function downloadFileRange(
  client: Client,
  fileId: string,
  offset: number,
  limit: number,
  expectedSize: bigint,
 ): Promise<Buffer> {
  const numericId = parseInt(fileId, 10);
  const alignedOffset = Math.max(0, offset - (offset % 1024));
  const alignedLimit = limit + (offset - alignedOffset);
  const file = await withFloodWait(
    () =>
      new Promise<{ local: { path: string; download_offset: number; downloaded_prefix_size: number } }>(
        (resolve, reject) => {
          const timer = setTimeout(() => reject(new Error(`Range download timed out for ${fileId}`)), RANGE_TIMEOUT_MS);
          client
            .invoke({
              _: "downloadFile",
              file_id: numericId,
              priority: 1,
              offset: alignedOffset,
              limit: alignedLimit,
              synchronous: true,
            } as never)
            .then((f) => { clearTimeout(timer); resolve(f as never); })
            .catch((e) => { clearTimeout(timer); reject(e); });
        },
      ),
    `downloadFileRange:${fileId}`,
  );
  const start = offset;
  const fh = await open(file.local.path, "r");
  try {
    const buf = Buffer.alloc(limit);
    const { bytesRead } = await fh.read(buf, 0, limit, start);
    log.debug({ fileId, offset, limit, bytesRead }, "range read");
    return bytesRead < limit ? buf.subarray(0, bytesRead) : buf;
  } finally {
    await fh.close();
  }
 }
@@ -64,7 +64,11 @@ export async function getForumTopicList(
  const topics: ForumTopic[] = [];
  let offsetDate = 0;
  let offsetMessageId = 0;
-  let offsetMessageThreadId = 0;
+  // TDLib 1.8.64+ renamed `offset_message_thread_id` → `offset_forum_topic_id`
  // in the getForumTopics request, and `next_offset_message_thread_id` →
  // `next_offset_forum_topic_id` in the response. Individual topic infos
  // also moved from `info.message_thread_id` → `info.forum_topic_id`.
  let offsetForumTopicId = 0;
  let pageCount = 0;
  // eslint-disable-next-line no-constant-condition
@@ -80,12 +84,16 @@ export async function getForumTopicList(
    const prevOffsetDate = offsetDate;
    const prevOffsetMessageId = offsetMessageId;
-    const prevOffsetMessageThreadId = offsetMessageThreadId;
+    const prevOffsetForumTopicId = offsetForumTopicId;
    const result = await invokeWithTimeout<{
      topics?: {
        info?: {
          // Both names — 1.8.50 used the first, 1.8.64+ uses the second.
          // Read both so a future TDLib downgrade or transition build is
          // still handled.
          message_thread_id?: number;
          forum_topic_id?: number;
          name?: string;
          is_general?: boolean;
        };
@@ -93,45 +101,49 @@ export async function getForumTopicList(
      next_offset_date?: number;
      next_offset_message_id?: number;
      next_offset_message_thread_id?: number;
      next_offset_forum_topic_id?: number;
    }>(client, {
      _: "getForumTopics",
      chat_id: Number(chatId),
      query: "",
      offset_date: offsetDate,
      offset_message_id: offsetMessageId,
-      offset_message_thread_id: offsetMessageThreadId,
+      offset_forum_topic_id: offsetForumTopicId,
      limit: 100,
    });
    if (!result.topics || result.topics.length === 0) break;
    for (const t of result.topics) {
-      if (!t.info?.message_thread_id) continue;
+      const topicId = t.info?.forum_topic_id ?? t.info?.message_thread_id;
      if (!topicId) continue;
      topics.push({
-        topicId: BigInt(t.info.message_thread_id),
+        topicId: BigInt(topicId),
-        name: t.info.is_general ? "General" : (t.info.name ?? "Unnamed"),
+        name: t.info?.is_general ? "General" : (t.info?.name ?? "Unnamed"),
      });
    }
    // Check if there are more pages
    const nextForumTopicId =
      result.next_offset_forum_topic_id ?? result.next_offset_message_thread_id;
    if (
      !result.next_offset_date &&
      !result.next_offset_message_id &&
-      !result.next_offset_message_thread_id
+      !nextForumTopicId
    ) {
      break;
    }
    offsetDate = result.next_offset_date ?? 0;
    offsetMessageId = result.next_offset_message_id ?? 0;
-    offsetMessageThreadId = result.next_offset_message_thread_id ?? 0;
+    offsetForumTopicId = nextForumTopicId ?? 0;
    // Stuck detection: if offsets didn't advance, break
    if (
      offsetDate === prevOffsetDate &&
      offsetMessageId === prevOffsetMessageId &&
-      offsetMessageThreadId === prevOffsetMessageThreadId
+      offsetForumTopicId === prevOffsetForumTopicId
    ) {
      log.warn(
        { chatId: chatId.toString(), topicCount: topics.length },
@@ -178,6 +190,7 @@ export async function getTopicMessages(
  const archives: TelegramMessage[] = [];
  const photos: TelegramPhoto[] = [];
  const boundary = lastProcessedMessageId ? Number(lastProcessedMessageId) : null;
  let maxScannedMessageId: bigint | null = null;
  let currentFromId = 0;
  let totalScanned = 0;
@@ -209,6 +222,7 @@ export async function getTopicMessages(
            document?: {
              id: number;
              size: number;
              remote?: { unique_id?: string };
            };
          };
          photo?: {
@@ -225,20 +239,32 @@ export async function getTopicMessages(
    }>(client, {
      _: "searchChatMessages",
      chat_id: Number(chatId),
      // TDLib 1.8.64+ replaced the top-level `message_thread_id` and
      // `saved_messages_topic_id` parameters with a single tagged-union
      // `topic_id: MessageTopic$Input`. For a forum topic, use the
      // messageTopicForum variant carrying the forum_topic_id.
      topic_id: {
        _: "messageTopicForum",
        forum_topic_id: Number(topicId),
      },
      query: "",
      message_thread_id: Number(topicId),
      from_message_id: currentFromId,
      offset: 0,
      limit: Math.min(limit, 100),
      filter: null,
      sender_id: null,
      saved_messages_topic_id: 0,
    });
    if (!result.messages || result.messages.length === 0) break;
    totalScanned += result.messages.length;
    // Track highest message ID (first message = newest, since results are newest-first)
    const batchMaxId = BigInt(result.messages[0].id);
    if (maxScannedMessageId === null || batchMaxId > maxScannedMessageId) {
      maxScannedMessageId = batchMaxId;
    }
    for (const msg of result.messages) {
      // Check for archive documents
      const doc = msg.content?.document;
@@ -250,6 +276,7 @@ export async function getTopicMessages(
          fileSize: BigInt(doc.document.size),
          date: new Date(msg.date * 1000),
          mediaAlbumId: msg.media_album_id && msg.media_album_id !== "0" ? msg.media_album_id : undefined,
          remoteUniqueId: doc.document.remote?.unique_id || undefined,
        });
        continue;
      }
@@ -302,6 +329,7 @@ export async function getTopicMessages(
    archives: archives.reverse(),
    photos: photos.reverse(),
    totalScanned,
    maxScannedMessageId,
  };
 }
@@ -7,6 +7,18 @@ import { withFloodWait, extractFloodWaitSeconds } from "../util/retry.js";
 const log = childLogger("upload");
 /**
 * Custom error class to distinguish upload stalls from other errors.
 * When consecutive stalls occur, the caller can use this signal to
 * recreate the TDLib client (whose event stream may have degraded).
 */
 export class UploadStallError extends Error {
  constructor(message: string) {
    super(message);
    this.name = "UploadStallError";
  }
 }
 export interface UploadResult {
  messageId: bigint;
  messageIds: bigint[];
@@ -107,14 +119,42 @@ async function sendWithRetry(
        continue;
      }
-      // Stall or timeout — retry with a cooldown
+      // Stall or timeout — fail fast and let the caller recreate the TDLib
      // client. Retrying on the same degraded event stream wastes ~15 min
      // per attempt because the underlying issue (missing send-success
      // events) is client-level, not transient. The set ends up in
      // SkippedPackage and the caller's watermark cap ensures it gets
      // retried next cycle on a fresh client.
      const errMsg = err instanceof Error ? err.message : "";
-      if ((errMsg.includes("stalled") || errMsg.includes("timed out")) && !isLastAttempt) {
+      if (errMsg.includes("stalled") || errMsg.includes("timed out")) {
        log.warn(
-          { fileName, attempt: attempt + 1, maxRetries: MAX_UPLOAD_RETRIES },
+          { fileName, attempt: attempt + 1 },
-          "Upload stalled/timed out — retrying"
+          "Upload stalled — failing fast so caller can recreate TDLib client"
        );
-        await sleep(10_000);
+        throw new UploadStallError(
          `Upload stalled for ${fileName}: ${errMsg}`
        );
      }
      // Transient Telegram server-side error (HTTP 5xx returned via
      // updateMessageSendFailed). These are NOT FLOOD_WAIT, NOT stalls — just
      // TG having a bad moment. They typically resolve on a short backoff, so
      // retry up to MAX_UPLOAD_RETRIES with linear backoff before giving up.
      const lowerMsg = errMsg.toLowerCase();
      const isTransientServerError =
        lowerMsg.includes("internal server error") ||
        lowerMsg.includes("internal error") ||
        lowerMsg.includes("server error") ||
        lowerMsg.includes("bad gateway") ||
        lowerMsg.includes("service unavailable") ||
        lowerMsg.includes("gateway timeout");
      if (isTransientServerError && !isLastAttempt) {
        const backoffMs = 15_000 * (attempt + 1) + Math.random() * 5_000;
        log.warn(
          { fileName, attempt: attempt + 1, maxRetries: MAX_UPLOAD_RETRIES, backoffMs: Math.round(backoffMs) },
          `Transient Telegram server error — retrying after backoff`
        );
        await sleep(backoffMs);
        continue;
      }
@@ -148,6 +188,12 @@ async function sendAndWaitForUpload(
    let lastProgressBytes = 0;
    let lastProgressTime = Date.now();
    // Events for our message can arrive before `sendMessage` resolves
    // (TDLib emits them while our .then() is still in the microtask queue).
    // Buffer them and replay once tempMsgId is known.
    let pendingSuccess: { oldMsgId: number; finalId: number } | null = null;
    let pendingFailure: { oldMsgId: number; errorMsg: string; code?: number } | null = null;
    // Timeout: 20 minutes per GB, minimum 15 minutes
    const timeoutMs = Math.max(
      15 * 60_000,
@@ -166,8 +212,10 @@ async function sendAndWaitForUpload(
      }
    }, timeoutMs);
-    // Stall detection: no progress for 5 minutes after upload started → reject
+    // Stall detection: no progress for 3 minutes after upload started → reject
-    const STALL_TIMEOUT_MS = 5 * 60_000;
+    // (reduced from 5min — once data is fully sent, confirmation should arrive quickly;
    //  a 3min silence strongly indicates a degraded TDLib event stream)
    const STALL_TIMEOUT_MS = 3 * 60_000;
    const stallChecker = setInterval(() => {
      if (settled || !uploadStarted) return;
      const stallMs = Date.now() - lastProgressTime;
@@ -182,6 +230,26 @@ async function sendAndWaitForUpload(
      }
    }, 30_000);
    const completeWithSuccess = (finalId: number) => {
      if (settled) return;
      settled = true;
      cleanup();
      log.info(
        { fileName, tempMsgId, finalMsgId: finalId },
        "Upload confirmed by Telegram"
      );
      resolve(BigInt(finalId));
    };
    const completeWithFailure = (errorMsg: string, code?: number) => {
      if (settled) return;
      settled = true;
      cleanup();
      const error = new Error(`Upload failed for ${fileName}: ${errorMsg}`);
      (error as Error & { code?: number }).code = code;
      reject(error);
    };
    // eslint-disable-next-line @typescript-eslint/no-explicit-any
    const handleUpdate = (update: any) => {
      // Track upload progress via updateFile events
@@ -212,33 +280,29 @@ async function sendAndWaitForUpload(
      // The money event: upload succeeded, we get the final server message ID
      if (update?._ === "updateMessageSendSucceeded") {
        const msg = update.message;
-        const oldMsgId = update.old_message_id;
+        const oldMsgId: number = update.old_message_id;
-        if (tempMsgId !== null && oldMsgId === tempMsgId) {
+        if (tempMsgId === null) {
-          if (!settled) {
+          // Race: event arrived before our .then() assigned tempMsgId.
-            settled = true;
+          // Buffer it and process once tempMsgId is known.
-            cleanup();
+          pendingSuccess = { oldMsgId, finalId: msg.id };
-            const finalId = BigInt(msg.id);
+          return;
-            log.info(
+        }
-              { fileName, tempMsgId, finalMsgId: Number(finalId) },
+        if (oldMsgId === tempMsgId) {
-              "Upload confirmed by Telegram"
+          completeWithSuccess(msg.id);
            );
            resolve(finalId);
          }
        }
      }
      // Upload failed
      if (update?._ === "updateMessageSendFailed") {
-        const oldMsgId = update.old_message_id;
+        const oldMsgId: number = update.old_message_id;
-        if (tempMsgId !== null && oldMsgId === tempMsgId) {
+        const errorMsg: string = update.error?.message ?? "Unknown upload error";
-          if (!settled) {
+        const code: number | undefined = update.error?.code;
-            settled = true;
+        if (tempMsgId === null) {
-            cleanup();
+          pendingFailure = { oldMsgId, errorMsg, code };
-            const errorMsg = update.error?.message ?? "Unknown upload error";
+          return;
-            const error = new Error(`Upload failed for ${fileName}: ${errorMsg}`);
+        }
-            (error as Error & { code?: number }).code = update.error?.code;
+        if (oldMsgId === tempMsgId) {
-            reject(error);
+          completeWithFailure(errorMsg, code);
          }
        }
      }
    };
@@ -282,6 +346,13 @@ async function sendAndWaitForUpload(
          { fileName, tempMsgId },
          "Message queued, waiting for upload confirmation"
        );
        // Replay any event that arrived before we knew tempMsgId
        if (pendingSuccess && pendingSuccess.oldMsgId === tempMsgId) {
          completeWithSuccess(pendingSuccess.finalId);
        } else if (pendingFailure && pendingFailure.oldMsgId === tempMsgId) {
          completeWithFailure(pendingFailure.errorMsg, pendingFailure.code);
        }
      })
      .catch((err) => {
        if (!settled) {
@@ -20,4 +20,26 @@ export const config = {
  apiDelayMs: 1000,
  /** Max retries for rate-limited requests */
  maxRetries: 5,
  /** After this many failed attempts on the same source message, the worker
   *  stops auto-retrying and lets the watermark advance past it. The user can
   *  manually retry via the UI to reset and try again. */
  maxSkipAttempts: parseInt(process.env.WORKER_MAX_SKIP_ATTEMPTS ?? "5", 10),
  /** Window in which a recent successful empty scan lets us skip the next
   *  scan entirely. Default 5 minutes. */
  skipRecentScanWindowMs: parseInt(
    process.env.WORKER_SKIP_RECENT_SCAN_WINDOW_MS ?? "300000",
    10
  ),
  /** After this many consecutive empty scans, a channel/topic enters
   *  backoff and is only scanned every Nth cycle. */
  emptyScanBackoffThreshold: parseInt(
    process.env.WORKER_EMPTY_SCAN_BACKOFF_THRESHOLD ?? "5",
    10
  ),
  /** While in backoff, scan only every Nth cycle. Default 5 = scan every
   *  fifth cycle = once every ~5 hours given the 60-min default interval. */
  emptyScanBackoffEveryNth: parseInt(
    process.env.WORKER_EMPTY_SCAN_BACKOFF_EVERY_NTH ?? "5",
    10
  ),
 } as const;
@@ -2,39 +2,66 @@ import { childLogger } from "./logger.js";
 const log = childLogger("mutex");
 let locked = false;
 let holder = "";
 const queue: Array<{ resolve: () => void; reject: (err: Error) => void; label: string }> = [];
 /**
 * Maximum time to wait for the TDLib mutex (ms).
 * If the mutex is not available within this time, the operation is rejected.
 * Default: 30 minutes (long enough for large downloads, short enough to detect hangs).
 */
 const MUTEX_WAIT_TIMEOUT_MS = 30 * 60 * 1000;
 const locks = new Map<string, boolean>();
 const holders = new Map<string, string>();
 const queues = new Map<
  string,
  Array<{ resolve: () => void; reject: (err: Error) => void; label: string }>
 >();
 /**
- * Ensures only one TDLib client runs at a time across the entire worker process.
+ * Force-release a stuck mutex.
- * Both the scheduler (auth, ingestion) and the fetch listener acquire this
+ * This should only be called when the holder is known to be stuck (e.g. after
- * before creating any TDLib client.
+ * a cycle timeout). It releases the lock and lets the next queued waiter proceed.
 */
 export function forceReleaseMutex(key: string): void {
  if (!locks.has(key)) return;
  const holder = holders.get(key);
  log.warn({ key, holder }, "Force-releasing stuck TDLib mutex");
  locks.delete(key);
  holders.delete(key);
  const next = queues.get(key)?.shift();
  if (next) {
    log.info({ key, next: next.label }, "TDLib mutex force-released to next waiter");
    next.resolve();
  } else {
    queues.delete(key);
    log.info({ key }, "TDLib mutex force-released (no waiters)");
  }
 }
 /**
 * Ensures only one TDLib operation runs at a time FOR THE SAME KEY.
 * Different keys run concurrently — this allows two accounts to ingest in parallel
 * while still preventing concurrent use of the same account's TDLib state dir.
 *
- * Includes a wait timeout to prevent indefinite blocking if the current holder hangs.
+ * key:   the account phone number for account-specific ops (auth, ingest),
 *        or 'global' for ops that don't belong to a specific account.
 * label: human-readable name for logging.
 */
 export async function withTdlibMutex<T>(
  key: string,
  label: string,
  fn: () => Promise<T>
 ): Promise<T> {
-  if (locked) {
+  if (locks.get(key)) {
-    log.info({ waiting: label, holder }, "Waiting for TDLib mutex");
+    log.info({ waiting: label, key, holder: holders.get(key) }, "Waiting for TDLib mutex");
    await new Promise<void>((resolve, reject) => {
      const timer = setTimeout(() => {
-        const idx = queue.indexOf(entry);
+        const q = queues.get(key) ?? [];
        const idx = q.indexOf(entry);
        if (idx !== -1) {
-          queue.splice(idx, 1);
+          q.splice(idx, 1);
-          reject(new Error(
+          reject(
-            `TDLib mutex wait timeout after ${MUTEX_WAIT_TIMEOUT_MS / 60_000}min ` +
+            new Error(
-            `(waiting: ${label}, holder: ${holder})`
+              `TDLib mutex wait timeout after ${MUTEX_WAIT_TIMEOUT_MS / 60_000}min ` +
-          ));
+                `(waiting: ${label}, key: ${key}, holder: ${holders.get(key)})`
            )
          );
        }
      }, MUTEX_WAIT_TIMEOUT_MS);
@@ -46,25 +73,28 @@ export async function withTdlibMutex<T>(
        reject,
        label,
      };
-      queue.push(entry);
+
      if (!queues.has(key)) queues.set(key, []);
      queues.get(key)!.push(entry);
    });
  }
-  locked = true;
+  locks.set(key, true);
-  holder = label;
+  holders.set(key, label);
-  log.debug({ label }, "TDLib mutex acquired");
+  log.debug({ key, label }, "TDLib mutex acquired");
  try {
    return await fn();
  } finally {
-    locked = false;
+    locks.delete(key);
-    holder = "";
+    holders.delete(key);
-    const next = queue.shift();
+    const next = queues.get(key)?.shift();
    if (next) {
-      log.debug({ next: next.label }, "TDLib mutex releasing to next waiter");
+      log.debug({ key, next: next.label }, "TDLib mutex releasing to next waiter");
      next.resolve();
    } else {
-      log.debug({ label }, "TDLib mutex released");
+      queues.delete(key);
      log.debug({ key, label }, "TDLib mutex released");
    }
  }
 }
--- a/Show More
+++ b/Show More
		`@@ -0,0 +1,2 @@`
							`0 3 * * * /backup.sh >> /proc/1/fd/1 2>&1`
							`0 4 * * 0 restic check >> /proc/1/fd/1 2>&1`
		`@@ -0,0 +1,2 @@`
							`-- AlterTable`
							`ALTER TABLE "telegram_accounts" ADD COLUMN "isPremium" BOOLEAN NOT NULL DEFAULT false;`