fix(worker): clamp RAR header re-read to 8MB to bound corrupt-archive reads

Add MAX_RAR_HEADER_BYTES constant to prevent unbounded ranged reads when a RAR block's HeaderSize is bogus. Real RAR block headers are far smaller; this guards against amplification attacks on corrupt/desynced archives. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
feat(worker): enable RAR ranged listing + format-aware destination reads
2026-09-21 13:31:42 +00:00 · 2026-07-27 12:34:00 +02:00 · 2026-07-27 12:06:46 +02:00 · 2026-07-27 12:03:14 +02:00 · 2026-07-27 11:58:33 +02:00 · 2026-07-27 11:55:59 +02:00
111 changed files with 16105 additions and 576 deletions
@@ -54,9 +54,24 @@ steps:
      password:
        from_secret: gitea_password

+  - name: build-backup
+    image: plugins/docker
+    depends_on: [clone]
+    settings:
+      repo: git.samagsteribbe.nl/admin/dragonsstash-backup
+      registry: git.samagsteribbe.nl
+      dockerfile: backup/Dockerfile
+      tags:
+        - latest
+        - "${DRONE_COMMIT_SHA:0:8}"
+      username:
+        from_secret: gitea_username
+      password:
+        from_secret: gitea_password
+
  - name: deploy
    image: alpine
-    depends_on: [build-app, build-worker, build-bot]
+    depends_on: [build-app, build-worker, build-bot, build-backup]
    environment:
      SSH_KEY:
        from_secret: ssh_key
@@ -36,3 +36,12 @@ TDLIB_STATE_DIR="/data/tdlib"
 WORKER_MAX_ZIP_SIZE_MB=4096
 MULTIPART_TIMEOUT_HOURS=0
 LOG_LEVEL="info"
+
+# Backup (NAS via SMB/CIFS + restic)
+NAS_HOST=""              # Synology NAS IP or hostname reachable from this host
+NAS_SHARE=""             # SMB share name, e.g. dragonsstash_backups
+NAS_USERNAME=""          # SMB user with read/write on the share
+NAS_PASSWORD=""          # SMB user password (avoid commas — they delimit cifs mount opts)
+RESTIC_PASSWORD=""       # generate with: openssl rand -base64 32
+KUMA_PUSH_URL=""         # optional: Uptime Kuma Push monitor URL; leave empty to disable alerting
+TZ="Etc/UTC"
@@ -54,3 +54,4 @@ src/generated
 # temp files
 nul
 tmpclaude-*
+.worktrees/
@@ -0,0 +1,13 @@
+FROM alpine:3.20
+
+# Note: use busybox's built-in crond (Alpine base), NOT the dcron package —
+# dcron's crond fails with "setpgid: Operation not permitted" in this runtime.
+RUN apk add --no-cache restic postgresql16-client curl tzdata tar bash
+
+COPY backup/backup.sh /backup.sh
+COPY backup/entrypoint.sh /entrypoint.sh
+COPY backup/crontab /etc/crontabs/root
+
+RUN chmod +x /backup.sh /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]
@@ -0,0 +1,32 @@
+#!/bin/bash
+set -euo pipefail
+
+report_failure() {
+  [ -n "${KUMA_PUSH_URL:-}" ] || return 0
+  curl -fsS "$KUMA_PUSH_URL" --get \
+    --data-urlencode "status=down" \
+    --data-urlencode "msg=$BASH_COMMAND failed" || true
+}
+trap report_failure ERR
+
+DUMP_FILE=/tmp/dragonsstash.dump
+TAR_FILE=/tmp/tdlib.tar.gz
+
+trap 'rm -f "$DUMP_FILE" "$TAR_FILE"' EXIT
+
+pg_dump -h dragonsstash-db -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Fc -f "$DUMP_FILE"
+
+# TDLib volumes are tarred live (best-effort, per design). A file changing
+# mid-read makes GNU tar exit 1 (warning) — that is expected here and must not
+# abort the backup. Only a genuine error (exit >= 2) is fatal.
+tar --warning=no-file-changed -czf "$TAR_FILE" -C /data tdlib-worker tdlib-bot \
+  || { rc=$?; [ "$rc" -le 1 ] || exit "$rc"; }
+
+restic backup "$DUMP_FILE" "$TAR_FILE"
+restic forget --keep-daily 14 --prune
+
+if [ -n "${KUMA_PUSH_URL:-}" ]; then
+  curl -fsS "$KUMA_PUSH_URL" --get \
+    --data-urlencode "status=up" \
+    --data-urlencode "msg=OK"
+fi
@@ -0,0 +1,2 @@
+0 3 * * * /backup.sh >> /proc/1/fd/1 2>&1
+0 4 * * 0 restic check >> /proc/1/fd/1 2>&1
@@ -0,0 +1,11 @@
+#!/bin/bash
+set -uo pipefail
+
+# Ensure the repo exists, but never crash-loop on it: a transient error reading
+# the repo (CIFS hiccup, stale lock) must not kill PID 1. `restic init` failing
+# because the repo already exists is expected and harmless here.
+if ! restic cat config >/dev/null 2>&1; then
+  restic init || echo "restic init skipped (repo already exists or temporarily unreachable)"
+fi
+
+exec crond -f -l 2
@@ -12,8 +12,8 @@
        "@prisma/client": "^7.4.0",
        "pg": "^8.18.0",
        "pino": "^9.6.0",
-        "prebuilt-tdlib": "^0.1008050.0",
-        "tdl": "^8.0.0"
+        "prebuilt-tdlib": "^0.1008064.0",
+        "tdl": "^8.1.0"
      },
      "devDependencies": {
        "@types/node": "^20",
@@ -566,9 +566,9 @@
      "license": "MIT"
    },
    "node_modules/@prebuilt-tdlib/darwin-arm64": {
-      "version": "0.1008050.0",
-      "resolved": "https://registry.npmjs.org/@prebuilt-tdlib/darwin-arm64/-/darwin-arm64-0.1008050.0.tgz",
-      "integrity": "sha512-XrWN7M1gfvnzOBRX0YdXVfhSxIDSs/ZJ16QJ0ILDKe+grOFl/cfl7lwB/hK/MlHC6Rev56f5X7xaWnjMh0vktQ==",
+      "version": "0.1008064.0",
+      "resolved": "https://registry.npmjs.org/@prebuilt-tdlib/darwin-arm64/-/darwin-arm64-0.1008064.0.tgz",
+      "integrity": "sha512-Oq5us+o0g68Jag74RIV3LdLkZxQxJMcOdrVbgmyE7Unk+WcifqTb/gZw1rS6BrW+2SX2LNeGY4zQqqBTNDr17Q==",
      "cpu": [
        "arm64"
      ],
@@ -579,9 +579,9 @@
      ]
    },
    "node_modules/@prebuilt-tdlib/darwin-x64": {
-      "version": "0.1008050.0",
-      "resolved": "https://registry.npmjs.org/@prebuilt-tdlib/darwin-x64/-/darwin-x64-0.1008050.0.tgz",
-      "integrity": "sha512-a1UfBW0lYx4tUy5viMPtsbqBfBncCAgDu3FPjljfYTHjP8wfkKFxpp5+8wdxhyqdy3QriWaipVtUXQgOeEWMJg==",
+      "version": "0.1008064.0",
+      "resolved": "https://registry.npmjs.org/@prebuilt-tdlib/darwin-x64/-/darwin-x64-0.1008064.0.tgz",
+      "integrity": "sha512-Pz11xjET2Y3uUJKxkWKBc0dmOtlykmBdZ9D6Ahh+EsoLDLIWHm7M91p6nZT396YZ4n2BL+FtDYK65Ae3LDIA5g==",
      "cpu": [
        "x64"
      ],
@@ -592,9 +592,22 @@
      ]
    },
    "node_modules/@prebuilt-tdlib/linux-arm64-glibc": {
-      "version": "0.1008050.0",
-      "resolved": "https://registry.npmjs.org/@prebuilt-tdlib/linux-arm64-glibc/-/linux-arm64-glibc-0.1008050.0.tgz",
-      "integrity": "sha512-HRGspdQYzaBkU+W2M8uY5OgOkmgfTkyHkTYan/dn7EE/38QdIFW0YTvmGrl3DoFV2PA+SeJQw0xqK8tMSyHKaA==",
+      "version": "0.1008064.0",
+      "resolved": "https://registry.npmjs.org/@prebuilt-tdlib/linux-arm64-glibc/-/linux-arm64-glibc-0.1008064.0.tgz",
+      "integrity": "sha512-1kML9+RCfTOTWLzxq2klCN962/XwYhd+SGd4BxOwcmvPniYDNRUXtgMi3qRyV/Flola8dchGFrqZJU4kNZNLuQ==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "0BSD",
+      "optional": true,
+      "os": [
+        "linux"
+      ]
+    },
+    "node_modules/@prebuilt-tdlib/linux-arm64-musl": {
+      "version": "0.1008064.0",
+      "resolved": "https://registry.npmjs.org/@prebuilt-tdlib/linux-arm64-musl/-/linux-arm64-musl-0.1008064.0.tgz",
+      "integrity": "sha512-tN9FJOR8VDfmOoTHMivAqBfQ/d9Bry9T/9cGSTcms3H4ORun/WO5U5zT8VqadAsqjuiQ8Y9HaUqqz65xBDtcgw==",
      "cpu": [
        "arm64"
      ],
@@ -605,9 +618,9 @@
      ]
    },
    "node_modules/@prebuilt-tdlib/linux-x64-glibc": {
-      "version": "0.1008050.0",
-      "resolved": "https://registry.npmjs.org/@prebuilt-tdlib/linux-x64-glibc/-/linux-x64-glibc-0.1008050.0.tgz",
-      "integrity": "sha512-Yf6ve3Dzxc66kV1cijFLn7EXKhPN5YHTjtJABEaCR5euetCI2wZp/1uBsXvyYTuFXqQbMfjO3xUCXUIBhLoChw==",
+      "version": "0.1008064.0",
+      "resolved": "https://registry.npmjs.org/@prebuilt-tdlib/linux-x64-glibc/-/linux-x64-glibc-0.1008064.0.tgz",
+      "integrity": "sha512-7fyCp2uk0BdeHKJ9PyQOCditC9vBXeeIjYPAKKBcrkum5bi1e9txy2g5kkGjqwUkN0ntIniS5QfHEyr17Idr9g==",
      "cpu": [
        "x64"
      ],
@@ -617,10 +630,30 @@
        "linux"
      ]
    },
+    "node_modules/@prebuilt-tdlib/linux-x64-musl": {
+      "version": "0.1008064.0",
+      "resolved": "https://registry.npmjs.org/@prebuilt-tdlib/linux-x64-musl/-/linux-x64-musl-0.1008064.0.tgz",
+      "integrity": "sha512-e2zRucrRrrK6M04iQWMfwtrts+VvVtyUwtTP1hF2g3a6jW+AHMzFoB9Wu8fWr+vuJflLIQ6sG9r3lI07Q8NenQ==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "0BSD",
+      "optional": true,
+      "os": [
+        "linux"
+      ]
+    },
+    "node_modules/@prebuilt-tdlib/types": {
+      "version": "0.1008064.0",
+      "resolved": "https://registry.npmjs.org/@prebuilt-tdlib/types/-/types-0.1008064.0.tgz",
+      "integrity": "sha512-eqr1+fiHZ+Gj4lwcITzMp6FwPg8UrxlxxaFjhiJRHL9BlbmD2QkCRHac4wW1Sx8Dzwzd7f+xO21Pgi7TBRSwmw==",
+      "license": "0BSD",
+      "optional": true
+    },
    "node_modules/@prebuilt-tdlib/win32-x64": {
-      "version": "0.1008050.0",
-      "resolved": "https://registry.npmjs.org/@prebuilt-tdlib/win32-x64/-/win32-x64-0.1008050.0.tgz",
-      "integrity": "sha512-4v8tU5bodMcLhzrWWXzIzqdHBIpq0wim+7sDmQWQIMy3kDeIzVtpuM+vQjxrGoeH9oWr2WXSRKuj93ld7G5NbQ==",
+      "version": "0.1008064.0",
+      "resolved": "https://registry.npmjs.org/@prebuilt-tdlib/win32-x64/-/win32-x64-0.1008064.0.tgz",
+      "integrity": "sha512-rkacZWexQw52/EUaLAmbsu2+P3C1/AtinlCjfiX07oQAEg3327BCEZqrcY0ER83D8+MMf2pfwMPCDJKytr4hcg==",
      "cpu": [
        "x64"
      ],
@@ -1669,16 +1702,19 @@
      }
    },
    "node_modules/prebuilt-tdlib": {
-      "version": "0.1008050.0",
-      "resolved": "https://registry.npmjs.org/prebuilt-tdlib/-/prebuilt-tdlib-0.1008050.0.tgz",
-      "integrity": "sha512-CfeQE1rG51d2iC6m72fzrbCW4mqI17ugil9pVurWHtfUJi1Fcn7zadpTzDoUl4oc1dEtKgM7S24DVP67gcl4SQ==",
+      "version": "0.1008064.0",
+      "resolved": "https://registry.npmjs.org/prebuilt-tdlib/-/prebuilt-tdlib-0.1008064.0.tgz",
+      "integrity": "sha512-jJLowKZoH4slXYrkTkKlEgyGsIGv61AWjDZcxxVxJYu21X3kmukGwbCpk4ML99cJp2CwRsD41GCEQBkKJAwCUg==",
      "license": "MIT",
      "optionalDependencies": {
-        "@prebuilt-tdlib/darwin-arm64": "0.1008050.0",
-        "@prebuilt-tdlib/darwin-x64": "0.1008050.0",
-        "@prebuilt-tdlib/linux-arm64-glibc": "0.1008050.0",
-        "@prebuilt-tdlib/linux-x64-glibc": "0.1008050.0",
-        "@prebuilt-tdlib/win32-x64": "0.1008050.0"
+        "@prebuilt-tdlib/darwin-arm64": "0.1008064.0",
+        "@prebuilt-tdlib/darwin-x64": "0.1008064.0",
+        "@prebuilt-tdlib/linux-arm64-glibc": "0.1008064.0",
+        "@prebuilt-tdlib/linux-arm64-musl": "0.1008064.0",
+        "@prebuilt-tdlib/linux-x64-glibc": "0.1008064.0",
+        "@prebuilt-tdlib/linux-x64-musl": "0.1008064.0",
+        "@prebuilt-tdlib/types": "0.1008064.0",
+        "@prebuilt-tdlib/win32-x64": "0.1008064.0"
      }
    },
    "node_modules/prisma": {
@@ -1971,13 +2007,13 @@
      "license": "MIT"
    },
    "node_modules/tdl": {
-      "version": "8.0.2",
-      "resolved": "https://registry.npmjs.org/tdl/-/tdl-8.0.2.tgz",
-      "integrity": "sha512-KYxlJ4eao7FUu91U1dCDkaHmK70JAyZ1KqitkKqpPC7rxAiXWhaYxddWvt84UxIYoWbgdd0B70FYJ4p/YqpFCA==",
+      "version": "8.1.0",
+      "resolved": "https://registry.npmjs.org/tdl/-/tdl-8.1.0.tgz",
+      "integrity": "sha512-idpw60gjJdiJALQg0+6UbxtJTMxVhzZAgCO6QzL81gqBYCkEFjm9zM9HwTTQGeOaAavw4yRHymR68yUUiCoKrA==",
      "hasInstallScript": true,
      "license": "MIT",
      "dependencies": {
-        "debug": "^4.4.0",
+        "debug": "^4.4.3",
        "node-addon-api": "^7.1.1",
        "node-gyp-build": "^4.8.4"
      },
@@ -13,8 +13,8 @@
    "@prisma/client": "^7.4.0",
    "pg": "^8.18.0",
    "pino": "^9.6.0",
-    "prebuilt-tdlib": "^0.1008050.0",
-    "tdl": "^8.0.0"
+    "prebuilt-tdlib": "^0.1008064.0",
+    "tdl": "^8.1.0"
  },
  "devDependencies": {
    "@types/node": "^20",
@@ -10,7 +10,10 @@ import {
  getSubscriptions,
  addSubscription,
  removeSubscription,
+  getGroupById,
+  searchGroups,
 } from "./db/queries.js";
+import { db } from "./db/client.js";
 import { sendTextMessage, sendPhotoMessage } from "./tdlib/client.js";

 const log = childLogger("commands");
@@ -78,6 +81,12 @@ export async function handleMessage(msg: IncomingMessage): Promise<void> {
      case "/status":
        await handleStatus(chatId, userId);
        break;
+      case "/group":
+        await handleGroup(chatId, args);
+        break;
+      case "/sendgroup":
+        await handleSendGroup(chatId, userId, args);
+        break;
      default:
        await sendTextMessage(
          chatId,
@@ -117,6 +126,8 @@ async function handleStart(
    `/search &lt;query&gt; — Search packages`,
    `/latest [n] — Show latest packages`,
    `/package &lt;id&gt; — Package details`,
+    `/group &lt;id or name&gt; — View group info and package list`,
+    `/sendgroup &lt;id&gt; — Send all packages in a group to yourself`,
    `/link &lt;code&gt; — Link your Telegram to your web account`,
    `/subscribe &lt;keyword&gt; — Get notified for new packages`,
    `/subscriptions — View your subscriptions`,
@@ -136,6 +147,8 @@ async function handleHelp(chatId: bigint): Promise<void> {
    `/search &lt;query&gt; — Search by filename or creator`,
    `/latest [n] — Show n most recent packages (default: 5)`,
    `/package &lt;id&gt; — View package details and file list`,
+    `/group &lt;id or name&gt; — View group info and package list`,
+    `/sendgroup &lt;id&gt; — Send all packages in a group to yourself`,
    ``,
    `🔗 <b>Account Linking</b>`,
    `/link &lt;code&gt; — Link Telegram to your web account`,
@@ -432,6 +445,168 @@ async function handleStatus(chatId: bigint, userId: bigint): Promise<void> {
  }
 }

+async function handleGroup(chatId: bigint, query: string): Promise<void> {
+  if (!query) {
+    await sendTextMessage(
+      chatId,
+      "Usage: /group &lt;id or name&gt;\n\nProvide a group ID (starts with 'c') or a name to search.",
+      "textParseModeHTML"
+    );
+    return;
+  }
+
+  const trimmed = query.trim();
+
+  // If it looks like a cuid (starts with 'c', ~25 chars), look up by ID directly
+  if (/^c[a-z0-9]{20,}$/i.test(trimmed)) {
+    const group = await getGroupById(trimmed);
+    if (!group) {
+      await sendTextMessage(chatId, "Group not found.", "textParseModeHTML");
+      return;
+    }
+
+    const packageLines = group.packages.slice(0, 20).map((pkg, i) => {
+      const size = formatSize(pkg.fileSize);
+      return `  ${i + 1}. <b>${escapeHtml(pkg.fileName)}</b> (${size}, ${pkg.fileCount} files) — <code>${pkg.id}</code>`;
+    });
+    const more = group.packages.length > 20
+      ? `\n  ... and ${group.packages.length - 20} more`
+      : "";
+
+    const response = [
+      `📦 <b>Group: ${escapeHtml(group.name)}</b>`,
+      ``,
+      `Packages: ${group.packages.length}`,
+      `ID: <code>${group.id}</code>`,
+      ``,
+      `<b>Contents:</b>`,
+      ...packageLines,
+      more,
+      ``,
+      `Use /sendgroup ${group.id} to receive all packages.`,
+    ]
+      .filter((l) => l !== "")
+      .join("\n");
+
+    await sendTextMessage(chatId, response, "textParseModeHTML");
+    return;
+  }
+
+  // Otherwise search by name
+  const groups = await searchGroups(trimmed, 5);
+
+  if (groups.length === 0) {
+    await sendTextMessage(
+      chatId,
+      `No groups found matching "<b>${escapeHtml(trimmed)}</b>".`,
+      "textParseModeHTML"
+    );
+    return;
+  }
+
+  const lines = groups.map(
+    (g, i) =>
+      `${i + 1}. <b>${escapeHtml(g.name)}</b> — ${g._count.packages} package(s)\n   ID: <code>${g.id}</code>`
+  );
+
+  const response = [
+    `🔍 <b>Groups matching "${escapeHtml(trimmed)}":</b>`,
+    ``,
+    ...lines,
+    ``,
+    `Use /group &lt;id&gt; for full details.`,
+  ].join("\n");
+
+  await sendTextMessage(chatId, response, "textParseModeHTML");
+}
+
+async function handleSendGroup(
+  chatId: bigint,
+  userId: bigint,
+  args: string
+): Promise<void> {
+  if (!args) {
+    await sendTextMessage(
+      chatId,
+      "Usage: /sendgroup &lt;group-id&gt;",
+      "textParseModeHTML"
+    );
+    return;
+  }
+
+  const groupId = args.trim();
+  const group = await getGroupById(groupId);
+
+  if (!group) {
+    await sendTextMessage(chatId, "Group not found.", "textParseModeHTML");
+    return;
+  }
+
+  // Require account linking
+  const link = await findLinkByTelegramUserId(userId);
+  if (!link) {
+    await sendTextMessage(
+      chatId,
+      "You must link your account before receiving packages.\nUse /link &lt;code&gt; to connect.",
+      "textParseModeHTML"
+    );
+    return;
+  }
+
+  // Only send packages that have been uploaded to the destination channel
+  const sendable = group.packages.filter(
+    (pkg) => pkg.destChannelId && pkg.destMessageId
+  );
+
+  if (sendable.length === 0) {
+    await sendTextMessage(
+      chatId,
+      `No packages in group "<b>${escapeHtml(group.name)}</b>" are ready to send yet.`,
+      "textParseModeHTML"
+    );
+    return;
+  }
+
+  // Create a BotSendRequest for each sendable package
+  const requests = await Promise.all(
+    sendable.map((pkg) =>
+      db.botSendRequest.create({
+        data: {
+          packageId: pkg.id,
+          telegramLinkId: link.id,
+          requestedByUserId: link.userId,
+          status: "PENDING",
+        },
+      })
+    )
+  );
+
+  // Fire pg_notify for each request so the send listener picks them up
+  for (const req of requests) {
+    await db.$queryRawUnsafe(
+      `SELECT pg_notify('bot_send', $1)`,
+      req.id
+    ).catch(() => {
+      // Best-effort — the bot also processes PENDING requests on its send queue
+    });
+  }
+
+  await sendTextMessage(
+    chatId,
+    [
+      `✅ <b>Queued ${requests.length} package(s) from "${escapeHtml(group.name)}"</b>`,
+      ``,
+      `You'll receive each archive shortly. Use /package &lt;id&gt; to check individual packages.`,
+    ].join("\n"),
+    "textParseModeHTML"
+  );
+
+  log.info(
+    { groupId, packageCount: requests.length, userId: userId.toString() },
+    "Group send queued"
+  );
+}
+
 function escapeHtml(text: string): string {
  return text
    .replace(/&/g, "&amp;")
@@ -53,7 +53,52 @@ export async function createTelegramLink(
 // ── Package search ──

 export async function searchPackages(query: string, limit = 10) {
-  const packages = await db.package.findMany({
+  // Try full-text search first
+  if (query.length >= 3) {
+    const tsQuery = query
+      .trim()
+      .split(/\s+/)
+      .filter((w) => w.length >= 2)
+      .map((w) => w.replace(/[^a-zA-Z0-9]/g, ""))
+      .filter(Boolean)
+      .join(" & ");
+
+    if (tsQuery) {
+      try {
+        const ftsResults = await db.$queryRawUnsafe<{ id: string }[]>(
+          `SELECT id FROM packages
+           WHERE "searchVector" @@ to_tsquery('english', $1)
+           ORDER BY ts_rank("searchVector", to_tsquery('english', $1)) DESC
+           LIMIT $2`,
+          tsQuery,
+          limit
+        );
+
+        if (ftsResults.length > 0) {
+          return db.package.findMany({
+            where: { id: { in: ftsResults.map((r) => r.id) } },
+            orderBy: { indexedAt: "desc" },
+            select: {
+              id: true,
+              fileName: true,
+              fileSize: true,
+              archiveType: true,
+              fileCount: true,
+              creator: true,
+              indexedAt: true,
+              destChannelId: true,
+              destMessageId: true,
+            },
+          });
+        }
+      } catch {
+        // FTS failed — fall back to ILIKE
+      }
+    }
+  }
+
+  // Fallback: ILIKE search
+  return db.package.findMany({
    where: {
      OR: [
        { fileName: { contains: query, mode: "insensitive" } },
@@ -74,7 +119,44 @@ export async function searchPackages(query: string, limit = 10) {
      destMessageId: true,
    },
  });
-  return packages;
+}
+
+// ── Group queries ──
+
+export async function getGroupById(groupId: string) {
+  return db.packageGroup.findUnique({
+    where: { id: groupId },
+    include: {
+      packages: {
+        orderBy: { indexedAt: "desc" },
+        select: {
+          id: true,
+          fileName: true,
+          fileSize: true,
+          archiveType: true,
+          fileCount: true,
+          creator: true,
+          destChannelId: true,
+          destMessageId: true,
+        },
+      },
+    },
+  });
+}
+
+export async function searchGroups(query: string, limit = 5) {
+  return db.packageGroup.findMany({
+    where: {
+      name: { contains: query, mode: "insensitive" },
+    },
+    orderBy: { createdAt: "desc" },
+    take: limit,
+    select: {
+      id: true,
+      name: true,
+      _count: { select: { packages: true } },
+    },
+  });
 }

 export async function getLatestPackages(limit = 5) {
@@ -28,6 +28,8 @@ services:
      timeout: 5s
      retries: 3
      start_period: 60s
+    volumes:
+      - manual_uploads:/data/uploads
    restart: unless-stopped
    deploy:
      resources:
@@ -54,6 +56,7 @@ services:
    volumes:
      - tdlib_state:/data/tdlib
      - tmp_zips:/tmp/zips
+      - manual_uploads:/data/uploads
    depends_on:
      db:
        condition: service_healthy
@@ -94,6 +97,35 @@ services:
    networks:
      - backend

+  backup:
+    build:
+      context: .
+      dockerfile: backup/Dockerfile
+    pull_policy: never
+    environment:
+      - POSTGRES_USER=${POSTGRES_USER:-dragons}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-stash}
+      - PGPASSWORD=${POSTGRES_PASSWORD:-stash}
+      - POSTGRES_DB=${POSTGRES_DB:-dragonsstash}
+      - RESTIC_REPOSITORY=/backups/restic-repo
+      - RESTIC_PASSWORD=${RESTIC_PASSWORD:?Set RESTIC_PASSWORD in .env}
+      - KUMA_PUSH_URL=${KUMA_PUSH_URL:-}
+      - TZ=${TZ:-Etc/UTC}
+    volumes:
+      - tdlib_state:/data/tdlib-worker:ro
+      - tdlib_bot_state:/data/tdlib-bot:ro
+      - nas_backups:/backups
+    depends_on:
+      db:
+        condition: service_healthy
+    restart: unless-stopped
+    deploy:
+      resources:
+        limits:
+          memory: 1G
+    networks:
+      - backend
+
  db:
    image: postgres:16-alpine
    environment:
@@ -113,14 +145,22 @@ services:
        limits:
          memory: 1G
    networks:
-      - frontend
-      - backend
+      frontend: {}
+      backend:
+        aliases:
+          - dragonsstash-db

 volumes:
  postgres_data:
  tdlib_state:
  tdlib_bot_state:
  tmp_zips:
+  manual_uploads:
+  nas_backups:
+    driver_opts:
+      type: cifs
+      o: "username=${NAS_USERNAME},password=${NAS_PASSWORD},vers=3.0,uid=0,gid=0,file_mode=0660,dir_mode=0770"
+      device: "//${NAS_HOST}/${NAS_SHARE}"

 networks:
  frontend:
@@ -0,0 +1,67 @@
+# Grouping Phase 1: Foundation + Time-Window Grouping
+
+> **For agentic workers:** Use superpowers:subagent-driven-development to implement this plan.
+
+**Goal:** Add grouping infrastructure (schema, enums, notifications model), an ungrouped staging queue in the UI, and time-window auto-grouping as the first automatic signal beyond album grouping.
+
+**Architecture:** Schema changes lay the foundation. Ungrouped tab is a query filter. Time-window grouping runs as a post-processing pass after album grouping in the worker pipeline.
+
+**Tech Stack:** Prisma schema + migration, worker TypeScript, Next.js App Router.
+
+---
+
+## Task 1: Schema Migration
+
+**Files:**
+- Modify: `prisma/schema.prisma`
+- Create: migration SQL
+
+Add:
+1. `GroupingSource` enum: `ALBUM`, `MANUAL`, `AUTO_TIME`, `AUTO_PATTERN`, `AUTO_REPLY`, `AUTO_ZIP`, `AUTO_CAPTION`
+2. `groupingSource GroupingSource @default(MANUAL)` on `PackageGroup`
+3. `SystemNotification` model with `type`, `severity`, `title`, `message`, `context` (Json), `isRead`
+4. `NotificationType` enum: `HASH_MISMATCH`, `MISSING_PART`, `UPLOAD_FAILED`, `DOWNLOAD_FAILED`, `GROUPING_CONFLICT`, `INTEGRITY_AUDIT`
+5. `NotificationSeverity` enum: `INFO`, `WARNING`, `ERROR`
+
+Backfill: `UPDATE package_groups SET "groupingSource" = 'ALBUM' WHERE "mediaAlbumId" IS NOT NULL`
+
+---
+
+## Task 2: Ungrouped Staging Tab in STL Page
+
+**Files:**
+- Modify: `src/lib/telegram/queries.ts` — add `listUngroupedPackages()` query
+- Modify: `src/app/(app)/stls/page.tsx` — add tab parameter support
+- Modify: `src/app/(app)/stls/_components/stl-table.tsx` — add "Ungrouped" tab
+
+Add a tab next to the existing "Skipped" tab that shows packages where `packageGroupId IS NULL`. Uses the existing `PackageListItem` type and table rendering. This gives users a clear view of files that need manual grouping.
+
+---
+
+## Task 3: Time-Window Auto-Grouping in Worker
+
+**Files:**
+- Create: `worker/src/grouping.ts` — add `processTimeWindowGroups()` after existing `processAlbumGroups()`
+- Modify: `worker/src/worker.ts` — call time-window grouping after album grouping
+- Modify: `worker/src/util/config.ts` — add `autoGroupTimeWindowMinutes` config
+
+After album grouping completes, find remaining ungrouped packages from the same channel scan. Cluster packages whose `sourceMessageId` timestamps are within the configured window (default 5 minutes). Create groups for clusters of 2+ with `groupingSource = AUTO_TIME` and name derived from the common filename prefix or first file's base name.
+
+---
+
+## Task 4: Hash Verification After Split
+
+**Files:**
+- Modify: `worker/src/worker.ts` — add hash re-check after concat+split
+- Modify: `worker/src/archive/hash.ts` — (no changes needed, reuse `hashParts`)
+
+After `concatenateFiles()` + `byteLevelSplit()`, re-hash the split parts and compare to the original `contentHash`. If mismatch, log error and create a `SystemNotification` (once that table exists). This closes the integrity gap identified in the audit.
+
+---
+
+## Task 5: Build & Deploy
+
+Rebuild worker and app images. Deploy. Verify:
+- Worker logs show `maxPartSizeMB` and new `autoGroupTimeWindowMinutes` in config
+- Ungrouped tab visible in STL page
+- Previously-skipped large archives begin processing
@@ -0,0 +1,454 @@
+# Send All From Creator Implementation Plan
+
+> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
+
+**Goal:** Add a toolbar button on the STL packages page that queues every sendable package from the currently-filtered creator for delivery to the user's Telegram.
+
+**Architecture:** A new server action `sendAllFromCreatorAction(creatorName)` mirrors the existing `sendAllInGroupAction`: it fetches sendable packages for the creator, dedups against live `BotSendRequest`s, creates a `BotSendRequest` per package, and fires `pg_notify('bot_send', id)`. The bot's existing `send-listener` consumes these unchanged. The client table renders a "Send all from [Creator]" button in the packages-tab toolbar only when `?creator=<name>` is active, wired to the action with a confirm dialog and a count toast.
+
+**Tech Stack:** Next.js 16 App Router, TypeScript, Prisma v7, server actions, `pg_notify`, sonner toasts, lucide-react icons.
+
+**Testing note:** This repo has no test framework (`CLAUDE.md`: "Testing is manual"). Verification for each task is `npm run lint` + `npm run build`, plus manual UI checks at the end. Do not scaffold a test harness.
+
+---
+
+### Task 1: Add `sendAllFromCreatorAction` server action
+
+**Files:**
+- Modify: `src/app/(app)/stls/actions.ts` (append new action after `sendAllInGroupAction`, which ends at line 591)
+
+Reference implementation to mirror: `sendAllInGroupAction` at `src/app/(app)/stls/actions.ts:515-591`.
+Existing imports already present in the file (do not re-add): `auth` from `@/lib/auth`, `prisma` from `@/lib/prisma`, `ActionResult` from `@/types/api.types`, `revalidatePath` from `next/cache`.
+`ActionResult` is generic: `ActionResult<T = void>` (`src/types/api.types.ts`).
+
+- [ ] **Step 1: Append the new action**
+
+Add this to the end of `src/app/(app)/stls/actions.ts`:
+
+```ts
+export async function sendAllFromCreatorAction(
+  creatorName: string
+): Promise<ActionResult<{ queued: number; skipped: number }>> {
+  const session = await auth();
+  if (!session?.user?.id) return { success: false, error: "Unauthorized" };
+
+  const creator = creatorName.trim();
+  if (!creator) {
+    return { success: false, error: "No creator specified" };
+  }
+
+  try {
+    const telegramLink = await prisma.telegramLink.findUnique({
+      where: { userId: session.user.id },
+    });
+
+    if (!telegramLink) {
+      return { success: false, error: "No linked Telegram account. Link one in Settings." };
+    }
+
+    const sendablePackages = await prisma.package.findMany({
+      where: {
+        creator,
+        destChannelId: { not: null },
+        destMessageId: { not: null },
+      },
+      select: { id: true },
+    });
+
+    if (sendablePackages.length === 0) {
+      return { success: false, error: "No uploaded packages found for this creator" };
+    }
+
+    let queued = 0;
+    let skipped = 0;
+    for (const pkg of sendablePackages) {
+      // Only create if no existing PENDING/SENDING request for this package+link combo
+      const existing = await prisma.botSendRequest.findFirst({
+        where: {
+          packageId: pkg.id,
+          telegramLinkId: telegramLink.id,
+          status: { in: ["PENDING", "SENDING"] },
+        },
+      });
+
+      if (existing) {
+        skipped++;
+        continue;
+      }
+
+      const sendRequest = await prisma.botSendRequest.create({
+        data: {
+          packageId: pkg.id,
+          telegramLinkId: telegramLink.id,
+          requestedByUserId: session.user.id,
+          status: "PENDING",
+        },
+      });
+
+      // Notify the bot via pg_notify
+      try {
+        await prisma.$queryRawUnsafe(
+          `SELECT pg_notify('bot_send', $1)`,
+          sendRequest.id
+        );
+      } catch {
+        // Best-effort — the bot also polls periodically
+      }
+
+      queued++;
+    }
+
+    revalidatePath("/stls");
+    return { success: true, data: { queued, skipped } };
+  } catch {
+    return { success: false, error: "Failed to send creator packages" };
+  }
+}
+```
+
+- [ ] **Step 2: Verify lint + typecheck pass**
+
+Run: `npm run lint`
+Expected: no new errors in `src/app/(app)/stls/actions.ts`.
+
+- [ ] **Step 3: Commit**
+
+```bash
+git add src/app/\(app\)/stls/actions.ts
+git commit -m "feat(stls): add sendAllFromCreatorAction server action"
+```
+
+---
+
+### Task 2: Wire up the toolbar button in the STL table
+
+**Files:**
+- Modify: `src/app/(app)/stls/_components/stl-table.tsx`
+  - lucide import (line 6)
+  - actions import block (lines 44-54)
+  - handler (after `handleSendAllInGroup`, which ends at line 278)
+  - `activeCreator` derivation (near `activeTag` at line 445)
+  - toolbar JSX (packages-tab toolbar row starting at line 478)
+
+- [ ] **Step 1: Add the `Send` icon to the lucide import**
+
+Change line 6 from:
+
+```ts
+import { Search, Layers, Upload } from "lucide-react";
+```
+
+to:
+
+```ts
+import { Search, Layers, Upload, Send } from "lucide-react";
+```
+
+- [ ] **Step 2: Import the new action**
+
+In the actions import block (lines 44-54), add `sendAllFromCreatorAction` next to `sendAllInGroupAction`:
+
+```ts
+import {
+  updatePackageCreator,
+  updatePackageTags,
+  renameGroupAction,
+  dissolveGroupAction,
+  createGroupAction,
+  removeFromGroupAction,
+  sendAllInGroupAction,
+  sendAllFromCreatorAction,
+  updateGroupPreviewAction,
+  mergeGroupsAction,
+} from "../actions";
+```
+
+- [ ] **Step 3: Derive the active creator from the URL**
+
+Immediately after the `activeTag` line (`src/app/(app)/stls/_components/stl-table.tsx:445`):
+
+```ts
+  const activeTag = searchParams.get("tag") ?? "";
+```
+
+add:
+
+```ts
+  const activeCreator = searchParams.get("creator") ?? "";
+```
+
+- [ ] **Step 4: Add the click handler**
+
+After `handleSendAllInGroup` (which ends at line 278), add:
+
+```ts
+  const handleSendAllFromCreator = useCallback(() => {
+    if (!confirm(`Send all packages from "${activeCreator}" to your Telegram?`)) return;
+    startTransition(async () => {
+      const result = await sendAllFromCreatorAction(activeCreator);
+      if (result.success) {
+        const { queued, skipped } = result.data;
+        toast.success(
+          `Queued ${queued} package${queued === 1 ? "" : "s"} from ${activeCreator}` +
+            (skipped ? ` (${skipped} already queued)` : "")
+        );
+        router.refresh();
+      } else {
+        toast.error(result.error);
+      }
+    });
+  }, [activeCreator, router]);
+```
+
+- [ ] **Step 5: Render the toolbar button**
+
+In the packages-tab toolbar (`flex flex-wrap items-center gap-2` row at line 478), add the button right after the "Upload Files" button (which closes at line 507, before the `selectedPackages.size >= 2` block at line 508):
+
+```tsx
+            {activeCreator && (
+              <Button
+                variant="outline"
+                size="sm"
+                className="h-9 gap-1.5"
+                onClick={handleSendAllFromCreator}
+              >
+                <Send className="h-3.5 w-3.5" />
+                Send all from {activeCreator}
+              </Button>
+            )}
+```
+
+- [ ] **Step 6: Verify lint + build pass**
+
+Run: `npm run lint && npm run build`
+Expected: no new errors; build completes.
+
+- [ ] **Step 7: Manual verification**
+
+1. `npm run dev`, open `/stls`.
+2. Click a creator name in the Creator column to apply `?creator=<name>` (or navigate to `/stls?creator=<known creator>`).
+3. Confirm the "Send all from [Creator]" button appears in the toolbar.
+4. Remove the creator filter → confirm the button disappears.
+5. Click the button → confirm the dialog appears; on confirm, a toast reports the queued count.
+6. (If a linked Telegram account + uploaded packages exist) confirm packages arrive via the bot.
+
+- [ ] **Step 8: Commit**
+
+```bash
+git add src/app/\(app\)/stls/_components/stl-table.tsx
+git commit -m "feat(stls): add 'send all from creator' toolbar button"
+```
+
+---
+
+### Task 3: Searchable creator filter combobox (makes the filter reachable)
+
+**Why:** Discovered during execution — the `?creator=` filter that reveals the Task 2
+button had no UI trigger. The creator table cell click only opens the edit prompt
+(`package-columns.tsx:339` → `onSetCreator`). This task adds a searchable
+"All Creators" combobox to the packages-tab toolbar (like the existing Tags select,
+but type-to-filter since there can be many creators). The creator cell stays
+edit-only.
+
+**Files:**
+- Modify: `src/lib/telegram/queries.ts` — add `getAllPackageCreators()`.
+- Modify: `src/app/(app)/stls/page.tsx` — fetch + pass `availableCreators`.
+- Create: `src/app/(app)/stls/_components/creator-filter.tsx` — the combobox.
+- Modify: `src/app/(app)/stls/_components/stl-table.tsx` — new prop, handler, render.
+
+- [ ] **Step 1: Add the distinct-creators query**
+
+Append to `src/lib/telegram/queries.ts` (mirrors `getAllPackageTags` at line 530):
+
+```ts
+export async function getAllPackageCreators(): Promise<string[]> {
+  const result = await prisma.$queryRaw<{ creator: string }[]>`
+    SELECT DISTINCT creator FROM packages
+    WHERE creator IS NOT NULL AND creator <> ''
+    ORDER BY creator
+  `;
+  return result.map((r) => r.creator);
+}
+```
+
+- [ ] **Step 2: Create the combobox component**
+
+Create `src/app/(app)/stls/_components/creator-filter.tsx` (mirrors the
+Popover+Command pattern in `src/components/shared/data-table-faceted-filter.tsx`):
+
+```tsx
+"use client";
+
+import { useState } from "react";
+import { Check, ChevronsUpDown } from "lucide-react";
+import { cn } from "@/lib/utils";
+import { Button } from "@/components/ui/button";
+import {
+  Command,
+  CommandEmpty,
+  CommandGroup,
+  CommandInput,
+  CommandItem,
+  CommandList,
+} from "@/components/ui/command";
+import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover";
+
+interface CreatorFilterProps {
+  creators: string[];
+  value: string; // active creator, "" when none
+  onChange: (creator: string) => void; // "" clears the filter
+}
+
+export function CreatorFilter({ creators, value, onChange }: CreatorFilterProps) {
+  const [open, setOpen] = useState(false);
+
+  return (
+    <Popover open={open} onOpenChange={setOpen}>
+      <PopoverTrigger asChild>
+        <Button
+          variant="outline"
+          size="sm"
+          role="combobox"
+          aria-expanded={open}
+          className="h-9 w-[200px] justify-between"
+        >
+          <span className="truncate">{value || "All Creators"}</span>
+          <ChevronsUpDown className="ml-2 h-4 w-4 shrink-0 opacity-50" />
+        </Button>
+      </PopoverTrigger>
+      <PopoverContent className="w-[240px] p-0" align="start">
+        <Command>
+          <CommandInput placeholder="Search creators..." className="h-9" />
+          <CommandList>
+            <CommandEmpty>No creators found.</CommandEmpty>
+            <CommandGroup>
+              <CommandItem
+                value="__all__"
+                onSelect={() => {
+                  onChange("");
+                  setOpen(false);
+                }}
+              >
+                <Check
+                  className={cn("mr-2 h-4 w-4", value === "" ? "opacity-100" : "opacity-0")}
+                />
+                All Creators
+              </CommandItem>
+              {creators.map((creator) => (
+                <CommandItem
+                  key={creator}
+                  value={creator}
+                  onSelect={() => {
+                    onChange(creator);
+                    setOpen(false);
+                  }}
+                >
+                  <Check
+                    className={cn(
+                      "mr-2 h-4 w-4",
+                      value === creator ? "opacity-100" : "opacity-0"
+                    )}
+                  />
+                  <span className="truncate">{creator}</span>
+                </CommandItem>
+              ))}
+            </CommandGroup>
+          </CommandList>
+        </Command>
+      </PopoverContent>
+    </Popover>
+  );
+}
+```
+
+- [ ] **Step 3: Wire the query through the page**
+
+In `src/app/(app)/stls/page.tsx`:
+- Add `getAllPackageCreators` to the import from `@/lib/telegram/queries` (line 3).
+- Add `getAllPackageCreators()` to the `Promise.all` (line 27) and destructure
+  `availableCreators`:
+  ```ts
+  const [result, ingestionStatus, availableTags, availableCreators, skippedCount, ungroupedCount] =
+    await Promise.all([
+      // ...existing entries unchanged...
+      getIngestionStatus(),
+      getAllPackageTags(),
+      getAllPackageCreators(),
+      countSkippedPackages(),
+      countUngroupedPackages(),
+    ]);
+  ```
+  (Insert `getAllPackageCreators()` immediately after `getAllPackageTags()`, and add
+  `availableCreators` in the matching position of the destructure.)
+- Pass the prop to `<StlTable>`: `availableCreators={availableCreators}` (next to
+  `availableTags={availableTags}`).
+
+- [ ] **Step 4: Add prop + handler + render in the table**
+
+In `src/app/(app)/stls/_components/stl-table.tsx`:
+- Import the component: `import { CreatorFilter } from "./creator-filter";`
+- Add to `StlTableProps` (next to `availableTags: string[];`): `availableCreators: string[];`
+- Add to the destructured params (next to `availableTags,`): `availableCreators,`
+- Add the handler after `updateTagFilter` (ends ~line 210):
+  ```ts
+  const updateCreatorFilter = useCallback(
+    (value: string) => {
+      const params = new URLSearchParams(searchParams.toString());
+      if (value) {
+        params.set("creator", value);
+        params.set("page", "1");
+      } else {
+        params.delete("creator");
+      }
+      router.push(`${pathname}?${params.toString()}`, { scroll: false });
+    },
+    [router, pathname, searchParams]
+  );
+  ```
+- Render it in the toolbar right after the Tags `Select` block (the
+  `{availableTags.length > 0 && (...)}` block, ~lines 507-522):
+  ```tsx
+  {availableCreators.length > 0 && (
+    <CreatorFilter
+      creators={availableCreators}
+      value={activeCreator}
+      onChange={updateCreatorFilter}
+    />
+  )}
+  ```
+
+- [ ] **Step 5: Verify + manual test**
+
+Run: `npx tsc --noEmit` (filter for the touched files — expect none new). Note: full
+`npm run build` fails on a pre-existing stale Prisma client issue in
+`src/lib/telegram/*` unrelated to this change; do not attempt to fix it.
+Manual: open `/stls`, use the "All Creators" combobox, type to filter, pick a
+creator → list filters and the "Send all from [Creator]" button appears; pick
+"All Creators" → filter clears and button disappears.
+
+- [ ] **Step 6: Commit**
+
+```bash
+git add "src/lib/telegram/queries.ts" "src/app/(app)/stls/page.tsx" \
+  "src/app/(app)/stls/_components/creator-filter.tsx" \
+  "src/app/(app)/stls/_components/stl-table.tsx"
+git commit -m "feat(stls): add searchable creator filter combobox to toolbar"
+```
+
+---
+
+## Self-Review
+
+**Spec coverage:**
+- Req 1 (button when filtered by creator) → Task 2 Step 5 (`{activeCreator && ...}`).
+- Req 2 (hidden when no filter) → Task 2 Step 5 conditional.
+- Req 3 (confirm dialog) → Task 2 Step 4 `confirm(...)`.
+- Req 4 (all pages, not current page) → Task 1 queries `prisma.package.findMany` by `creator`, unpaginated.
+- Req 5 (skip not-uploaded + dedup live requests) → Task 1 `where` filter on `destChannelId`/`destMessageId` + `existing` check.
+- Req 6 (report counts) → Task 1 returns `{ queued, skipped }`; Task 2 Step 4 toast.
+- Req 7 (no polling) → Task 2 handler queues + `router.refresh()`, no poll loop.
+- Blank-creator guard → Task 1 `creator.trim()` check.
+
+**Placeholder scan:** No TBD/TODO/placeholder steps; all code shown in full.
+
+**Type consistency:** Action returns `ActionResult<{ queued: number; skipped: number }>`; handler destructures `result.data.{queued,skipped}` inside the `result.success` branch (where `data` is typed). `sendAllFromCreatorAction` name matches between Task 1 definition, Task 2 import, and Task 2 call site. `activeCreator` defined once (Step 3) and used in Steps 4-5.
@@ -0,0 +1,555 @@
+# NAS Backup for Postgres + TDLib State Implementation Plan
+
+> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
+
+**Goal:** Add a `backup` container to the DragonsStash stack that takes daily, encrypted, deduplicated backups of the Postgres database and both TDLib state volumes, and ships them to a Synology NAS over NFS.
+
+**Architecture:** A small Alpine-based image (restic + postgresql16-client + curl + dcron) runs as its own compose service. A crontab fires `backup.sh` daily at 03:00, which dumps Postgres, tars the TDLib volumes, hands both to `restic backup` against an NFS-backed Docker volume, prunes with `restic forget --keep-daily 14`, and reports success/failure to an Uptime Kuma push monitor. Matches the existing `worker`/`bot` pattern: build context in the repo's `docker-compose.yml`, prebuilt image in `/opt/stacks/DragonsStash/docker-compose.yml`, built and pushed by `.drone.yml`.
+
+**Tech Stack:** Alpine 3.20, restic 0.16, postgresql16-client, dcron, bash, Docker Compose NFS volume driver.
+
+## Global Constraints
+
+- Retention: `restic forget --keep-daily 14 --prune` (14-day window, per approved spec).
+- Schedule: daily backup at 03:00, weekly `restic check` at 04:00 Sunday.
+- No Docker socket mount, no `privileged: true` — the backup container must not be able to control sibling containers.
+- No host-level mount — NFS access only via Docker's native `driver_opts: type: nfs` volume, never `/etc/fstab`.
+- Encryption and retention are restic's job — no hand-rolled `age`/`gpg`/`find -mtime` logic.
+- TDLib volumes are tarred live (best-effort) — never pause `worker`/`bot` for the backup.
+- Restore is a manual, documented procedure only — never scripted/automated.
+
+**Required user input before Task 5 can run:** `NAS_HOST` and `NAS_EXPORT_PATH` (the Synology NFS share details) and a Kuma Push-monitor URL (`KUMA_PUSH_URL`, created manually in the existing Uptime Kuma instance, ~26h expected heartbeat interval). Tasks 1–4 need none of these and can proceed immediately; do not substitute placeholder values for them in Task 5 — stop and ask the user instead.
+
+---
+
+### Task 1: Backup image (Dockerfile + entrypoint)
+
+**Files:**
+- Create: `backup/Dockerfile`
+- Create: `backup/entrypoint.sh`
+- Create: `backup/crontab`
+
+**Interfaces:**
+- Produces: a buildable image tagged `dragonsstash-backup:test` locally, with `/entrypoint.sh` as `ENTRYPOINT`, `/backup.sh` present at the image root (written in Task 2 — this task only needs the `COPY` line and a placeholder-free stub isn't acceptable, so create an empty‑body-but-real `backup/backup.sh` here containing just `#!/bin/bash` + `exit 0`, and Task 2 replaces its contents), `restic`, `pg_dump`/`pg_restore`, `curl`, `tar`, `bash`, `dcron` all on `PATH`.
+- Consumes: nothing from earlier tasks.
+
+- [ ] **Step 1: Write `backup/backup.sh` stub**
+
+```bash
+#!/bin/bash
+set -euo pipefail
+exit 0
+```
+
+- [ ] **Step 2: Write `backup/entrypoint.sh`**
+
+```bash
+#!/bin/bash
+set -euo pipefail
+
+if ! restic snapshots >/dev/null 2>&1; then
+  restic init
+fi
+
+exec crond -f -l 2
+```
+
+- [ ] **Step 3: Write `backup/crontab`**
+
+```
+0 3 * * * /backup.sh >> /proc/1/fd/1 2>&1
+0 4 * * 0 restic check >> /proc/1/fd/1 2>&1
+```
+
+- [ ] **Step 4: Write `backup/Dockerfile`**
+
+```dockerfile
+FROM alpine:3.20
+
+RUN apk add --no-cache restic postgresql16-client curl tzdata dcron tar bash
+
+COPY backup/backup.sh /backup.sh
+COPY backup/entrypoint.sh /entrypoint.sh
+COPY backup/crontab /etc/crontabs/root
+
+RUN chmod +x /backup.sh /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]
+```
+
+- [ ] **Step 5: Build the image**
+
+Run: `cd /home/sam/Documents/DragonsStash && docker build -t dragonsstash-backup:test -f backup/Dockerfile .`
+Expected: build completes with `Successfully tagged dragonsstash-backup:test` (or Buildkit's equivalent final `naming to docker.io/library/dragonsstash-backup:test done`), no errors.
+
+- [ ] **Step 6: Verify the tools are present**
+
+Run: `docker run --rm dragonsstash-backup:test restic version && docker run --rm dragonsstash-backup:test pg_dump --version`
+Expected: `restic 0.16.x ...` and `pg_dump (PostgreSQL) 16.x` printed, both commands exit 0.
+
+- [ ] **Step 7: Verify the entrypoint initializes an empty repo and starts cron**
+
+```bash
+mkdir -p /tmp/backup-repo-smoke
+docker run -d --name backup-smoke \
+  -e RESTIC_REPOSITORY=/backups/restic-repo -e RESTIC_PASSWORD=smoketest \
+  -v /tmp/backup-repo-smoke:/backups \
+  dragonsstash-backup:test
+sleep 2
+docker logs backup-smoke
+docker exec backup-smoke restic snapshots
+docker rm -f backup-smoke
+rm -rf /tmp/backup-repo-smoke
+```
+
+Expected: `docker logs` shows no errors (restic init ran silently); `restic snapshots` prints an empty snapshot list (repo exists, header row only, no error).
+
+- [ ] **Step 8: Commit**
+
+```bash
+git add backup/Dockerfile backup/entrypoint.sh backup/backup.sh backup/crontab
+git commit -m "Add backup service image (Dockerfile, entrypoint, crontab)"
+```
+
+---
+
+### Task 2: `backup.sh` script
+
+**Files:**
+- Modify: `backup/backup.sh` (replace Task 1's stub with the real script)
+
+**Interfaces:**
+- Consumes: the image built in Task 1 (`dragonsstash-backup:test`), rebuilt after this change.
+- Produces: `/backup.sh`, invoked by cron in Task 1's `crontab` and manually in Task 6's verification. Reads env vars `POSTGRES_USER`, `PGPASSWORD`, `POSTGRES_DB`, `RESTIC_REPOSITORY`, `RESTIC_PASSWORD`, `KUMA_PUSH_URL`. Assumes network hostname `dragonsstash-db:5432` for Postgres and mounts `/data/tdlib-worker`, `/data/tdlib-bot` (read-only) for TDLib state.
+
+- [ ] **Step 1: Replace `backup/backup.sh` with the real script**
+
+```bash
+#!/bin/bash
+set -euo pipefail
+
+report_failure() {
+  curl -fsS "$KUMA_PUSH_URL" --get \
+    --data-urlencode "status=down" \
+    --data-urlencode "msg=$BASH_COMMAND failed" || true
+}
+trap report_failure ERR
+
+DUMP_FILE=/tmp/dragonsstash.dump
+TAR_FILE=/tmp/tdlib.tar.gz
+
+pg_dump -h dragonsstash-db -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Fc -f "$DUMP_FILE"
+
+tar czf "$TAR_FILE" -C /data tdlib-worker tdlib-bot
+
+restic backup "$DUMP_FILE" "$TAR_FILE"
+restic forget --keep-daily 14 --prune
+
+rm -f "$DUMP_FILE" "$TAR_FILE"
+
+curl -fsS "$KUMA_PUSH_URL" --get \
+  --data-urlencode "status=up" \
+  --data-urlencode "msg=OK"
+```
+
+- [ ] **Step 2: Rebuild the image**
+
+Run: `docker build -t dragonsstash-backup:test -f backup/Dockerfile .`
+Expected: build succeeds.
+
+- [ ] **Step 3: Stand up a scratch Postgres aliased as `dragonsstash-db`**
+
+```bash
+docker network create backup-test-net 2>/dev/null || true
+docker run -d --name test-pg --network backup-test-net --network-alias dragonsstash-db \
+  -e POSTGRES_USER=dragons -e POSTGRES_PASSWORD=stash -e POSTGRES_DB=dragonsstash \
+  postgres:16-alpine
+sleep 5
+docker exec test-pg pg_isready -U dragons -d dragonsstash
+```
+
+Expected: `accepting connections`.
+
+- [ ] **Step 4: Stand up a mock Kuma push endpoint**
+
+```bash
+mkdir -p /tmp/mock-kuma-root && touch /tmp/mock-kuma-root/push
+docker run -d --name mock-kuma --network backup-test-net \
+  -v /tmp/mock-kuma-root:/srv -w /srv python:3-alpine \
+  python3 -m http.server 8000
+sleep 1
+```
+
+- [ ] **Step 5: Prepare fake TDLib state and a local restic repo dir**
+
+```bash
+mkdir -p /tmp/backup-test/tdlib-worker /tmp/backup-test/tdlib-bot /tmp/backup-test/repo
+echo "fake-session" > /tmp/backup-test/tdlib-worker/state.bin
+echo "fake-session" > /tmp/backup-test/tdlib-bot/state.bin
+```
+
+- [ ] **Step 6: Initialize the test restic repo and run `backup.sh` (success path)**
+
+```bash
+docker run --rm --network backup-test-net \
+  -e RESTIC_REPOSITORY=/backups/restic-repo -e RESTIC_PASSWORD=testpassword \
+  -v /tmp/backup-test/repo:/backups \
+  --entrypoint restic dragonsstash-backup:test init
+
+docker run --rm --network backup-test-net \
+  -e POSTGRES_USER=dragons -e POSTGRES_PASSWORD=stash -e PGPASSWORD=stash -e POSTGRES_DB=dragonsstash \
+  -e RESTIC_REPOSITORY=/backups/restic-repo -e RESTIC_PASSWORD=testpassword \
+  -e KUMA_PUSH_URL=http://mock-kuma:8000/push \
+  -v /tmp/backup-test/tdlib-worker:/data/tdlib-worker:ro \
+  -v /tmp/backup-test/tdlib-bot:/data/tdlib-bot:ro \
+  -v /tmp/backup-test/repo:/backups \
+  --entrypoint /backup.sh dragonsstash-backup:test
+echo "exit code: $?"
+```
+
+Expected: exit code `0`, restic prints a line like `snapshot xxxxxxxx saved`, no error output.
+
+- [ ] **Step 7: Verify the snapshot landed and contains both files**
+
+```bash
+docker run --rm -v /tmp/backup-test/repo:/backups \
+  -e RESTIC_REPOSITORY=/backups/restic-repo -e RESTIC_PASSWORD=testpassword \
+  --entrypoint restic dragonsstash-backup:test snapshots
+
+docker run --rm -v /tmp/backup-test/repo:/backups \
+  -e RESTIC_REPOSITORY=/backups/restic-repo -e RESTIC_PASSWORD=testpassword \
+  --entrypoint restic dragonsstash-backup:test ls latest
+```
+
+Expected: `snapshots` shows exactly one entry; `ls latest` lists `/tmp/dragonsstash.dump` and `/tmp/tdlib.tar.gz`.
+
+- [ ] **Step 8: Verify the failure path reports to Kuma**
+
+```bash
+docker run --rm --network backup-test-net \
+  -e POSTGRES_USER=dragons -e POSTGRES_PASSWORD=wrongpass -e PGPASSWORD=wrongpass -e POSTGRES_DB=dragonsstash \
+  -e RESTIC_REPOSITORY=/backups/restic-repo -e RESTIC_PASSWORD=testpassword \
+  -e KUMA_PUSH_URL=http://mock-kuma:8000/push \
+  -v /tmp/backup-test/tdlib-worker:/data/tdlib-worker:ro \
+  -v /tmp/backup-test/tdlib-bot:/data/tdlib-bot:ro \
+  -v /tmp/backup-test/repo:/backups \
+  --entrypoint /backup.sh dragonsstash-backup:test
+echo "exit code: $?"
+docker logs mock-kuma | tail -5
+```
+
+Expected: exit code nonzero (pg_dump auth failure trips `set -e`); `docker logs mock-kuma` shows a GET request line containing `status=down`.
+
+- [ ] **Step 9: Clean up test resources**
+
+```bash
+docker rm -f test-pg mock-kuma
+docker network rm backup-test-net
+rm -rf /tmp/backup-test /tmp/mock-kuma-root
+```
+
+- [ ] **Step 10: Commit**
+
+```bash
+git add backup/backup.sh
+git commit -m "Implement backup.sh: pg_dump + tdlib tar + restic backup/forget + Kuma reporting"
+```
+
+---
+
+### Task 3: Wire the `backup` service into the repo's `docker-compose.yml`
+
+**Files:**
+- Modify: `docker-compose.yml`
+- Modify: `.env.example`
+
+**Interfaces:**
+- Consumes: `backup/Dockerfile` (Task 1), `backup/backup.sh` (Task 2).
+- Produces: a `backup` compose service buildable via `docker compose build backup`, and a `nas_backups` named volume other tasks (5) will mirror into the production compose file.
+
+- [ ] **Step 1: Add the `backup` service to `docker-compose.yml`**
+
+Insert after the existing `bot` service (before `db`):
+
+```yaml
+  backup:
+    build:
+      context: .
+      dockerfile: backup/Dockerfile
+    pull_policy: never
+    environment:
+      - POSTGRES_USER=${POSTGRES_USER:-dragons}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-stash}
+      - PGPASSWORD=${POSTGRES_PASSWORD:-stash}
+      - POSTGRES_DB=${POSTGRES_DB:-dragonsstash}
+      - RESTIC_REPOSITORY=/backups/restic-repo
+      - RESTIC_PASSWORD=${RESTIC_PASSWORD:?Set RESTIC_PASSWORD in .env}
+      - KUMA_PUSH_URL=${KUMA_PUSH_URL:?Set KUMA_PUSH_URL in .env}
+      - TZ=${TZ:-Etc/UTC}
+    volumes:
+      - tdlib_state:/data/tdlib-worker:ro
+      - tdlib_bot_state:/data/tdlib-bot:ro
+      - nas_backups:/backups
+    depends_on:
+      db:
+        condition: service_healthy
+    restart: unless-stopped
+    deploy:
+      resources:
+        limits:
+          memory: 256M
+    networks:
+      - backend
+```
+
+- [ ] **Step 2: Add the `nas_backups` volume to the `volumes:` block**
+
+```yaml
+  nas_backups:
+    driver_opts:
+      type: nfs
+      o: "addr=${NAS_HOST},rw,nfsvers=4,soft,timeo=100"
+      device: ":${NAS_EXPORT_PATH}"
+```
+
+- [ ] **Step 3: Document the new env vars in `.env.example`**
+
+Append:
+
+```
+# Backup (NAS via NFS + restic)
+NAS_HOST=""              # Synology NAS IP or hostname reachable from this host
+NAS_EXPORT_PATH=""       # NFS export path, e.g. /volume1/dragonsstash-backups
+RESTIC_PASSWORD=""       # generate with: openssl rand -base64 32
+KUMA_PUSH_URL=""         # Uptime Kuma Push monitor URL (create the monitor first)
+TZ="Etc/UTC"
+```
+
+- [ ] **Step 4: Validate the compose file parses**
+
+Run (with dummy values so the `:?` guards don't fail parsing — `AUTH_SECRET` is required by the existing `app` service, not by this change, but `config` validates the whole file):
+```bash
+RESTIC_PASSWORD=dummy KUMA_PUSH_URL=http://dummy NAS_HOST=dummy NAS_EXPORT_PATH=/dummy AUTH_SECRET=dummy \
+  docker compose -f docker-compose.yml config --quiet
+```
+Expected: no output, exit code 0 (a syntax/interpolation error would print to stderr and exit nonzero).
+
+- [ ] **Step 5: Validate the service actually builds through Compose**
+
+Run:
+```bash
+RESTIC_PASSWORD=dummy KUMA_PUSH_URL=http://dummy NAS_HOST=dummy NAS_EXPORT_PATH=/dummy AUTH_SECRET=dummy \
+  docker compose -f docker-compose.yml build backup
+```
+Expected: build succeeds (reuses Task 1's image layers).
+
+- [ ] **Step 6: Commit**
+
+```bash
+git add docker-compose.yml .env.example
+git commit -m "Add backup service and nas_backups volume to docker-compose.yml"
+```
+
+---
+
+### Task 4: CI — build and push the backup image
+
+**Files:**
+- Modify: `.drone.yml`
+
+**Interfaces:**
+- Consumes: `backup/Dockerfile` (Task 1).
+- Produces: `git.samagsteribbe.nl/admin/dragonsstash-backup:latest` (and `:<short-sha>`), pushed on every push to `main`. Task 5's production compose file references this image tag.
+
+- [ ] **Step 1: Add a `build-backup` step, mirroring `build-worker`/`build-bot`**
+
+Insert after the existing `build-bot` step in `.drone.yml`:
+
+```yaml
+  - name: build-backup
+    image: plugins/docker
+    depends_on: [clone]
+    settings:
+      repo: git.samagsteribbe.nl/admin/dragonsstash-backup
+      registry: git.samagsteribbe.nl
+      dockerfile: backup/Dockerfile
+      tags:
+        - latest
+        - "${DRONE_COMMIT_SHA:0:8}"
+      username:
+        from_secret: gitea_username
+      password:
+        from_secret: gitea_password
+```
+
+- [ ] **Step 2: Add `build-backup` to the `deploy` step's `depends_on`**
+
+Change:
+```yaml
+  - name: deploy
+    image: alpine
+    depends_on: [build-app, build-worker, build-bot]
+```
+to:
+```yaml
+  - name: deploy
+    image: alpine
+    depends_on: [build-app, build-worker, build-bot, build-backup]
+```
+
+- [ ] **Step 3: Validate YAML syntax**
+
+Run: `python3 -c "import yaml; yaml.safe_load(open('.drone.yml')); print('OK')"`
+Expected: `OK` printed, no exception.
+
+- [ ] **Step 4: Commit**
+
+```bash
+git add .drone.yml
+git commit -m "Add build-backup CI step, include it in deploy dependencies"
+```
+
+---
+
+### Task 5: Wire production (`/opt/stacks/DragonsStash`) — requires NAS + Kuma details from the user
+
+**Do not start this task until the user has supplied `NAS_HOST` and `NAS_EXPORT_PATH` for the Synology NFS share, and has created an Uptime Kuma Push monitor (name it `dragonsstash-backup`, ~26h expected heartbeat interval) and shared its push URL. If any of these are missing, stop and ask — do not substitute placeholder values here, since this file drives the real deployment.**
+
+**Files:**
+- Modify: `/opt/stacks/DragonsStash/docker-compose.yml`
+- Modify: `/opt/stacks/DragonsStash/.env`
+
+**Interfaces:**
+- Consumes: `git.samagsteribbe.nl/admin/dragonsstash-backup:latest` (published by Task 4's CI step once merged/pushed), the real `NAS_HOST`/`NAS_EXPORT_PATH`/`KUMA_PUSH_URL` values gathered above.
+- Produces: a running `dragonsstash-backup` container on the production host, verified in Task 6.
+
+- [ ] **Step 1: Generate `RESTIC_PASSWORD` and add all four new vars to `/opt/stacks/DragonsStash/.env`**
+
+```bash
+cd /opt/stacks/DragonsStash
+printf '\n# Backup (NAS via NFS + restic)\nNAS_HOST="<value from user>"\nNAS_EXPORT_PATH="<value from user>"\nRESTIC_PASSWORD="%s"\nKUMA_PUSH_URL="<value from user>"\nTZ="Etc/UTC"\n' "$(openssl rand -base64 32)" >> .env
+```
+
+Replace the two `<value from user>` placeholders with the real NAS details and Kuma push URL before saving — this step cannot be completed with the literal placeholder text left in place.
+
+- [ ] **Step 2: Add the `backup` service to `/opt/stacks/DragonsStash/docker-compose.yml`**
+
+Insert after the existing `bot` service (before `db`):
+
+```yaml
+  backup:
+    image: git.samagsteribbe.nl/admin/dragonsstash-backup:latest
+    container_name: dragonsstash-backup
+    restart: unless-stopped
+    environment:
+      - POSTGRES_USER=${POSTGRES_USER:-dragons}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-stash}
+      - PGPASSWORD=${POSTGRES_PASSWORD:-stash}
+      - POSTGRES_DB=${POSTGRES_DB:-dragonsstash}
+      - RESTIC_REPOSITORY=/backups/restic-repo
+      - RESTIC_PASSWORD=${RESTIC_PASSWORD:?Set RESTIC_PASSWORD in .env}
+      - KUMA_PUSH_URL=${KUMA_PUSH_URL:?Set KUMA_PUSH_URL in .env}
+      - TZ=${TZ:-Etc/UTC}
+    volumes:
+      - tdlib_state:/data/tdlib-worker:ro
+      - tdlib_bot_state:/data/tdlib-bot:ro
+      - nas_backups:/backups
+    depends_on:
+      db:
+        condition: service_healthy
+    deploy:
+      resources:
+        limits:
+          memory: 256M
+    networks:
+      - internal
+```
+
+- [ ] **Step 3: Add the `nas_backups` volume**
+
+```yaml
+  nas_backups:
+    driver_opts:
+      type: nfs
+      o: "addr=${NAS_HOST},rw,nfsvers=4,soft,timeo=100"
+      device: ":${NAS_EXPORT_PATH}"
+```
+
+- [ ] **Step 4: Validate the production compose file parses with the real `.env`**
+
+Run: `cd /opt/stacks/DragonsStash && docker compose config --quiet`
+Expected: no output, exit code 0.
+
+- [ ] **Step 5: Commit is not applicable here** — `/opt/stacks/DragonsStash` is a deployed copy, not the git repo (confirm with `git -C /opt/stacks/DragonsStash status` — expect "not a git repository"). Skip committing; Task 6 deploys these file changes directly.
+
+---
+
+### Task 6: Deploy and verify
+
+**Files:** none (operational task)
+
+**Interfaces:**
+- Consumes: everything from Tasks 1–5.
+- Produces: a running, verified backup on the real NAS, and one completed restore drill.
+
+- [ ] **Step 1: Confirm with the user before pushing/deploying**
+
+Pushing to `main` triggers Drone CI to build all four images and deploy to the production host via SSH (`docker compose pull && docker compose up -d`). Confirm the user wants this to happen now before proceeding — this affects the live stack.
+
+- [ ] **Step 2: Push to `main`**
+
+```bash
+cd /home/sam/Documents/DragonsStash
+git push origin main
+```
+
+Expected: Drone pipeline runs `build-app`, `build-worker`, `build-bot`, `build-backup`, then `deploy`, all green. Check via the Drone UI or `drone build info admin/DragonsStash <build-number>` if the `drone` CLI is configured.
+
+- [ ] **Step 3: Confirm the container is up on the production host**
+
+```bash
+ssh sam@192.168.68.68 "docker ps --filter name=dragonsstash-backup --format '{{.Names}}\t{{.Status}}'"
+```
+
+Expected: `dragonsstash-backup   Up ...`.
+
+- [ ] **Step 4: Trigger one manual backup run and confirm a snapshot lands on the NAS**
+
+```bash
+ssh sam@192.168.68.68 "docker exec dragonsstash-backup /backup.sh"
+ssh sam@192.168.68.68 "docker exec dragonsstash-backup restic snapshots"
+```
+
+Expected: `backup.sh` exits 0; `restic snapshots` lists exactly one entry.
+
+- [ ] **Step 5: Confirm the Uptime Kuma push monitor shows green**
+
+Open the Uptime Kuma dashboard and check the `dragonsstash-backup` monitor's status is up with a recent heartbeat.
+
+- [ ] **Step 6: Restore drill — prove the backup is actually restorable**
+
+```bash
+ssh sam@192.168.68.68 "docker exec dragonsstash-backup restic restore latest --target /tmp/restore-drill"
+ssh sam@192.168.68.68 "docker exec dragonsstash-backup ls -la /tmp/restore-drill/tmp"
+```
+
+Expected: `/tmp/restore-drill/tmp/dragonsstash.dump` and `/tmp/restore-drill/tmp/tdlib.tar.gz` both present with nonzero size. Then, on a scratch Postgres (not the live `dragonsstash-db`), confirm the dump restores cleanly:
+
+```bash
+ssh sam@192.168.68.68 "docker run -d --rm --name restore-drill-pg --network dragonsstash_internal \
+  -e POSTGRES_USER=drill -e POSTGRES_PASSWORD=drill -e POSTGRES_DB=drill postgres:16-alpine"
+ssh sam@192.168.68.68 "docker cp dragonsstash-backup:/tmp/restore-drill/tmp/dragonsstash.dump /tmp/dragonsstash.dump"
+ssh sam@192.168.68.68 "docker cp /tmp/dragonsstash.dump restore-drill-pg:/tmp/dragonsstash.dump"
+ssh sam@192.168.68.68 "docker exec -e PGPASSWORD=drill restore-drill-pg pg_restore -U drill -d drill --clean --if-exists /tmp/dragonsstash.dump"
+ssh sam@192.168.68.68 "docker exec -e PGPASSWORD=drill restore-drill-pg psql -U drill -d drill -c '\\dt' | head -20"
+ssh sam@192.168.68.68 "docker rm -f restore-drill-pg"
+```
+
+Expected: `pg_restore` completes without fatal errors; `\dt` lists the app's tables (e.g. `Package`, `User`, `TelegramLink`).
+
+- [ ] **Step 7: Clean up drill artifacts**
+
+```bash
+ssh sam@192.168.68.68 "docker exec dragonsstash-backup rm -rf /tmp/restore-drill"
+ssh sam@192.168.68.68 "rm -f /tmp/dragonsstash.dump"
+```
@@ -0,0 +1,184 @@
+# Worker Improvements Design
+
+**Date:** 2026-05-02  
+**Status:** Approved  
+**Scope:** Dragon's Stash Telegram ingestion worker
+
+## Problem Statement
+
+Three issues to address:
+
+1. **Double-uploads**: The same archive occasionally appears twice in the destination Telegram channel. Root causes: (a) the worker crashes between `uploadToChannel()` confirming success and `createPackageWithFiles()` writing to the DB — no DB record means `recoverIncompleteUploads()` can't detect the orphaned Telegram message, and the next cycle re-uploads; (b) two accounts scanning the same source channel can both pass the hash dedup check before either creates a DB record, racing to upload the same file.
+
+2. **Sequential account processing**: Both Telegram accounts are processed one after another via `withTdlibMutex`, even though TDLib fully supports multiple concurrent clients in the same process (each with separate `databaseDirectory` and `filesDirectory`). This halves throughput unnecessarily.
+
+3. **Premium upload limit not used**: The Premium account can upload up to 4 GB per file, but `MAX_UPLOAD_SIZE` is hardcoded at ~1,950 MB. This causes unnecessary file splitting and expensive repack operations for files that could upload directly.
+
+## Solution Overview
+
+Three targeted changes, no architectural overhaul:
+
+1. Two-phase DB write + hash advisory lock (fixes double-uploads)
+2. Remove TDLib mutex from the scheduler loop (enables parallel accounts)
+3. Per-account `maxUploadSize` from `getMe().is_premium` (enables 4 GB for Premium)
+
+---
+
+## Section 1: Double-Upload Fix
+
+### 1a. Two-Phase DB Write
+
+**Current flow:**
+```
+uploadToChannel() → preview download → metadata extraction → createPackageWithFiles()
+```
+
+If the worker crashes anywhere between upload confirmation and `createPackageWithFiles()`, no DB record exists. `recoverIncompleteUploads()` only checks packages with an existing `destMessageId` in the DB — it cannot find an orphaned Telegram message with no corresponding row.
+
+**New flow:**
+```
+uploadToChannel()
+  → createPackageStub()          ← minimal record, destMessageId set immediately
+  → preview download
+  → metadata extraction
+  → updatePackageWithMetadata()  ← adds file list, preview, creator, tags
+```
+
+`createPackageStub()` writes: `contentHash`, `fileName`, `fileSize`, `archiveType`, `sourceChannelId`, `sourceMessageId`, `destChannelId`, `destMessageId`, `isMultipart`, `partCount`, `ingestionRunId`. File list and preview are left empty.
+
+If the worker crashes after the stub is written:
+- `recoverIncompleteUploads()` finds the record (has `destMessageId`), verifies the Telegram message exists, keeps it.
+- Next cycle: `packageExistsByHash()` returns true → skips re-upload.
+- The stub has `fileCount = 0` and no file listing. The UI shows "metadata pending" rather than failing silently.
+
+Stubs with `fileCount = 0` are valid deliverable packages (the bot can still send the file). Backfilling metadata on stubs is out of scope for this change — the crash case is rare and the stub is functional.
+
+### 1b. Hash Advisory Lock
+
+**The race (two accounts, shared source channel):**
+```
+Worker A: packageExistsByHash(X) → false  (no record yet)
+Worker B: packageExistsByHash(X) → false  (no record yet)
+Worker A: uploads file → destMessageId_A
+Worker B: uploads file → destMessageId_B  ← duplicate Telegram message
+Worker A: createPackageStub() → succeeds  (contentHash @unique satisfied)
+Worker B: createPackageStub() → fails unique constraint on contentHash
+```
+Result: two Telegram messages, one DB record. Worker B's upload is wasted.
+
+**Fix:** Before calling `uploadToChannel()`, acquire a PostgreSQL session advisory lock keyed on the content hash:
+
+```sql
+SELECT pg_try_advisory_lock(hash_bigint)
+```
+
+Where `hash_bigint` is the first 8 bytes of the SHA-256 content hash interpreted as a signed bigint.
+
+- `pg_try_advisory_lock` is non-blocking. If another worker holds the lock (same file, shared channel), return `false` → treat as duplicate, skip.
+- After acquiring the lock, **re-run `packageExistsByHash()`** before uploading. This catches the case where another worker finished and released the lock between the first check and this one — without the re-check, the current worker would proceed to re-upload.
+- The lock is session-scoped: released automatically on DB session end. No manual cleanup needed on crash.
+- The lock is released explicitly after `createPackageStub()` completes (or on any error path).
+
+**Implementation location:** New helper `tryAcquireHashLock(contentHash)` / `releaseHashLock(contentHash)` in `worker/src/db/locks.ts`, reusing the existing DB client pattern.
+
+---
+
+## Section 2: Parallel Account Processing
+
+### Current Constraint
+
+`withTdlibMutex` in `scheduler.ts` serializes all TDLib operations across accounts. This was a conservative guard, but TDLib explicitly supports multiple concurrent clients in the same process provided each has its own `databaseDirectory` and `filesDirectory`.
+
+The codebase already satisfies this requirement:
+```typescript
+// worker/src/tdlib/client.ts
+const dbPath = path.join(config.tdlibStateDir, account.id);
+const client = createClient({
+  databaseDirectory: dbPath,
+  filesDirectory: path.join(dbPath, "files"),
+});
+```
+
+Each account gets `<TDLIB_STATE_DIR>/<account.id>/` — fully isolated.
+
+### Change
+
+Replace the sequential `for` loop in `scheduler.ts` with `Promise.allSettled()`:
+
+```typescript
+// Before
+for (const account of accounts) {
+  await withTdlibMutex(`ingest:${account.phone}`, () => runWorkerForAccount(account));
+}
+
+// After
+await Promise.allSettled(accounts.map((account) => runWorkerForAccount(account)));
+```
+
+The per-account PostgreSQL advisory lock in `db/locks.ts` already prevents any account from being processed twice simultaneously. `Promise.allSettled()` ensures one account's failure doesn't abort the other.
+
+The `withTdlibMutex` wrapper can be removed from the ingest path entirely. The auth path (`authenticateAccount`) should also be run in parallel but may remain guarded if TDLib auth flows have ordering dependencies — verify during implementation.
+
+**No Docker Compose changes needed.** Both accounts run in the same container.
+
+### Speed Limit Notifications
+
+TDLib fires `updateSpeedLimitNotification` when an account's upload or download speed is throttled (non-Premium accounts). Log this event at `warn` level in the client update handler so it's visible in logs without being actionable.
+
+---
+
+## Section 3: Per-Account Premium Upload Limit
+
+### Premium Detection
+
+After successful authentication, call `getMe()` and read `is_premium: bool` from the returned `user` object. Store this on `TelegramAccount.isPremium` (new boolean field, default `false`, updated on each successful auth).
+
+```typescript
+const me = await client.invoke({ _: 'getMe' }) as { is_premium?: boolean };
+await updateAccountPremiumStatus(account.id, me.is_premium ?? false);
+```
+
+### Upload Size Limits
+
+| Account type | `maxUploadSize` | Effect |
+|---|---|---|
+| Premium | 3,950 MB | Parts ≤ 3.95 GB upload as-is; repack only for parts >3.95 GB (extremely rare) |
+| Non-Premium | 1,950 MB | Current behavior unchanged |
+
+Pass `maxUploadSize` into `processOneArchiveSet()` as a parameter (currently hardcoded as `MAX_UPLOAD_SIZE` at `worker.ts:1023` and in `archive/split.ts`).
+
+The `hasOversizedPart` check and `byteLevelSplit` call both use this value, so the repack step is effectively eliminated for Premium accounts in practice — no separate "skip repack" flag needed.
+
+### Migration
+
+```prisma
+model TelegramAccount {
+  // ... existing fields
+  isPremium Boolean @default(false)
+}
+```
+
+One migration, one new query `updateAccountPremiumStatus(accountId, isPremium)`.
+
+---
+
+## Files to Change
+
+| File | Change |
+|---|---|
+| `prisma/schema.prisma` | Add `isPremium Boolean @default(false)` to `TelegramAccount` |
+| `worker/src/db/queries.ts` | Add `updateAccountPremiumStatus()`, `createPackageStub()`, `updatePackageWithMetadata()` |
+| `worker/src/db/locks.ts` | Add `tryAcquireHashLock()`, `releaseHashLock()` |
+| `worker/src/tdlib/client.ts` | Call `getMe()` after auth, return `isPremium` from `createTdlibClient()` |
+| `worker/src/worker.ts` | Two-phase write, hash lock acquire/release, pass `maxUploadSize` per account |
+| `worker/src/archive/split.ts` | Accept `maxPartSize` parameter instead of hardcoded constant |
+| `worker/src/scheduler.ts` | Replace sequential loop with `Promise.allSettled()`, remove `withTdlibMutex` from ingest path |
+
+---
+
+## What Is Explicitly Out of Scope
+
+- Backfilling metadata on stub records (rare crash case, functional without it)
+- Download pre-fetching / pipeline parallelism within one account
+- Two separate worker containers (single container is sufficient)
+- Bot or app changes (worker-only)
@@ -0,0 +1,353 @@
+# Channel-Scan Skip Optimization — Design
+
+**Goal:** stop the worker from re-scanning channels and forum topics that haven't changed since the last scan, especially on restart. Reduce the per-cycle API call count for the Model Printing Emporium channel (1,086 forum topics) from ~1,000+ to ~50.
+
+**Non-goals:**
+- Replacing polling with event-driven ingestion (`updateNewMessage`). That's a separate, larger design (Phase 2 in the original brainstorm).
+- Surfacing per-channel scan history in the UI (also a separate, observability-only design).
+
+**Architecture sketch:**
+Add three persisted columns to `AccountChannelMap` and `TopicProgress`, plus one runtime `getChat`/`getForumTopicInfo` lookup before each scan. The new state survives restarts because it's in PostgreSQL; the lookup is a cheap TDLib local-cache call. Failure-retry semantics (`d99a506` + `901f32f`) must be preserved — a channel sitting on retryable `SkippedPackage` rows is never considered idle.
+
+---
+
+## Problem statement
+
+### Today's behavior
+
+Every ingestion cycle the worker walks every linked source channel for every authenticated account. For each channel/topic it calls TDLib's `searchChatMessages` paginated from `lastProcessedMessageId`. Even when nothing has changed since the previous scan:
+
+- One `searchChatMessages` call (sometimes paginated) is still made
+- For Model Printing Emporium, that's ~1,086 calls per cycle (one per forum topic)
+- The 1-second `apiDelayMs` between pages multiplies the cost
+- Most calls return zero new messages — the work is wasted
+
+The cost is most acute right after a restart: the worker boots, runs recovery, then issues 1,000+ effectively-empty calls before any productive work happens.
+
+### What we already track
+
+- `AccountChannelMap.lastProcessedMessageId` — highest processed message ID (per non-forum channel, per account)
+- `TopicProgress.lastProcessedMessageId` — same per forum topic
+- Both are advanced incrementally per archive set (`77aeb4c`)
+- Both are pulled back below failed messages by the `SkippedPackage` retry pass (`901f32f`)
+
+### What we don't track and want to add
+
+- When was the last scan?
+- Did the last scan find any archives, OR is there outstanding retry work?
+- How many cycles in a row have been totally idle?
+
+These let us skip the scan entirely when nothing has changed.
+
+---
+
+## High-level approach
+
+Three guards at the top of the per-channel and per-topic processing loops:
+
+1. **DB-persistent "skip if recently scanned and truly idle"** — checks `lastScannedAt`, `lastScanFoundArchives`, and a `retryableSkippedCount` query. If all three say "nothing new, nothing failing", skip without any TDLib call.
+
+2. **Adaptive backoff for cold channels** — `consecutiveEmptyScans` counter. After it crosses a threshold, scan only every Nth cycle. Reset to 0 whenever the channel is "not idle".
+
+3. **`chat.last_message.id` short-circuit** — if (1) and (2) don't skip but the channel's last server-side message ID matches our watermark, skip the `searchChatMessages` paginated call. This runs after the existing `SkippedPackage` retry pass, which pulls the watermark back below failures, so it correctly forces a scan when retries are pending.
+
+The retry pass from `901f32f` is preserved untouched — it runs in front of these guards and adjusts the watermark, so retries always happen.
+
+---
+
+## Schema changes
+
+### `AccountChannelMap` (worker/src/db/schema.prisma)
+
+```prisma
+model AccountChannelMap {
+  // ... existing fields ...
+  lastScannedAt          DateTime?
+  lastScanFoundArchives  Boolean   @default(false)
+  consecutiveEmptyScans  Int       @default(0)
+}
+```
+
+### `TopicProgress`
+
+```prisma
+model TopicProgress {
+  // ... existing fields ...
+  lastScannedAt          DateTime?
+  lastScanFoundArchives  Boolean   @default(false)
+  consecutiveEmptyScans  Int       @default(0)
+}
+```
+
+### Migration
+
+```sql
+-- Both tables get the same three columns. Existing rows get defaults:
+--   lastScannedAt        = NULL  (next scan will populate)
+--   lastScanFoundArchives = false (safe default — will be overwritten by next scan)
+--   consecutiveEmptyScans = 0    (resets backoff for existing channels)
+
+ALTER TABLE "account_channel_map"
+  ADD COLUMN "lastScannedAt" TIMESTAMP(3),
+  ADD COLUMN "lastScanFoundArchives" BOOLEAN NOT NULL DEFAULT false,
+  ADD COLUMN "consecutiveEmptyScans" INTEGER NOT NULL DEFAULT 0;
+
+ALTER TABLE "topic_progress"
+  ADD COLUMN "lastScannedAt" TIMESTAMP(3),
+  ADD COLUMN "lastScanFoundArchives" BOOLEAN NOT NULL DEFAULT false,
+  ADD COLUMN "consecutiveEmptyScans" INTEGER NOT NULL DEFAULT 0;
+```
+
+NULL `lastScannedAt` means "never scanned" — every channel will be scanned the first cycle after deploy. Subsequent cycles benefit from the new fields.
+
+---
+
+## Configuration
+
+Two new env vars in `worker/src/util/config.ts`:
+
+```typescript
+/** Window in which a recent successful empty scan lets us skip. Default 5 min. */
+skipRecentScanWindowMs:
+  parseInt(process.env.WORKER_SKIP_RECENT_SCAN_WINDOW_MS ?? "300000", 10),
+
+/** After this many consecutive empty scans, channel enters backoff mode. */
+emptyScanBackoffThreshold:
+  parseInt(process.env.WORKER_EMPTY_SCAN_BACKOFF_THRESHOLD ?? "5", 10),
+
+/** Backoff factor — N means "scan every Nth cycle once in backoff". */
+emptyScanBackoffEveryNth:
+  parseInt(process.env.WORKER_EMPTY_SCAN_BACKOFF_EVERY_NTH ?? "5", 10),
+```
+
+All three are tunable per deployment without code changes.
+
+---
+
+## Decision logic per channel / topic
+
+The skip decision sits at the top of each channel/topic iteration in `runWorkerForAccount`. It runs BEFORE the existing `SkippedPackage` retry pass.
+
+```text
+For each channel (or topic):
+
+  1. Query retryableSkippedCount for this scope (already a query we do elsewhere)
+
+  2. If retryableSkippedCount > 0:
+       Force scan (don't skip — failures need retry)
+       Proceed to existing flow (retry pass → scan)
+
+  3. Else if lastScannedAt is NULL:
+       Force scan (we've never touched this)
+       Proceed to existing flow
+
+  4. Else if Date.now() - lastScannedAt.getTime() < skipRecentScanWindowMs
+          AND lastScanFoundArchives === false:
+       Skip — recently scanned and truly idle
+
+  5. Else if consecutiveEmptyScans >= emptyScanBackoffThreshold
+          AND (cycleCount % emptyScanBackoffEveryNth !== 0):
+       Skip — channel is cold, not its turn to scan
+
+  6. Else:
+       Run the existing flow:
+         a. SkippedPackage retry pass (901f32f) — may pull watermark back
+         b. NEW: getChat (or getForumTopicInfo) — if last_message.id <= watermark, skip
+         c. searchChatMessages scan
+```
+
+`cycleCount` is the global ingestion-cycle counter from `scheduler.ts`. It already increments per cycle.
+
+---
+
+## End-of-scan bookkeeping
+
+After every scan (whether it found archives or not), update the three new fields atomically with the existing watermark write:
+
+```typescript
+// "Truly idle" means: nothing new this scan AND nothing failed AND no leftover
+// retryable failures. The retry-pending check is critical — without it, a
+// scan that found no new archives but left SkippedPackage retries pending
+// would be marked idle and incorrectly skipped next cycle.
+const retryablePending = await getRetryableSkippedMessageIds({
+  accountId, sourceChannelId, topicId, cap: maxSkipAttempts,
+});
+const trulyIdle =
+  scanResult.archives.length === 0
+  && minFailedId === null
+  && retryablePending.length === 0;
+
+const newConsecutiveEmpty = trulyIdle
+  ? (prev.consecutiveEmptyScans ?? 0) + 1
+  : 0;
+
+await upsertChannelOrTopicScanState({
+  // ... existing watermark fields ...
+  lastScannedAt: new Date(),
+  lastScanFoundArchives: !trulyIdle,
+  consecutiveEmptyScans: newConsecutiveEmpty,
+});
+```
+
+The `consecutiveEmptyScans` counter resets to 0 the moment *anything* happens — archives found, archives failed, or unresolved retries pending. A channel with a chronically-failing archive (whose attemptCount is still below the cap) keeps the counter at 0 and never enters backoff.
+
+If a SkippedPackage hits `attemptCount === maxSkipAttempts`, it's no longer "retryable pending" (it's been given up on), so the counter increments correctly. Same for SkippedPackages that get deleted via the UI's "retry" button — the counter behaves correctly without special-casing.
+
+---
+
+## `getChat` / `getForumTopicInfo` short-circuit
+
+After the retry pass has finalized the effective watermark, but before `searchChatMessages`:
+
+```typescript
+// For non-forum channels:
+const chat = await client.invoke({ _: "getChat", chat_id: Number(channel.telegramId) });
+const channelLastMessageId = chat.last_message?.id;
+
+if (channelLastMessageId && BigInt(channelLastMessageId) <= effectiveWatermark) {
+  // Nothing new server-side — skip the paginated search entirely.
+  // Still update lastScannedAt / consecutiveEmptyScans so the recent-scan
+  // skip kicks in next cycle.
+  await persistScanState({ trulyIdle: true });
+  continue;
+}
+
+// For forum topics:
+const topicInfo = await client.invoke({
+  _: "getForumTopicInfo",
+  chat_id: Number(channel.telegramId),
+  message_thread_id: Number(topic.topicId),
+});
+const topicLastMessageId = topicInfo.info?.last_message_id;
+
+if (topicLastMessageId && BigInt(topicLastMessageId) <= effectiveWatermark) {
+  await persistScanState({ trulyIdle: true });
+  continue;
+}
+```
+
+`getChat` is served from TDLib's local cache (no network) for chats we've already loaded, which we do up front via `loadChats`. `getForumTopicInfo` is a single round-trip but much cheaper than a paginated `searchChatMessages` call.
+
+The comparison is `<=` because the watermark is the highest message we've fully processed — if the server's last is the same, we're caught up.
+
+This step is correct in the failure-retry case because the retry pass runs FIRST: if there were retryable failures, the retry pass pulled the watermark back below them, and `channelLastMessageId > effectiveWatermark` (since the failed message exists in TG), so we don't skip — we scan and re-pick-up the failure.
+
+---
+
+## Restart behavior
+
+The improvements compose for restart safety:
+
+| Scenario | Today | After this change |
+|---|---|---|
+| Restart 5 min after a clean cycle | ~2,000 API calls for MPE | ~10 calls (only retryable + truly-active topics) |
+| Restart 1 hour later (one missed cycle) | ~2,000 API calls | `getChat` per channel + scan only those where `last_message.id > watermark` (≈ 50 for MPE) |
+| Restart after long downtime (12h) | ~2,000 calls + lots of new content | `getChat` per channel, scan everything with new activity |
+
+The three new columns are in PostgreSQL — they survive container restarts directly. `consecutiveEmptyScans = 47` for a cold topic stays at 47 across restart, so backoff continues to apply.
+
+---
+
+## Edge cases and their handling
+
+### 1. Manual SkippedPackage retry via UI between cycles
+The UI's `retrySkippedPackageAction` lowers the watermark and deletes the SkippedPackage. Next cycle: `retryableSkippedCount === 0` (the row is gone), but the watermark is lower than `chat.last_message.id` (the retried message exists in TG). So step 6 in the decision tree triggers a scan via the `getChat` check. ✓
+
+### 2. SkippedPackage hits the attempt cap mid-cycle
+Once `attemptCount === maxSkipAttempts`, the row is no longer in `getRetryableSkippedMessageIds` results. The channel correctly becomes idle-eligible. The capped SkippedPackage stays in the table as "permanently failed (manual retry only)" — that's the existing behavior. ✓
+
+### 3. New SkippedPackage is created mid-cycle (e.g., an upload fails)
+At the end of that scan, `retryablePending` includes the new row → `trulyIdle = false` → `lastScanFoundArchives = true` → next cycle does NOT skip. ✓
+
+### 4. Channel/topic added after deploy
+New rows in `AccountChannelMap` / `TopicProgress` have `lastScannedAt = NULL`, so step 3 in the decision tree always triggers a scan. After the first scan, the fields are populated normally. ✓
+
+### 5. Clock skew / drift
+The `lastScannedAt < 5 min ago` check uses `Date.now() - lastScannedAt.getTime()`. Both are application-side clocks (Node.js + PostgreSQL `NOW()` at write). A few seconds of drift doesn't matter; an hour of clock jump (rare but possible) just means one cycle either skips or re-scans — recoverable.
+
+### 6. TDLib `getChat` returns stale data
+TDLib's local cache could theoretically be stale (e.g., the account hasn't received the latest update yet). If `channelLastMessageId` is stale (lower than server reality), we'd skip a scan that should have happened. Mitigation: the next cycle's `getChat` likely has fresh data; the watermark guards correctness (we don't lose data, we just process it one cycle later). Acceptable.
+
+### 7. `getForumTopicInfo` rate limit
+Calling it per-topic could add up for channels with 1000+ topics. Mitigation: skip-on-recent-scan (step 4) eliminates the call for most topics; only "stale-but-was-active" topics get the call. Worst case is ~50 calls per cycle for MPE, comfortably under the 30 req/sec global limit.
+
+### 8. Channel becomes a forum (or vice versa) between cycles
+Existing code handles this — `isChatForum` is rechecked each cycle and `setChannelForum` updates the DB. The new fields live on the same rows, so no extra handling needed.
+
+---
+
+## File-level changes
+
+### New / modified
+
+- `prisma/schema.prisma` — add the six new columns
+- `prisma/migrations/<timestamp>_channel_scan_state/migration.sql` — the ALTER TABLE
+- `worker/src/util/config.ts` — three new env vars
+- `worker/src/db/queries.ts` — new helpers:
+  - `getChannelScanState(mappingId)` and `getTopicScanState(topicProgressId)`
+  - `upsertChannelScanState(...)` and `upsertTopicScanState(...)`
+  - Both wrap the existing `updateLastProcessedMessage` / `upsertTopicProgress` so callers don't need to remember to update the new fields too.
+- `worker/src/worker.ts` — top-of-loop skip checks in both the forum and non-forum branches, plus end-of-scan state writes
+- `worker/src/tdlib/chats.ts` — small helper `getChatLastMessageId(client, chatId)` and `getForumTopicLastMessageId(client, chatId, topicId)` wrapping the TDLib calls with the existing `invokeWithTimeout` pattern
+
+### Untouched
+
+- `recovery.ts` — recovery is per-startup and one-shot; not affected
+- `scheduler.ts` — `cycleCount` is already there; just expose it where needed
+- The existing `SkippedPackage` retry pass logic in `runWorkerForAccount` is unchanged
+
+---
+
+## Testing plan
+
+The project has no automated tests, so verification is manual via Docker logs after deploy:
+
+1. **Build cleanly:** `docker compose up -d --build worker` — no migration errors
+2. **First cycle after deploy:** all channels scan (NULL `lastScannedAt`), all fields populated at end of cycle. Log lines confirm normal scan flow.
+3. **Second cycle 5 min later:**
+   - Check logs for `"Skipping recently-scanned idle channel"` — should appear for any channel/topic that was empty last cycle
+   - Total `searchChatMessages` calls per cycle should drop dramatically (compare to first cycle)
+4. **Failure-retry preservation:**
+   - Find a SkippedPackage with `attemptCount < cap`
+   - Run a cycle — confirm the channel/topic is NOT skipped (log says it's scanned)
+   - Confirm the SkippedPackage gets re-tried
+5. **Backoff:**
+   - Pick a cold channel, wait for it to scan 5+ cycles cleanly
+   - Confirm `consecutiveEmptyScans` climbs to 5+
+   - Confirm subsequent cycles skip it (only scan every 5th)
+6. **`getChat` short-circuit:**
+   - Pick an active channel
+   - Trigger an immediate cycle (UI button)
+   - If `last_message.id <= watermark`, expect log `"Channel caught up via getChat — skipping searchChatMessages"`
+7. **Restart safety:**
+   - Push the change, restart worker
+   - First cycle after restart should log multiple "Skipping recently-scanned idle channel" lines (because the DB state survived)
+   - Total cycle time should be a fraction of a baseline restart
+
+---
+
+## Risks and mitigations
+
+| Risk | Mitigation |
+|---|---|
+| Skip incorrectly applied → real failures never retried | Rule 1 (truly-idle includes `retryablePending === 0`) + dedicated test step 4 |
+| `getChat` returns stale data | Next cycle's `getChat` corrects it; watermark guards correctness (no data loss) |
+| `getForumTopicInfo` not available in TDLib 1.8.64 | Verify the method exists in the schema; fall back to scan if it throws |
+| Backoff applies during legitimate activity bursts | Counter resets to 0 the moment any archive is found OR any retry is pending |
+| Migration takes too long on the live DB | Both columns have NOT NULL defaults — Postgres can add them as fast metadata changes (no table rewrite) |
+
+---
+
+## What's explicitly NOT in this design
+
+To keep scope tight:
+- **Event-driven ingestion via `updateNewMessage`.** Bigger design, addressed separately. This design is compatible with it — when (D) lands, polling becomes a 4-hour safety net using these same skip rules.
+- **Per-channel scan history UI.** Observability layer; separate design.
+- **Surfacing the new counters in the admin dashboard.** Can come after the worker-side change is verified.
+- **Backfilling `consecutiveEmptyScans` from historical `IngestionRun` data.** Not worth it — it'll converge to the correct value within ~6 cycles.
+
+---
+
+## Open questions
+
+None — the failure-retry interaction was the main risk and is handled by Rule 1 + the existing retry pass.
@@ -0,0 +1,170 @@
+# Send All From Creator — Design
+
+**Date:** 2026-07-04
+**Status:** Approved for planning
+
+## Summary
+
+Add a "Send all from [Creator]" button to the STL packages page that queues every
+sendable package from a given creator for delivery to the user's Telegram. The
+button appears in the packages-tab toolbar only when the list is filtered by a
+single creator (`?creator=<name>`).
+
+This reuses the existing single-send infrastructure (`BotSendRequest` +
+`pg_notify('bot_send', id)` + the bot's `send-listener`) and closely mirrors the
+existing `sendAllInGroupAction`. No bot, DB schema, or send-listener changes are
+required.
+
+## Context
+
+Relevant existing pieces:
+
+- **Creator field:** `Package.creator` is a nullable, indexed `String` on the
+  `Package` model (`prisma/schema.prisma:476-524`). Filtering by exact creator is
+  cheap.
+- **Creator filter:** The packages list already supports `?creator=<name>`
+  (`src/app/(app)/stls/page.tsx:22,38`), rendered by the packages column as a
+  clickable value (`package-columns.tsx`).
+- **Single send:** `send-to-telegram-button.tsx` → `POST /api/telegram/bot/send`
+  creates a `BotSendRequest` (status `PENDING`) and fires
+  `pg_notify('bot_send', requestId)`; the button then polls the request to
+  completion.
+- **Existing bulk send (group):** `sendAllInGroupAction(groupId)` in
+  `src/app/(app)/stls/actions.ts:515-591` fetches the group's packages, filters to
+  those with `destChannelId` + `destMessageId`, skips any with an existing
+  `PENDING`/`SENDING` request for the same package + telegram link, creates a
+  `BotSendRequest` per remaining package, and fires `pg_notify` per package. It is
+  wired into the table via `handleSendAllInGroup` (`stl-table.tsx:264-278`) with a
+  `confirm()` dialog and a success toast — no polling.
+- **Bot side:** `bot/src/send-listener.ts` processes every `BotSendRequest`
+  uniformly; nothing there needs to change.
+
+## Requirements
+
+1. When the packages tab is filtered by a single creator (`?creator=<name>` is
+   present), show a toolbar button labeled **"Send all from [Creator]"**.
+2. When no creator filter is active, the button is not rendered.
+3. Clicking the button shows a confirmation dialog before sending (consistent with
+   the single-send and group-send flows, since this pushes files to Telegram).
+4. On confirm, queue **all** sendable packages from that creator across all pages —
+   not just the current page.
+5. Skip packages that are not yet uploaded (missing `destChannelId` /
+   `destMessageId`) and packages that already have a `PENDING`/`SENDING` request for
+   the same package + telegram link (dedup).
+6. Report real counts back to the user via toast, e.g.
+   "Queued 12 packages from [Creator]" and, when applicable, a skipped count.
+7. No polling — the bulk action queues and returns; sends process in the background
+   via the bot listener (same behavior as group send).
+
+## Design
+
+### Server action — `sendAllFromCreatorAction(creatorName: string)`
+
+New action in `src/app/(app)/stls/actions.ts`, mirroring `sendAllInGroupAction`:
+
+1. Auth check (`auth()`), return `Unauthorized` if no session.
+2. Load the user's `TelegramLink`; if none, return
+   "No linked Telegram account. Link one in Settings."
+3. Reject an empty/blank `creatorName` (guard against sending the entire "no
+   creator" set — see Edge cases).
+4. Fetch sendable packages directly with a `where` filter (no in-memory filtering):
+   ```ts
+   prisma.package.findMany({
+     where: {
+       creator: creatorName,
+       destChannelId: { not: null },
+       destMessageId: { not: null },
+     },
+     select: { id: true },
+   })
+   ```
+5. If none, return "No uploaded packages found for this creator."
+6. For each package, skip if an existing `PENDING`/`SENDING` `BotSendRequest`
+   exists for that package + telegram link; otherwise create a `BotSendRequest`
+   (status `PENDING`) and fire `pg_notify('bot_send', id)` (best-effort, wrapped in
+   try/catch — the bot also polls).
+7. `revalidatePath("/stls")`.
+8. Return counts so the UI can show them. This is a small improvement over
+   `sendAllInGroupAction`, which currently returns `{ success: true, data: undefined }`.
+   Shape:
+   ```ts
+   { success: true, data: { queued: number, skipped: number } }
+   ```
+   where `skipped` counts packages that already had a live request. (Packages not
+   yet uploaded are excluded by the query and not counted as skipped.)
+
+### UI — toolbar button in `stl-table.tsx`
+
+- Derive the active creator from the URL: `const activeCreator =
+  searchParams.get("creator") ?? "";` (parallel to the existing
+  `activeTag` at `stl-table.tsx:445`).
+- In the packages-tab toolbar (`stl-table.tsx:478` `flex flex-wrap items-center
+  gap-2` row), render the button only when `activeCreator` is non-empty:
+  ```tsx
+  {activeCreator && (
+    <Button variant="outline" size="sm" className="h-9 gap-1.5"
+            onClick={handleSendAllFromCreator}>
+      <Send className="h-3.5 w-3.5" />
+      Send all from {activeCreator}
+    </Button>
+  )}
+  ```
+  (Reuse the icon already used by the send action; place near the Upload / Group
+  buttons.)
+- Add `handleSendAllFromCreator`, modeled on `handleSendAllInGroup`
+  (`stl-table.tsx:264-278`):
+  ```ts
+  const handleSendAllFromCreator = useCallback(() => {
+    if (!confirm(`Send all packages from "${activeCreator}" to your Telegram?`)) return;
+    startTransition(async () => {
+      const result = await sendAllFromCreatorAction(activeCreator);
+      if (result.success) {
+        const { queued, skipped } = result.data;
+        toast.success(
+          `Queued ${queued} package${queued === 1 ? "" : "s"} from ${activeCreator}` +
+          (skipped ? ` (${skipped} already queued)` : "")
+        );
+        router.refresh();
+      } else {
+        toast.error(result.error);
+      }
+    });
+  }, [activeCreator, router]);
+  ```
+- Import `sendAllFromCreatorAction` alongside the existing `sendAllInGroupAction`
+  import (`stl-table.tsx:51`).
+
+### Data flow
+
+```
+User (filtered by creator) → clicks button → confirm dialog
+  → sendAllFromCreatorAction(creatorName)
+    → for each sendable, un-queued package: create BotSendRequest + pg_notify
+  → returns { queued, skipped } → toast + router.refresh()
+
+(background) bot send-listener consumes each BotSendRequest → delivers to user
+```
+
+## Edge cases
+
+- **Blank creator:** The action rejects an empty/whitespace `creatorName` so a
+  stray `?creator=` never queues every package. The UI already gates the button on
+  a non-empty `activeCreator`, so this is defense-in-depth.
+- **No linked Telegram account:** Returns an error toast, queues nothing.
+- **No uploaded packages for creator:** Returns an error toast, queues nothing.
+- **All packages already queued:** `queued: 0`, `skipped: N` → toast reflects it.
+- **pg_notify failure:** Swallowed per-package (best-effort); the bot's periodic
+  poll picks the request up. Matches existing behavior.
+
+## Out of scope / non-goals
+
+- No per-row "send all from this creator" action (toolbar-only, per decision).
+- No progress polling / live completion tracking for the bulk send.
+- No changes to the bot, `send-listener`, `BotSendRequest` schema, or API routes.
+- No batching/throttling beyond what the bot listener already does.
+
+## Files touched
+
+- `src/app/(app)/stls/actions.ts` — add `sendAllFromCreatorAction`.
+- `src/app/(app)/stls/_components/stl-table.tsx` — derive `activeCreator`, add
+  `handleSendAllFromCreator`, render the toolbar button, import the new action.
@@ -0,0 +1,205 @@
+# NAS Backup for Postgres + TDLib State — Design
+
+**Date:** 2026-07-23
+**Status:** Approved for planning
+
+## Summary
+
+Add a dedicated `backup` container to the DragonsStash stack that takes daily,
+encrypted, deduplicated backups of the two things that can't be regenerated —
+the Postgres database (inventory/STL metadata, users, everything the app
+manages) and the two TDLib state volumes (Telegram session/auth state for the
+worker and bot) — and ships them to a Synology NAS over NFS. STL archive
+contents themselves are explicitly out of scope: they only live on this host
+temporarily and are not backed up.
+
+Backups are stored via [restic](https://restic.net/), which provides
+encryption-at-rest, block-level dedup, and retention pruning natively, so no
+custom encryption or pruning scripts need to be written or maintained.
+
+## Context
+
+Current state (as of this design):
+
+- Production stack runs from `/opt/stacks/DragonsStash/docker-compose.yml` on
+  this Dockge-managed host, pulling prebuilt images from
+  `git.samagsteribbe.nl`. The `docker-compose.yml` in this repo is the
+  build/dev reference and should be kept in sync.
+- Named volumes in use: `postgres_data` (Postgres 16 data directory),
+  `tdlib_state` (worker's TDLib session), `tdlib_bot_state` (bot's TDLib
+  session), `tmp_zips` and `manual_uploads` (both transient, explicitly out of
+  scope here).
+- No backup mechanism, NFS mount, or host cron currently exists anywhere in
+  this deployment.
+- The host already runs Uptime Kuma (used here for backup alerting) and Loki
+  (container logs are presumably already collected there).
+
+## Requirements
+
+1. Daily backup of the Postgres database and both TDLib state volumes.
+2. Backups stored on a Synology NAS via NFS, not on local disk.
+3. 14-day retention, oldest snapshots pruned automatically.
+4. Backups encrypted at rest (Postgres dumps and TDLib session files both
+   contain sensitive material — password hashes, Telegram API secrets, live
+   session state).
+5. Postgres backups must be transactionally consistent regardless of live app
+   traffic. TDLib state backups are best-effort (see Decisions below) — this
+   is an accepted trade-off, not a defect.
+6. Alert (via existing Uptime Kuma) if a backup run fails or doesn't happen.
+7. No new host-level state (no `/etc/fstab` entries, no host crontab) — the
+   backup mechanism should be a container, consistent with how everything
+   else on this host is deployed and versioned.
+8. No new privileged access — specifically, the backup container must not
+   have Docker socket access or any ability to control sibling containers.
+
+## Decisions
+
+- **NFS mounted via Docker's native NFS volume driver** (`driver_opts: type:
+  nfs`), not a host-level mount. Keeps all backup-related state inside the
+  compose file instead of split across host config.
+- **Restic, not hand-rolled tar+age+find.** Restic already solves encryption,
+  dedup, and retention correctly; hand-rolled scripts would be reinventing
+  that logic with more room for bugs.
+- **TDLib state is tarred live (best-effort), not paused.** Pausing the
+  worker/bot for a clean snapshot would require mounting the Docker socket
+  into the backup container so it could stop/start sibling containers — a
+  real privilege escalation (a compromised backup container could then
+  control any container on the host). The downside of a best-effort tar is
+  bounded: worst case, a bad TDLib restore means redoing the Telegram SMS
+  auth flow, which is the same outcome as having no backup at all. That
+  bounded, low-severity downside doesn't justify the privilege escalation.
+- **Fixed-time cron (`crond`), not a sleep-loop.** A `sleep 86400` loop drifts
+  on every container restart; a real crontab entry fires at a fixed time of
+  day regardless of restarts, for negligible extra complexity.
+- **Restore is manual, not automated.** A script capable of restoring can
+  overwrite live state; that should always require a human deliberately
+  running it, not run unattended.
+
+## Design
+
+### New service: `backup`
+
+Added to both `/opt/stacks/DragonsStash/docker-compose.yml` (production) and
+this repo's `docker-compose.yml` (dev/build reference).
+
+- **Image**: custom, `FROM alpine:3.20`, `apk add --no-cache restic
+  postgresql16-client curl tzdata dcron tar bash`. No dependency on app
+  source — independent Dockerfile, e.g. `backup/Dockerfile`.
+- **Scheduling**: `crond -f` in the foreground as the container's entrypoint,
+  with a crontab installed at build time:
+  ```
+  0 3 * * * /backup.sh >> /proc/1/fd/1 2>&1
+  0 4 * * 0 restic check >> /proc/1/fd/1 2>&1
+  ```
+  (daily dump/backup at 03:00, weekly repo integrity check at 04:00 Sunday).
+  `restic init` runs once at container startup (entrypoint, before `crond`
+  starts), swallowing the "already initialized" error on subsequent
+  container (re)starts.
+- **Network**: `internal` only — reaches `dragonsstash-db:5432` for
+  `pg_dump`. No ports exposed.
+- **Volumes**:
+  - `tdlib_state:/data/tdlib-worker:ro`
+  - `tdlib_bot_state:/data/tdlib-bot:ro`
+  - `nas_backups:/backups`, a named volume defined with:
+    ```yaml
+    nas_backups:
+      driver_opts:
+        type: nfs
+        o: "addr=${NAS_HOST},rw,nfsvers=4,soft,timeo=100"
+        device: ":${NAS_EXPORT_PATH}"
+    ```
+- **New `.env` entries**: `NAS_HOST`, `NAS_EXPORT_PATH` (NFS share details),
+  `RESTIC_PASSWORD` (repo encryption key), `KUMA_PUSH_URL` (Uptime Kuma push
+  monitor URL). All four are inputs to gather during implementation, not
+  hardcoded.
+- `restart: unless-stopped`, no `privileged`, no Docker socket mount.
+
+### `backup.sh`
+
+```
+set -euo pipefail
+trap 'curl -fsS "$KUMA_PUSH_URL" --get --data-urlencode "status=down" \
+  --data-urlencode "msg=$BASH_COMMAND failed"' ERR
+
+pg_dump -h dragonsstash-db -U "$POSTGRES_USER" -d "$POSTGRES_DB" \
+  -Fc -f /tmp/dragonsstash.dump
+
+tar czf /tmp/tdlib.tar.gz -C /data tdlib-worker tdlib-bot
+
+restic backup /tmp/dragonsstash.dump /tmp/tdlib.tar.gz
+restic forget --keep-daily 14 --prune
+
+rm -f /tmp/dragonsstash.dump /tmp/tdlib.tar.gz
+
+curl -fsS "$KUMA_PUSH_URL" --get --data-urlencode "status=up" \
+  --data-urlencode "msg=OK"
+```
+
+`PGPASSWORD` and `RESTIC_REPOSITORY=/backups/restic-repo` are set as
+container environment variables (from `.env`), not inline in the script.
+
+### Data flow
+
+```
+crond (daily 03:00)
+  → pg_dump (consistent snapshot via Postgres MVCC) → /tmp/dragonsstash.dump
+  → tar tdlib_state + tdlib_bot_state (best-effort, live) → /tmp/tdlib.tar.gz
+  → restic backup (encrypt + dedup) → NFS-mounted repo on Synology NAS
+  → restic forget --keep-daily 14 --prune
+  → curl Uptime Kuma push monitor (up on success, down + reason on any failure)
+```
+
+### Restore (manual, documented procedure — not scripted/automated)
+
+```
+restic -r /backups/restic-repo restore latest --target /tmp/restore
+pg_restore -h dragonsstash-db -U "$POSTGRES_USER" -d "$POSTGRES_DB" \
+  --clean --if-exists /tmp/restore/tmp/dragonsstash.dump
+# untar /tmp/restore/tmp/tdlib.tar.gz back into the tdlib_state /
+# tdlib_bot_state volumes (via a throwaway container mounting both)
+```
+
+## Alerting
+
+- One Uptime Kuma **Push** monitor, created manually in the existing Kuma
+  instance, with an expected heartbeat interval of ~26 hours (slack past the
+  24h schedule so one slow run doesn't false-positive). Whatever notification
+  channels are already configured on that monitor fire automatically — no new
+  alerting integration.
+- `backup.sh` pushes `status=up` on success and `status=down` (with the
+  failing command in `msg`) on any failure, via the `ERR` trap.
+- Container logs go to stdout, collected the same way every other container's
+  logs already are on this host.
+
+## Testing
+
+This repo has no automated test framework (documented convention: manual
+testing). For this infra change:
+
+- After deploy: manually run `docker exec dragonsstash-backup /backup.sh`
+  once, confirm a snapshot appears (`restic snapshots`), confirm the Kuma
+  monitor goes green.
+- **Restore drill** (once, during setup): actually restore the dump into a
+  scratch Postgres and untar the TDLib archive into scratch volumes, to prove
+  the backup is really restorable. Not automated or recurring for now.
+
+## Out of scope / non-goals
+
+- Backing up `tmp_zips` or `manual_uploads` — both transient by design.
+- Automated/scheduled restore testing.
+- Backing up any other stack on this host (this design is DragonsStash-only,
+  though the pattern — Docker-native NFS volume + restic — could be reused
+  for other stacks later).
+- Pausing worker/bot for a guaranteed-consistent TDLib snapshot (see
+  Decisions).
+
+## Files touched
+
+- `docker-compose.yml` (this repo) and
+  `/opt/stacks/DragonsStash/docker-compose.yml` (production) — add `backup`
+  service, `nas_backups` volume.
+- `backup/Dockerfile` — new.
+- `backup/backup.sh` — new.
+- `backup/crontab` — new.
+- `.env.example` / `.env` — add `NAS_HOST`, `NAS_EXPORT_PATH`,
+  `RESTIC_PASSWORD`, `KUMA_PUSH_URL`.
@@ -0,0 +1,271 @@
+# Provenance Backfill on Re-Index — Design
+
+**Date:** 2026-07-23
+**Status:** Approved for planning
+
+## Summary
+
+Recover the true origin of packages whose recorded "source" is a placeholder —
+specifically the manual-upload and `rebuild.ts`-created records whose
+`sourceChannelId` points at the destination (archive) channel itself. When a
+real source channel is re-indexed and the worker encounters an archive that
+matches such a placeholder package, it backfills the real
+`sourceChannelId` / `sourceMessageId` / `sourceTopicId` / `sourceCaption` /
+`creator` (and, for records that never had one, the file listing and preview)
+onto the existing package — **without downloading the full archive**.
+
+Matching is a two-stage process: a zero-download candidate lookup by
+`fileName` + total `fileSize`, confirmed by a CRC32 fingerprint read from just
+the archive's central directory via a **ranged (tail) download** of a few
+KB–MB, instead of the multi-GB whole file.
+
+This is the first of four linked sub-projects. The others (creator
+normalization, provenance display, missing-files) are out of scope here and get
+their own spec → plan cycles. "Missing files" is explicitly deferred until this
+lands.
+
+## Context
+
+Current state (as of this design):
+
+- `Package` records their origin via `sourceChannelId`, `sourceMessageId`,
+  `sourceTopicId`, `sourceCaption`, and (for content dedup) `contentHash` and
+  `remoteUniqueId`. `creator` is a free-text string extracted at ingestion.
+- Two ingestion paths create records with **placeholder provenance**, where the
+  "source" is the destination channel, not a real origin:
+  - **Manual uploads** (`worker/src/manual-upload.ts`) set
+    `sourceChannelId = destChannel.id` and `sourceMessageId = destResult.messageId`.
+    They *do* populate `PackageFile` (with `crc32`) from a local central-directory
+    read at upload time.
+  - **`rebuild.ts`** scans the destination channel and creates minimal records
+    with `fileCount == 0` and **no** `PackageFile` rows.
+- The main worker upload path (`worker/src/worker.ts` → `uploadToChannel`) sends
+  archives to the destination channel with **no caption**, so provenance cannot
+  be recovered by re-reading destination messages — it has to come from matching
+  against real source channels.
+- The scan/dedup ladder in `processOneArchiveSet` (`worker/src/worker.ts:1521`)
+  is entirely **source-channel-scoped**:
+  1. `findPackageByRemoteUniqueId(channel.id, …)` — same channel only
+  2. `packageExistsBySourceMessage(channel.id, …)` — same channel only
+  3. `findRepostedPackage(channel.id, fileName, size)` — same channel only
+  4. …then full download → `packageExistsByHash(contentHash)` — global, but only
+     *after* downloading the whole archive.
+- Because a placeholder package's `sourceChannelId` is the destination, checks
+  1–3 never match it when a real source channel is scanned. Today the worker
+  therefore downloads the entire archive, hits check 4, finds it is a duplicate,
+  and skips — **wasting the download and leaving the wrong provenance in place.**
+- There is already an in-production precedent for "match an already-known
+  archive during scan and enrich its metadata": when `findRepostedPackage`
+  matches, the worker backfills richer *topic* context onto the existing package
+  via `updatePackageTopicContext` (`worker/src/worker.ts:1608`+). Provenance
+  backfill is the same move, widened from same-channel to cross-channel.
+- `remote.unique_id` is **not** a reliable cross-channel key: it identifies a
+  stored file object on Telegram's servers, not the content. The same archive
+  independently uploaded to two channels gets different `unique_id`s; it only
+  matches for forwarded messages (same underlying file object). This is why the
+  existing dedup scopes it to a single channel.
+
+## Goals / Non-Goals
+
+**Goals**
+
+- Attribute true origin to placeholder-provenance packages during normal
+  re-index scans, opportunistically (no separate pass).
+- Do it without downloading whole archives (ranged central-directory read only).
+- Be non-destructive: only ever touch packages that currently have placeholder
+  provenance; never overwrite a real, non-placeholder source.
+- Be idempotent: re-running a re-index does not re-mutate or duplicate.
+
+**Non-Goals (separate sub-projects / deferred)**
+
+- Creator name normalization / canonical creator entity.
+- UI changes to display provenance.
+- Recovering "missing files."
+- Choosing a *preferred* origin when an archive genuinely exists in several
+  source channels (first confirmed source wins).
+
+## Design
+
+### 1. Candidate definition
+
+> A package is a backfill candidate iff **`sourceChannelId == destChannelId`
+> OR `sourceMessageId == 0`**.
+
+Two placeholder shapes exist (verified against live data 2026-07-23):
+- **Manual uploads** (`manual-upload.ts`): `sourceChannelId == destChannelId`,
+  real `contentHash`, real `sourceMessageId`, has a `PackageFile` listing.
+- **Rebuild records** (`rebuild.ts`): `sourceMessageId == 0n` (deliberate
+  "unknown" sentinel), synthetic `contentHash = "rebuild:<destChannelId>:<destMessageId>"`,
+  `fileCount == 0`, and `sourceChannelId` set to an **arbitrary fallback source
+  channel** (`sourceChannels[0]`) — NOT the destination. (This is the common
+  case: e.g. 59,893 records after a destination rebuild.)
+
+Normal ingestion always sets a real `sourceMessageId` (> 0) and a real source
+channel, so neither marker matches a genuinely-sourced package. Both markers are
+overwritten on backfill (source channel + message become real), so a record
+stops being a candidate once fixed — this is what makes re-scans idempotent.
+
+**Known limitation:** backfill does NOT rewrite a rebuild record's synthetic
+`"rebuild:"` `contentHash` (the true content hash would require a full download,
+which this feature avoids). That is acceptable — dedup after backfill relies on
+`remoteUniqueId` + name/size within the source channel, not on `contentHash`.
+
+### 2. Where it hooks
+
+A new step is inserted into `processOneArchiveSet` **between check #3
+(`findRepostedPackage`) and the full download**. It runs only after the
+same-channel checks have missed (so genuine same-channel reposts keep their
+existing fast paths).
+
+Flow for the scanned archive set:
+
+1. Stage A — **candidate lookup (zero download).** Query for a package where
+   `(sourceChannelId == destChannelId OR sourceMessageId == 0)` AND
+   `fileName == archiveName` AND `fileSize == totalArchiveSize`.
+   (`Package` has `@@index([fileName])`.)
+   - No candidate → fall through to normal ingestion unchanged.
+2. Stage B — **fingerprint confirmation (tiny download).** See §3.
+3. On confirmation → **backfill** (see §4) and return `null` (treated as a
+   duplicate; no full download, no new package). Increment a `zipsBackfilled`
+   counter.
+4. On mismatch / failure / ambiguity → fall through to normal ingestion (see §6).
+
+### 3. Fingerprint confirmation
+
+The confirmation signal is the **multiset of internal CRC32s** of the archive's
+entries (CRC32 of each entry's *uncompressed* data). This value is a property of
+the archive contents and is identical regardless of which channel hosts the
+file.
+
+- **Candidate side:** for manual uploads, `PackageFile.crc32` is already
+  populated from the local central-directory read — zero cost. For **rebuild
+  records** (`fileCount == 0`, no `crc32`), there is nothing stored to compare
+  against; obtain the candidate's fingerprint with a **second ranged tail read
+  of its destination copy** (ZIP/7z — still no full download). RAR candidates
+  cannot be tail-read on either side, so they fall back to name+size (see §5).
+- **Scanned-source side:** read via a **ranged (tail) download** of the central
+  directory:
+  - **ZIP** (incl. multipart): the End-of-Central-Directory record + central
+    directory live at the tail of the last part. Download only that tail and
+    parse entries. (Reuses the lightweight-listing mechanism that is
+    sub-project 4's core.)
+  - **7z**: a start header at byte 0 points to an end header at the tail; fetch
+    both small pieces.
+  - **RAR**: headers are scattered through the file — no cheap tail read. See §5.
+- **Match rule:** confirmed iff the sorted CRC32 multisets are equal **and** file
+  counts are equal.
+
+The comparison logic (CRC32 multiset equality) and the candidate-match predicate
+are implemented as **pure functions** with no TDLib dependency, so they are unit
+testable (see §7).
+
+### 4. What gets written
+
+On a confirmed match, update the candidate package in a single transaction,
+overwriting only placeholder/empty fields:
+
+| Field | New value | Condition |
+|---|---|---|
+| `sourceChannelId` | scanned `channel.id` | always |
+| `sourceMessageId` | scanned `parts[0].id` | always |
+| `sourceTopicId` | `ctx.sourceTopicId` | always |
+| `sourceCaption` | scanned message caption | always |
+| `remoteUniqueId` | scanned `firstRemoteUniqueId` | always (enables future same-channel dedup via check #1) |
+| `creator` | re-derived from source (topic > filename > channel) | **always** — un-normalized; the later creator-normalization sub-project cleans it up |
+| `fileCount` + `PackageFile[]` | from the central-directory read | only if candidate had none (`fileCount == 0`) — folds in the sub-project 4 outcome for rebuild records |
+| `previewData` / `previewMsgId` | matched preview from scan | only if candidate has none |
+
+**Left untouched:** `contentHash`, `destChannelId`, `destMessageId`,
+`destMessageIds` — the bytes physically live in the destination channel; that is
+correct and must not change.
+
+**Transaction safety:** re-check `sourceChannelId == destChannelId` *inside* the
+transaction before writing, so a concurrent worker that already backfilled the
+record causes this one to no-op (mirrors the existing `backfill.ts` guard).
+
+### 5. RAR handling
+
+RAR sources cannot be tail-read, so no cheap CRC fingerprint is available.
+Decision: **backfill RAR matches on `fileName` + `fileSize` alone**, flagged as
+lower-confidence in logs and via a `SystemNotification`, so they can be audited.
+No full download.
+
+This name+size-only fallback applies to any candidate that cannot produce a
+CRC32 fingerprint cheaply: a RAR **source**, a RAR **candidate**, or a rebuild
+candidate whose destination copy is RAR. ZIP/7z rebuild candidates are still
+confirmed by fingerprint via the second tail read described in §3.
+
+### 6. Conflict, mismatch, ambiguity
+
+- **Fingerprint mismatch** → the scanned archive is genuinely different content
+  that merely shares name+size. Fall through to **normal ingestion** (download +
+  index) — it is a new package for this source.
+- **Ambiguous candidates** (2+ match name+size and the fingerprint cannot
+  disambiguate) → log a `SystemNotification`, backfill nothing, and fall through
+  to normal ingestion; the post-download `packageExistsByHash` check still
+  dedups it safely.
+- **Same archive in multiple source channels** → the **first re-indexed source
+  that confirms wins.** After backfill the package has a real source, so it is no
+  longer a candidate; later scans treat it as an ordinary duplicate via checks #1
+  (the `remoteUniqueId` we set) or #3.
+
+### 7. Idempotency
+
+Falls out of the candidate definition. Once backfilled:
+- `sourceChannelId` is the real channel → no longer a candidate.
+- A re-scan of that source hits check #1 (`remoteUniqueId`, now set) or check #3
+  (`findRepostedPackage`) → normal dedup skip. No re-mutation, no duplicate.
+
+### 8. Error handling
+
+- **Tail-download failure** (network / `FLOOD_WAIT`) → cannot confirm this round.
+  Do **not** fall back to a full download and do **not** guess: leave the
+  candidate untouched and let the next re-index retry. Wrap the ranged read in
+  `withFloodWait` (per the TDLib skill).
+- All new TDLib calls follow the skill's patterns: `withFloodWait`, listener
+  attached before the async op, client closed in `finally`.
+
+### 9. Visibility
+
+- Add a `zipsBackfilled` counter to `IngestionRun` activity so a re-index run
+  reports "N provenance backfills" instead of silently mutating records.
+- Info log per backfill (candidate id, old vs new source, confidence:
+  fingerprint | name+size-RAR).
+- `SystemNotification` for ambiguous-candidate cases.
+
+## Testing
+
+The repo currently has no test framework (`CLAUDE.md`: "testing is manual"). This
+work introduces a **lightweight test harness** for the worker (e.g. `vitest` or
+node's built-in `node:test`) and unit-tests the correctness-critical pure logic:
+
+- **Unit (automated):**
+  - CRC32 multiset fingerprint equality (equal sets, different order, differing
+    counts, disjoint sets).
+  - Candidate-match predicate (name+size+placeholder true/false cases).
+  - Field-merge rules (which fields overwrite, which only fill-if-empty).
+- **Manual integration checklist:**
+  1. Re-index a real source channel containing a known manually-uploaded pack →
+     verify `sourceChannelId`/`sourceMessageId`/`sourceCaption`/`creator` are
+     backfilled and no full download occurs.
+  2. A rebuild-created record (`fileCount == 0`) → verify listing + provenance
+     are both populated from the single tail read.
+  3. A RAR pack → verify name+size backfill with the lower-confidence log/notice.
+  4. Re-run the same re-index → verify it is a no-op (idempotent).
+  5. A genuine name+size collision (different content) → verify it ingests as a
+     new package rather than being mis-attributed.
+
+## Open Questions
+
+None blocking. Preferred-origin selection among multiple real sources is
+deliberately out of scope (first confirmed wins).
+
+## Affected Code (indicative, for planning)
+
+- `worker/src/worker.ts` — `processOneArchiveSet`: new Stage A/B step; new counter.
+- `worker/src/archive/` — ranged central-directory reader (ZIP/7z tail); shared
+  with sub-project 4. Pure CRC32-fingerprint compare helper.
+- `worker/src/tdlib/download.ts` — ranged/partial download support (`offset`/`limit`).
+- `worker/src/db/queries.ts` — candidate lookup + transactional backfill update.
+- `prisma/schema.prisma` — `IngestionRun.zipsBackfilled` (and run-counter plumbing).
+- Worker test harness + first unit tests.
@@ -0,0 +1,180 @@
+# Ranged inner-file listing for RAR & 7z — design
+
+**Date:** 2026-07-27
+**Status:** Approved (design), pending spec review → implementation plan
+
+## Problem
+
+The reindex/provenance-backfill path (`worker/src/provenance-backfill.ts`) can index an
+archive's inner files *without* re-downloading it, by reading the file listing from a small
+ranged read of the copy already in the source/destination channel. This works **only for
+ZIP** today (ZIP keeps its central directory in a tail that `parseZipCentralDirectoryFromTail`
+reads). For **RAR and 7z**, `tryProvenanceBackfill` backfills provenance (creator, source
+channel, `remoteUniqueId`) so the file is skipped on re-scan and never re-downloaded — but it
+leaves the inner listing empty (`fileCount = 0`), because `scannedEntries` is only computed for
+ZIP.
+
+Scope of the gap (rebuild placeholders with `fileCount = 0`, as of 2026-07-27):
+
+| Type | Count | Total | Avg | Max | Multipart |
+|---|---|---|---|---|---|
+| 7z | 19,646 | 14 TB | 0.73 GB | 3.9 GB | 0 |
+| RAR | 12,780 | 13 TB | 1.04 GB | 116 GB | 1,016 |
+
+A "just download the whole archive" fallback for all of these means ~27 TB of re-downloads —
+the exact cost this path exists to avoid.
+
+## Goals
+
+- Index the inner files (names + sizes; CRCs where cheaply available) of RAR and 7z
+  placeholders **without** downloading the whole archive in the common case.
+- Reuse the existing, battle-tested CLI listing parsers (`parse7zOutput`,
+  `parseUnrarTechnical`) rather than reimplementing filename/size/CRC extraction.
+- Keep cost proportional to **file count**, not archive size (so even the 116 GB RAR is cheap).
+- Guarantee a listing for the rare archives the cheap path can't handle, via a full-download
+  fallback that respects the existing max-size guard.
+
+## Non-goals
+
+- No change to ingestion of genuinely-new files (those are downloaded in full to be re-uploaded
+  regardless, so a cheap listing does not help there). This feature only affects the
+  provenance-backfill / skip path.
+- No new ZIP behaviour — the existing ZIP tail reader stays as-is.
+- Not attempting to list password-encrypted-header archives from ranged reads (no password);
+  those take the fallback and, if oversized, are flagged.
+
+## Approach (chosen: "harvest header regions → sparse file → native CLI")
+
+Do the *minimum* binary parsing needed to locate an archive's header bytes, fetch only those
+via ranged reads, write them into a sparse temp file at their true offsets (data regions left
+as unwritten zero holes → ~no disk use), then run the real `7z l` / `unrar lt` and reuse the
+existing parsers. The native tools do the hard parsing (7z's LZMA-encoded headers, RAR's two
+format versions, Unicode names) — we only compute where the headers are.
+
+Rejected alternatives: full native TS parsers (most custom binary code, highest risk);
+RAR5-quick-open-only (most community RARs lack it → collapses to ~13 TB of RAR downloads).
+
+## Components
+
+New directory `worker/src/archive/ranged/`, one focused module per concern, all returning the
+existing `FileEntry[]` type from `zip-reader.ts`:
+
+- `sparse-list.ts` — `listFromSparse(parts, runner, parse) → FileEntry[] | null`, where each
+  `part` is `{ fileName, size, regions: {offset, bytes}[] }`. For each part it writes a sparse
+  temp file (`truncate` to `size`, then write only the header `regions` at their offsets),
+  co-locates all parts in one temp dir under their real names, invokes the supplied CLI runner
+  (`7z l` / `unrar lt`) on the first part, feeds stdout to the supplied `parse` fn
+  (`parse7zOutput` / `parseUnrarTechnical`), and cleans up. Single-part archives are just the
+  one-element case. Returns `null` on CLI error / empty parse.
+- `sevenz-ranged.ts` — `readSevenZListingRanged(client, parts) → FileEntry[] | null`.
+- `rar-ranged.ts` — `readRarListingRanged(client, parts) → FileEntry[] | null`.
+- Dispatcher in `provenance-backfill.ts`: `readScannedListingRanged(archiveType, client, parts)`
+  replacing the current `if (archiveType === "ZIP")` branch; the destination-copy read in
+  `resolveCandidateFingerprintEntries` gets the same dispatch.
+
+`FileEntry` shape (unchanged): `{ path, fileName, extension, compressedSize, uncompressedSize, crc32 }`.
+
+### 7z ranged listing
+
+7z layout: 32-byte signature header at offset 0 → packed streams → end header (lists files) at
+the end; the signature header stores the end header's location.
+
+1. Ranged-read `[0, 32)`; validate magic `37 7A BC AF 27 1C`. Read LE `uint64`
+   `NextHeaderOffset` (byte 12) and `NextHeaderSize` (byte 20). End header is at absolute offset
+   `32 + NextHeaderOffset`, length `NextHeaderSize`.
+2. Ranged-read `[32 + NextHeaderOffset, NextHeaderSize)` — the "next header".
+3. Branch on the next header's first byte (a 7z property id):
+   - **`0x01` (kHeader, plain/uncompressed header):** two regions suffice —
+     `{0: sigHeader}` and `{32+NextHeaderOffset: endHeader}`.
+   - **`0x17` (kEncodedHeader, LZMA-compressed header):** the next header is only a *descriptor*
+     whose `PackInfo` points at a packed header stream stored **in the middle** of the file (not
+     at EOF). Parse the descriptor's `StreamsInfo → kPackInfo (0x06)` to read `PackPos` and the
+     `PackSize`s (7z variable-length "numbers"; sum them). Ranged-read the contiguous packed
+     region `[32 + PackPos, Σ PackSize)` and add it as a **third** sparse region. `7z l` then
+     decodes the header from that region.
+   - **anything else:** return `null` (→ fallback).
+4. `listFromSparse` with the 2 or 3 regions, runner = `7z l`. `parse7zOutput` yields
+   names+sizes (`crc32: null`, as today). Return `null` on bad magic / read failure / CLI error.
+
+**Why the third region is required (spike finding, 2026-07-27):** the original two-region
+(start+end) reconstruction was proven insufficient in a live test — `7z l` rejected it with
+"Cannot open the file as [7z] archive" because these archives use an *encoded* header whose
+compressed bytes live in a packed stream in the file body (a sparse hole), not at EOF. The
+`0x17` branch fetches exactly that packed region. `7z l` still never touches the file-data
+packed streams (it only lists), so those gaps stay sparse. The `read7zNumber` reader (7z's
+base-128-ish variable-length integer with a first-byte length mask) and the minimal
+`kPackInfo` walk are the only new 7z binary parsing; `7z l` still does the actual file listing.
+All 7z placeholders are single-part.
+
+### RAR ranged listing
+
+RAR has no index; walk the block chain, parsing only each block's **size fields** to step
+forward and harvest header bytes.
+
+1. Read first ~16 bytes; detect **RAR4** (`52 61 72 21 1A 07 00`) vs **RAR5** (`…07 01 00`) and
+   the signature length.
+2. From just after the signature, loop:
+   - Ranged-read a header chunk (start 8 KB; if parsed `HeaderSize` exceeds it — long filenames
+     — re-read exactly).
+   - Minimal block-extent parse:
+     - RAR5: `CRC32(4)` + vint `HeaderSize` + vint `HeaderType` + vint `HeaderFlags`; if the
+       "extra area" flag (`0x0001`) → vint `ExtraAreaSize`; if the "data present" flag
+       (`0x0002`) → vint `DataSize`. Next block = `pos + 4 + len(HeaderSize vint) + HeaderSize
+       + DataSize`.
+     - RAR4: `HEAD_CRC(2)` + `HEAD_TYPE(1)` + `HEAD_FLAGS(2)` + `HEAD_SIZE(2)`; if flag `0x8000`
+       → `ADD_SIZE(4)`. Next block = `pos + HEAD_SIZE + ADD_SIZE`.
+   - Harvest `[blockOffset, blockOffset + HeaderSize)` into the regions list.
+   - Stop at the end-of-archive block or EOF.
+3. `listFromSparse` (headers present, data sparse) → `unrar lt` → `parseUnrarTechnical`. RAR
+   headers carry CRC32, so RAR contributes CRCs (fingerprint disambiguation keeps working).
+
+**Multipart RAR** (1,016): each volume starts with its own signature + headers. Walk **each
+part from its own signature**, reconstruct one sparse temp file per part with correct names
+(`name.part1.rar`, `.part2.rar`, …) co-located in a temp dir, and run `unrar lt` on part 1 —
+`unrar` auto-discovers co-located siblings (per the existing reader's note). The global-offset →
+`(part, offsetInPart)` mapping reuses the multipart size math the ZIP path already uses.
+
+## Fallback & integration
+
+- A ranged reader returning `null` = cheap read failed (bad magic, read error, walk gave up, or
+  CLI error on the sparse file) → **full-download fallback**: download the whole archive, run the
+  existing `readRarContents` / `read7zContents`, backfill.
+- The fallback is gated by `config.maxZipSizeMB` (the same guard used at ingest). Over the cap →
+  no download; write a `SystemNotification` (`INTEGRITY_AUDIT`, WARNING) and leave the listing
+  empty for manual review. This ensures nothing pathological (e.g. the 116 GB RAR) is pulled.
+- Downstream is unchanged: `compareFingerprints` already treats null/incomplete CRCs as
+  "incomplete" (name-size path), and `backfillProvenance` writes entries when the candidate's
+  `fileCount === 0`.
+- Observability: reuse the `zipsBackfilled` counter; add structured logs with
+  `confidence: "ranged" | "full-download-fallback"` and a WARN on fallback so miss-rate is
+  visible.
+
+## Risks & de-risking spike (do before the full build)
+
+On 3–4 real placeholder archives per format:
+
+1. Confirm `downloadFileRange` returns correct bytes at **arbitrary (non-tail) offsets** —
+   currently only tail-verified in production. Underpins everything; if it fails, stop and
+   rethink. (Note: `range-download.ts` flags absolute-offset behaviour as pending live
+   verification; tail reads are proven by the 43 ZIP backfills done 2026-07-26.)
+2. Confirm `7z l` and `unrar lt` list correctly from a **sparse reconstructed file** — single
+   part first, then multipart RAR (the highest-risk case).
+
+If multipart-RAR sparse reconstruction proves unreliable in the spike, multipart RAR uses the
+full-download fallback (respecting the size cap → oversized ones flagged, not downloaded).
+
+## Testing
+
+- **Unit (vitest, alongside `central-directory.test.ts`):** 7z signature-header parse; RAR4 &
+  RAR5 block-extent walk against committed small fixtures; `sparse-list` writes the correct
+  regions. Pure logic, no TDLib.
+- **Live post-deploy:** watch `zipsBackfilled` climb for RAR/7z via the ranged path; spot-check
+  a handful of backfilled packages' `package_files` against a real `unrar lt` / `7z l` on a full
+  download of the same file; confirm the fallback/flag path fires on a deliberately-broken case.
+
+## Rollout
+
+Local build + deploy (no GitHub push required), per the established recipe: build
+`worker/Dockerfile` locally, recreate the `dragonsstash-worker` container from the local image
+(no `pull`). No new DB migration. The scheduler re-runs hourly and will backfill RAR/7z
+placeholders on subsequent cycles.
@@ -0,0 +1,32 @@
+-- CreateEnum GroupingSource
+CREATE TYPE "GroupingSource" AS ENUM ('ALBUM', 'MANUAL', 'AUTO_TIME', 'AUTO_PATTERN', 'AUTO_REPLY', 'AUTO_ZIP', 'AUTO_CAPTION');
+
+-- CreateEnum NotificationType
+CREATE TYPE "NotificationType" AS ENUM ('HASH_MISMATCH', 'MISSING_PART', 'UPLOAD_FAILED', 'DOWNLOAD_FAILED', 'GROUPING_CONFLICT', 'INTEGRITY_AUDIT');
+
+-- CreateEnum NotificationSeverity
+CREATE TYPE "NotificationSeverity" AS ENUM ('INFO', 'WARNING', 'ERROR');
+
+-- AlterTable: add groupingSource to package_groups
+ALTER TABLE "package_groups" ADD COLUMN "groupingSource" "GroupingSource" NOT NULL DEFAULT 'MANUAL';
+
+-- Backfill: mark album-based groups
+UPDATE "package_groups" SET "groupingSource" = 'ALBUM' WHERE "mediaAlbumId" IS NOT NULL;
+
+-- CreateTable: system_notifications
+CREATE TABLE "system_notifications" (
+    "id" TEXT NOT NULL,
+    "type" "NotificationType" NOT NULL,
+    "severity" "NotificationSeverity" NOT NULL DEFAULT 'INFO',
+    "title" TEXT NOT NULL,
+    "message" TEXT NOT NULL,
+    "context" JSONB,
+    "isRead" BOOLEAN NOT NULL DEFAULT false,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "system_notifications_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "system_notifications_isRead_createdAt_idx" ON "system_notifications"("isRead", "createdAt");
+CREATE INDEX "system_notifications_type_idx" ON "system_notifications"("type");
@@ -0,0 +1,3 @@
+-- AlterTable: add sourceCaption and replyToMessageId to packages
+ALTER TABLE "packages" ADD COLUMN "sourceCaption" TEXT;
+ALTER TABLE "packages" ADD COLUMN "replyToMessageId" BIGINT;
@@ -0,0 +1,47 @@
+-- AlterTable: add autoGroupEnabled to telegram_channels
+ALTER TABLE "telegram_channels" ADD COLUMN "autoGroupEnabled" BOOLEAN NOT NULL DEFAULT true;
+
+-- CreateTable: grouping_rules
+CREATE TABLE "grouping_rules" (
+    "id" TEXT NOT NULL,
+    "sourceChannelId" TEXT NOT NULL,
+    "pattern" TEXT NOT NULL,
+    "signalType" "GroupingSource" NOT NULL,
+    "confidence" DOUBLE PRECISION NOT NULL DEFAULT 1.0,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "createdByGroupId" TEXT,
+
+    CONSTRAINT "grouping_rules_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "grouping_rules_sourceChannelId_idx" ON "grouping_rules"("sourceChannelId");
+
+-- AddForeignKey
+ALTER TABLE "grouping_rules" ADD CONSTRAINT "grouping_rules_sourceChannelId_fkey" FOREIGN KEY ("sourceChannelId") REFERENCES "telegram_channels"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- Full-text search: add tsvector column and GIN index
+ALTER TABLE "packages" ADD COLUMN IF NOT EXISTS "searchVector" tsvector;
+
+UPDATE "packages" SET "searchVector" = to_tsvector('english',
+  coalesce("fileName", '') || ' ' || coalesce("creator", '') || ' ' || coalesce("sourceCaption", '')
+) WHERE "searchVector" IS NULL;
+
+CREATE INDEX IF NOT EXISTS "packages_search_vector_idx" ON "packages" USING GIN ("searchVector");
+
+-- Trigger to auto-update searchVector on insert/update
+CREATE OR REPLACE FUNCTION packages_search_vector_update() RETURNS trigger AS $$
+BEGIN
+  NEW."searchVector" := to_tsvector('english',
+    coalesce(NEW."fileName", '') || ' ' || coalesce(NEW."creator", '') || ' ' || coalesce(NEW."sourceCaption", '')
+  );
+  RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+DROP TRIGGER IF EXISTS packages_search_vector_trigger ON "packages";
+CREATE TRIGGER packages_search_vector_trigger
+  BEFORE INSERT OR UPDATE OF "fileName", "creator", "sourceCaption"
+  ON "packages"
+  FOR EACH ROW
+  EXECUTE FUNCTION packages_search_vector_update();
@@ -0,0 +1,30 @@
+-- CreateEnum
+CREATE TYPE "ManualUploadStatus" AS ENUM ('PENDING', 'PROCESSING', 'COMPLETED', 'FAILED');
+
+-- CreateTable
+CREATE TABLE "manual_uploads" (
+    "id" TEXT NOT NULL,
+    "status" "ManualUploadStatus" NOT NULL DEFAULT 'PENDING',
+    "groupName" TEXT,
+    "userId" TEXT NOT NULL,
+    "errorMessage" TEXT,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "completedAt" TIMESTAMP(3),
+    CONSTRAINT "manual_uploads_pkey" PRIMARY KEY ("id")
+);
+
+CREATE TABLE "manual_upload_files" (
+    "id" TEXT NOT NULL,
+    "uploadId" TEXT NOT NULL,
+    "fileName" TEXT NOT NULL,
+    "filePath" TEXT NOT NULL,
+    "fileSize" BIGINT NOT NULL,
+    "packageId" TEXT,
+    CONSTRAINT "manual_upload_files_pkey" PRIMARY KEY ("id")
+);
+
+CREATE INDEX "manual_uploads_status_idx" ON "manual_uploads"("status");
+CREATE INDEX "manual_upload_files_uploadId_idx" ON "manual_upload_files"("uploadId");
+
+ALTER TABLE "manual_uploads" ADD CONSTRAINT "manual_uploads_userId_fkey" FOREIGN KEY ("userId") REFERENCES "User"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
+ALTER TABLE "manual_upload_files" ADD CONSTRAINT "manual_upload_files_uploadId_fkey" FOREIGN KEY ("uploadId") REFERENCES "manual_uploads"("id") ON DELETE CASCADE ON UPDATE CASCADE;
@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "telegram_accounts" ADD COLUMN "isPremium" BOOLEAN NOT NULL DEFAULT false;
@@ -0,0 +1,8 @@
+-- AlterTable: track how many times the worker has tried each skipped source message.
+-- Existing rows default to 1 (they represent a single past attempt that the worker
+-- chose to record). Future failures increment via upsertSkippedPackage.
+ALTER TABLE "skipped_packages" ADD COLUMN "attemptCount" INTEGER NOT NULL DEFAULT 1;
+
+-- AlterEnum: add CHANNEL_ACCESS_LOST so the worker can surface a notification
+-- when a source channel becomes inaccessible (account removed, channel deleted, etc.)
+ALTER TYPE "NotificationType" ADD VALUE 'CHANNEL_ACCESS_LOST';
@@ -0,0 +1,10 @@
+-- AlterTable: capture TDLib's stable per-content identifier for new packages.
+-- Existing rows are NULL; they fall through to the other dedup checks until
+-- they're re-encountered organically.
+ALTER TABLE "packages" ADD COLUMN "remoteUniqueId" TEXT;
+
+-- CreateIndex: scoped to source channel because we want to dedup
+-- per-channel (the same file appearing in two different channels is still
+-- worth indexing twice — they're different ingestion sources).
+CREATE INDEX "packages_sourceChannelId_remoteUniqueId_idx"
+  ON "packages"("sourceChannelId", "remoteUniqueId");
@@ -0,0 +1,11 @@
+-- AlterTable: per-channel scan-state columns
+ALTER TABLE "account_channel_map"
+  ADD COLUMN "lastScannedAt" TIMESTAMP(3),
+  ADD COLUMN "lastScanFoundArchives" BOOLEAN NOT NULL DEFAULT false,
+  ADD COLUMN "consecutiveEmptyScans" INTEGER NOT NULL DEFAULT 0;
+
+-- AlterTable: per-topic scan-state columns (forum channels)
+ALTER TABLE "topic_progress"
+  ADD COLUMN "lastScannedAt" TIMESTAMP(3),
+  ADD COLUMN "lastScanFoundArchives" BOOLEAN NOT NULL DEFAULT false,
+  ADD COLUMN "consecutiveEmptyScans" INTEGER NOT NULL DEFAULT 0;
@@ -0,0 +1,5 @@
+-- AlterTable: per-topic user-controlled fetch toggle (forum channels)
+-- Additive, safe default (true) so existing rows backfill to "enabled" and
+-- current behaviour is unchanged. Disabling is non-destructive.
+ALTER TABLE "topic_progress"
+  ADD COLUMN "fetchEnabled" BOOLEAN NOT NULL DEFAULT true;
@@ -0,0 +1,6 @@
+-- AlterTable: track the forum topic currently being processed on the live run
+-- so the worker status panel can offer a "skip & disable topic" action.
+-- Additive, nullable — no data change for existing rows.
+ALTER TABLE "ingestion_runs"
+  ADD COLUMN "currentTopicId" BIGINT,
+  ADD COLUMN "currentAccountChannelMapId" TEXT;
@@ -0,0 +1,5 @@
+-- AlterTable: count of packages whose provenance was backfilled during a run
+-- (opportunistic cross-channel provenance backfill). Additive, non-null with a
+-- default of 0 — no data change for existing rows.
+ALTER TABLE "ingestion_runs"
+  ADD COLUMN "zipsBackfilled" INTEGER NOT NULL DEFAULT 0;
@@ -42,6 +42,7 @@ model User {
  inviteCodes    InviteCode[]   @relation("InviteCreator")
  usedInvite     InviteCode?    @relation("InviteUser", fields: [usedInviteId], references: [id], onDelete: SetNull)
  usedInviteId   String?
+  manualUploads  ManualUpload[]
 }

 model Account {
@@ -405,6 +406,7 @@ model TelegramAccount {
  isActive    Boolean   @default(true)
  authState   AuthState @default(PENDING)
  authCode    String?
+  isPremium   Boolean   @default(false)
  lastSeenAt  DateTime?
  createdAt   DateTime  @default(now())
  updatedAt   DateTime  @updatedAt
@@ -429,10 +431,13 @@ model TelegramChannel {
  createdAt  DateTime    @default(now())
  updatedAt  DateTime    @updatedAt

+  autoGroupEnabled Boolean     @default(true)
+
  accountMaps     AccountChannelMap[]
  packages        Package[]
  skippedPackages SkippedPackage[]
  packageGroups   PackageGroup[]
+  groupingRules   GroupingRule[]

  @@index([type, isActive])
  @@index([category])
@@ -445,6 +450,17 @@ model AccountChannelMap {
  channelId              String
  role                   ChannelRole @default(READER)
  lastProcessedMessageId BigInt?
+  /// When this channel was last scanned (any reason, including skipped scans
+  /// that bumped the timestamp). Used by the recency-skip guard.
+  lastScannedAt          DateTime?
+  /// True if the last scan found archives OR left retryable SkippedPackages
+  /// pending. Tracks "this channel has work I might need to revisit" — not
+  /// just "I uploaded something this cycle".
+  lastScanFoundArchives  Boolean     @default(false)
+  /// Number of consecutive cycles where this channel was trulyIdle (no
+  /// archives, no failures, no retryables). Drives the backoff that lets
+  /// cold channels skip cycles entirely.
+  consecutiveEmptyScans  Int         @default(0)
  createdAt              DateTime    @default(now())

  account        TelegramAccount @relation(fields: [accountId], references: [id], onDelete: Cascade)
@@ -467,6 +483,11 @@ model Package {
  sourceChannelId String
  sourceMessageId BigInt
  sourceTopicId   BigInt?
+  /// TDLib's `remote.unique_id` for the FIRST part's file. Stable across
+  /// reposts of identical content in the same channel — used as the
+  /// strongest pre-download dedup signal (no false positives unlike
+  /// fileName + size matching).
+  remoteUniqueId  String?
  destChannelId   String?
  destMessageId   BigInt?
  destMessageIds  BigInt[]    @default([])
@@ -474,6 +495,8 @@ model Package {
  partCount       Int         @default(1)
  fileCount       Int         @default(0)
  tags            String[]    @default([])
+  sourceCaption   String?     // Caption text from source Telegram message
+  replyToMessageId BigInt?    // reply_to_message_id from source message (for reply chain grouping)
  previewData     Bytes?      // JPEG thumbnail from nearby Telegram photo (stored as raw bytes)
  previewMsgId    BigInt?     // Telegram message ID of the matched photo
  packageGroupId  String?
@@ -496,6 +519,7 @@ model Package {
  @@index([archiveType])
  @@index([creator])
  @@index([packageGroupId])
+  @@index([sourceChannelId, remoteUniqueId])
  @@map("packages")
 }

@@ -522,6 +546,7 @@ model PackageGroup {
  name            String
  mediaAlbumId    String?
  sourceChannelId String
+  groupingSource  GroupingSource   @default(MANUAL)
  previewData     Bytes?
  createdAt       DateTime         @default(now())
  updatedAt       DateTime         @updatedAt
@@ -544,6 +569,7 @@ model IngestionRun {
  zipsFound       Int             @default(0)
  zipsDuplicate   Int             @default(0)
  zipsIngested    Int             @default(0)
+  zipsBackfilled  Int             @default(0)
  errorMessage    String?

  // Live activity tracking — written by worker in real-time
@@ -557,6 +583,8 @@ model IngestionRun {
  totalBytes      BigInt?         // Total size of current download
  downloadPercent Int?            // 0-100
  lastActivityAt  DateTime?       // When activity was last updated
+  currentTopicId             BigInt?  // Forum topic currently being processed (live "skip topic")
+  currentAccountChannelMapId String?  // AccountChannelMap owning the topic being processed

  account  TelegramAccount @relation(fields: [accountId], references: [id])
  packages Package[]
@@ -572,7 +600,20 @@ model TopicProgress {
  accountChannelMapId    String
  topicId                BigInt
  topicName              String?
+  /// User-controlled fetch toggle. When false, the worker skips this topic
+  /// (no scanning, no fetching/transfer). Defaults true so every existing and
+  /// newly-discovered topic is fetched unless explicitly disabled. Disabling
+  /// is non-destructive — already-fetched packages are kept.
+  fetchEnabled           Boolean   @default(true)
  lastProcessedMessageId BigInt?
+  /// When this topic was last scanned (any reason). Used by recency-skip.
+  lastScannedAt          DateTime?
+  /// True if the last scan found archives OR has retryable SkippedPackages
+  /// pending for this topic. See AccountChannelMap doc for details.
+  lastScanFoundArchives  Boolean   @default(false)
+  /// Number of consecutive cycles where this topic was trulyIdle. Drives
+  /// backoff for cold topics.
+  consecutiveEmptyScans  Int       @default(0)

  accountChannelMap AccountChannelMap @relation(fields: [accountChannelMapId], references: [id], onDelete: Cascade)

@@ -733,6 +774,12 @@ model SkippedPackage {
  sourceTopicId   BigInt?
  isMultipart     Boolean          @default(false)
  partCount       Int              @default(1)
+  /// How many times the worker has tried to process this source message.
+  /// The worker auto-retries failures across cycles up to a configurable cap
+  /// (WORKER_MAX_SKIP_ATTEMPTS, default 5). After the cap, the watermark is
+  /// allowed to advance past the failure so cycles aren't pinned forever;
+  /// the user can manually retry via the UI to reset and try again.
+  attemptCount    Int              @default(1)
  accountId       String
  account         TelegramAccount  @relation(fields: [accountId], references: [id], onDelete: Cascade)
  createdAt       DateTime         @default(now())
@@ -802,3 +849,98 @@ model KickstarterPackage {
  @@id([kickstarterId, packageId])
  @@map("kickstarter_packages")
 }
+
+// ── Grouping & Notifications ──
+
+enum GroupingSource {
+  ALBUM
+  MANUAL
+  AUTO_TIME
+  AUTO_PATTERN
+  AUTO_REPLY
+  AUTO_ZIP
+  AUTO_CAPTION
+}
+
+enum NotificationType {
+  HASH_MISMATCH
+  MISSING_PART
+  UPLOAD_FAILED
+  DOWNLOAD_FAILED
+  GROUPING_CONFLICT
+  INTEGRITY_AUDIT
+  CHANNEL_ACCESS_LOST
+}
+
+enum NotificationSeverity {
+  INFO
+  WARNING
+  ERROR
+}
+
+model SystemNotification {
+  id        String               @id @default(cuid())
+  type      NotificationType
+  severity  NotificationSeverity @default(INFO)
+  title     String
+  message   String
+  context   Json?
+  isRead    Boolean              @default(false)
+  createdAt DateTime             @default(now())
+
+  @@index([isRead, createdAt])
+  @@index([type])
+  @@map("system_notifications")
+}
+
+model GroupingRule {
+  id              String          @id @default(cuid())
+  sourceChannelId String
+  pattern         String          // Regex or keyword pattern learned from manual grouping
+  signalType      GroupingSource  // Which grouping signal this rule applies to
+  confidence      Float           @default(1.0)
+  createdAt       DateTime        @default(now())
+  createdByGroupId String?        // The manual group that spawned this rule
+
+  sourceChannel   TelegramChannel @relation(fields: [sourceChannelId], references: [id], onDelete: Cascade)
+
+  @@index([sourceChannelId])
+  @@map("grouping_rules")
+}
+
+enum ManualUploadStatus {
+  PENDING
+  PROCESSING
+  COMPLETED
+  FAILED
+}
+
+model ManualUpload {
+  id              String             @id @default(cuid())
+  status          ManualUploadStatus @default(PENDING)
+  groupName       String?            // Group name if multiple files
+  userId          String
+  errorMessage    String?
+  createdAt       DateTime           @default(now())
+  completedAt     DateTime?
+
+  files           ManualUploadFile[]
+  user            User               @relation(fields: [userId], references: [id])
+
+  @@index([status])
+  @@map("manual_uploads")
+}
+
+model ManualUploadFile {
+  id              String       @id @default(cuid())
+  uploadId        String
+  fileName        String
+  filePath        String       // Path on shared volume
+  fileSize        BigInt
+  packageId       String?      // Set after processing
+
+  upload          ManualUpload @relation(fields: [uploadId], references: [id], onDelete: Cascade)
+
+  @@index([uploadId])
+  @@map("manual_upload_files")
+}
@@ -1,7 +1,23 @@
+import { redirect } from "next/navigation";
+import { auth } from "@/lib/auth";
+import { prisma } from "@/lib/prisma";
 import { Sidebar } from "@/components/layout/sidebar";
 import { Header } from "@/components/layout/header";

-export default function AppLayout({ children }: { children: React.ReactNode }) {
+export default async function AppLayout({ children }: { children: React.ReactNode }) {
+  // Guard against a stale JWT session whose user no longer exists in the
+  // database (e.g. after a DB reset). The signed cookie still passes edge
+  // middleware, but every downstream query keyed on session.user.id would fail.
+  // Send such sessions to /logout, which clears the cookie and returns to login.
+  const session = await auth();
+  if (session?.user?.id) {
+    const user = await prisma.user.findUnique({
+      where: { id: session.user.id },
+      select: { id: true },
+    });
+    if (!user) redirect("/logout");
+  }
+
  return (
    <div className="flex h-screen overflow-hidden">
      <div className="hidden lg:block">
@@ -7,6 +7,7 @@ import {
  Check,
  AlertCircle,
  ImageOff,
+  Maximize2,
 } from "lucide-react";
 import {
  Dialog,
@@ -20,6 +21,7 @@ import { Button } from "@/components/ui/button";
 import { cn } from "@/lib/utils";
 import { toast } from "sonner";
 import { setPreviewFromExtract } from "../actions";
+import { ImageLightbox } from "./image-lightbox";

 interface ArchiveImage {
  id: string;
@@ -65,6 +67,7 @@ export function ArchivePreviewPicker({
  const [thumbnails, setThumbnails] = useState<Map<string, ThumbnailState>>(new Map());
  const [selectedPath, setSelectedPath] = useState<string | null>(null);
  const [isPending, startTransition] = useTransition();
+  const [lightboxSrc, setLightboxSrc] = useState<string | null>(null);
  const pollTimers = useRef<Map<string, ReturnType<typeof setInterval>>>(new Map());
  // Track which paths have already been requested to avoid re-requesting
  const requestedPaths = useRef<Set<string>>(new Set());
@@ -290,12 +293,25 @@ export function ArchivePreviewPicker({
                  const isFailed = thumbState?.status === "failed";

                  return (
+                    <div key={img.id} className="group relative">
+                      {isLoaded && thumbState?.imageUrl && (
+                        <button
+                          type="button"
+                          className="absolute top-1.5 left-1.5 z-10 flex h-6 w-6 items-center justify-center rounded-md bg-black/60 text-white opacity-0 transition-opacity hover:bg-black/80 group-hover:opacity-100"
+                          onClick={(e) => {
+                            e.stopPropagation();
+                            setLightboxSrc(thumbState.imageUrl!);
+                          }}
+                          title="Enlarge"
+                        >
+                          <Maximize2 className="h-3.5 w-3.5" />
+                        </button>
+                      )}
                      <button
-                      key={img.id}
                        type="button"
                        className={cn(
-                        "relative aspect-square rounded-lg overflow-hidden border-2 transition-all",
-                        "hover:border-primary/50 cursor-pointer group",
+                          "relative aspect-square w-full rounded-lg overflow-hidden border-2 transition-all",
+                          "hover:border-primary/50 cursor-pointer",
                          isSelected
                            ? "border-primary ring-2 ring-primary/30"
                            : "border-border",
@@ -355,6 +371,7 @@ export function ArchivePreviewPicker({
                          </p>
                        </div>
                      </button>
+                    </div>
                  );
                })}
              </div>
@@ -394,6 +411,13 @@ export function ArchivePreviewPicker({
          </div>
        )}
      </DialogContent>
+      <ImageLightbox
+        src={lightboxSrc}
+        open={!!lightboxSrc}
+        onOpenChange={(open) => {
+          if (!open) setLightboxSrc(null);
+        }}
+      />
    </Dialog>
  );
 }
@@ -0,0 +1,82 @@
+"use client";
+
+import { useState } from "react";
+import { Check, ChevronsUpDown } from "lucide-react";
+import { cn } from "@/lib/utils";
+import { Button } from "@/components/ui/button";
+import {
+  Command,
+  CommandEmpty,
+  CommandGroup,
+  CommandInput,
+  CommandItem,
+  CommandList,
+} from "@/components/ui/command";
+import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover";
+
+interface CreatorFilterProps {
+  creators: string[];
+  value: string; // active creator, "" when none
+  onChange: (creator: string) => void; // "" clears the filter
+}
+
+export function CreatorFilter({ creators, value, onChange }: CreatorFilterProps) {
+  const [open, setOpen] = useState(false);
+
+  return (
+    <Popover open={open} onOpenChange={setOpen}>
+      <PopoverTrigger asChild>
+        <Button
+          variant="outline"
+          size="sm"
+          role="combobox"
+          aria-expanded={open}
+          className="h-9 w-[200px] justify-between"
+        >
+          <span className="truncate">{value || "All Creators"}</span>
+          <ChevronsUpDown className="ml-2 h-4 w-4 shrink-0 opacity-50" />
+        </Button>
+      </PopoverTrigger>
+      <PopoverContent className="w-[240px] p-0" align="start">
+        <Command>
+          <CommandInput placeholder="Search creators..." className="h-9" />
+          <CommandList>
+            <CommandEmpty>No creators found.</CommandEmpty>
+            <CommandGroup>
+              <CommandItem
+                value="__all__"
+                onSelect={() => {
+                  onChange("");
+                  setOpen(false);
+                }}
+              >
+                <Check
+                  className={cn("mr-2 h-4 w-4", value === "" ? "opacity-100" : "opacity-0")}
+                />
+                All Creators
+              </CommandItem>
+              {creators.map((creator) => (
+                <CommandItem
+                  key={creator}
+                  value={creator}
+                  onSelect={() => {
+                    onChange(creator);
+                    setOpen(false);
+                  }}
+                >
+                  <Check
+                    className={cn(
+                      "mr-2 h-4 w-4",
+                      value === creator ? "opacity-100" : "opacity-0"
+                    )}
+                  />
+                  <span className="truncate">{creator}</span>
+                </CommandItem>
+              ))}
+            </CommandGroup>
+          </CommandList>
+        </Command>
+      </PopoverContent>
+    </Popover>
+  );
+}
@@ -0,0 +1,41 @@
+"use client";
+
+import {
+  Dialog,
+  DialogContent,
+  DialogTitle,
+} from "@/components/ui/dialog";
+
+interface ImageLightboxProps {
+  src: string | null;
+  alt?: string;
+  open: boolean;
+  onOpenChange: (open: boolean) => void;
+}
+
+/**
+ * Full-size, in-page preview viewer. Renders the image at native size capped
+ * to the viewport (object-contain). Dismiss via the close button, Esc, or by
+ * clicking the overlay.
+ */
+export function ImageLightbox({
+  src,
+  alt = "",
+  open,
+  onOpenChange,
+}: ImageLightboxProps) {
+  return (
+    <Dialog open={open} onOpenChange={onOpenChange}>
+      <DialogContent className="w-auto max-w-[95vw] border-0 bg-transparent p-0 shadow-none sm:max-w-[90vw]">
+        <DialogTitle className="sr-only">Enlarged preview image</DialogTitle>
+        {src && (
+          <img
+            src={src}
+            alt={alt}
+            className="mx-auto max-h-[88vh] w-auto max-w-full rounded-lg object-contain"
+          />
+        )}
+      </DialogContent>
+    </Dialog>
+  );
+}
@@ -1,12 +1,14 @@
 "use client";

+import { useState } from "react";
 import { type ColumnDef } from "@tanstack/react-table";
-import { FileArchive, Eye, ChevronRight, Layers, Ungroup, Send, ImagePlus } from "lucide-react";
+import { FileArchive, Eye, ChevronRight, Layers, Ungroup, Send, ImagePlus, GitMerge, Maximize2 } from "lucide-react";
 import { DataTableColumnHeader } from "@/components/shared/data-table-column-header";
 import { Badge } from "@/components/ui/badge";
 import { Button } from "@/components/ui/button";
 import { Checkbox } from "@/components/ui/checkbox";
 import { SendToTelegramButton } from "./send-to-telegram-button";
+import { ImageLightbox } from "./image-lightbox";

 export interface PackageRow {
  id: string;
@@ -69,6 +71,9 @@ interface PackageColumnsProps {
  onGroupPreviewUpload: (groupId: string) => void;
  selectedPackages: Set<string>;
  onToggleSelect: (packageId: string) => void;
+  mergeSourceId: string | null;
+  onStartMerge: (groupId: string) => void;
+  onCompleteMerge: (targetGroupId: string) => void;
 }

 export function formatBytes(bytesStr: string): string {
@@ -81,14 +86,30 @@ export function formatBytes(bytesStr: string): string {
 }

 function PreviewCell({ pkg }: { pkg: PackageRow }) {
+  const [lightboxOpen, setLightboxOpen] = useState(false);
+
  if (pkg.hasPreview) {
+    const src = `/api/zips/${pkg.id}/preview`;
    return (
+      <>
+        <button
+          type="button"
+          className="group/preview relative h-9 w-9 overflow-hidden rounded-md bg-muted"
+          onClick={() => setLightboxOpen(true)}
+          title="Click to enlarge"
+        >
          <img
-        src={`/api/zips/${pkg.id}/preview`}
+            src={src}
            alt=""
-        className="h-9 w-9 rounded-md object-cover bg-muted"
+            className="h-9 w-9 object-cover"
            loading="lazy"
          />
+          <div className="absolute inset-0 flex items-center justify-center rounded-md bg-black/50 opacity-0 transition-opacity group-hover/preview:opacity-100">
+            <Maximize2 className="h-3.5 w-3.5 text-white" />
+          </div>
+        </button>
+        <ImageLightbox src={src} open={lightboxOpen} onOpenChange={setLightboxOpen} />
+      </>
    );
  }
  return (
@@ -148,6 +169,9 @@ export function getPackageColumns({
  onGroupPreviewUpload,
  selectedPackages,
  onToggleSelect,
+  mergeSourceId,
+  onStartMerge,
+  onCompleteMerge,
 }: PackageColumnsProps): ColumnDef<StlTableRow, unknown>[] {
  return [
    {
@@ -392,6 +416,8 @@ export function getPackageColumns({
      cell: ({ row }) => {
        const data = row.original;
        if (isGroupRow(data)) {
+          const isMergeSource = mergeSourceId === data.id;
+          const canMergeHere = mergeSourceId !== null && mergeSourceId !== data.id;
          return (
            <div className="flex items-center gap-0.5">
              <Button
@@ -403,6 +429,26 @@ export function getPackageColumns({
              >
                <Send className="h-4 w-4" />
              </Button>
+              <Button
+                variant="ghost"
+                size="icon"
+                className={`h-8 w-8 ${isMergeSource ? "text-amber-500 bg-amber-500/10 hover:bg-amber-500/20" : ""}`}
+                onClick={() => onStartMerge(data.id)}
+                title={isMergeSource ? "Cancel merge (this group is the merge source)" : "Start merge — mark this group as merge source"}
+              >
+                <GitMerge className="h-4 w-4" />
+              </Button>
+              {canMergeHere && (
+                <Button
+                  variant="ghost"
+                  size="icon"
+                  className="h-8 w-8 text-primary bg-primary/10 hover:bg-primary/20"
+                  onClick={() => onCompleteMerge(data.id)}
+                  title="Merge source group into this group"
+                >
+                  <Layers className="h-4 w-4" />
+                </Button>
+              )}
              <Button
                variant="ghost"
                size="icon"
@@ -13,6 +13,7 @@ import {
  Upload,
  ImagePlus,
  Images,
+  Maximize2,
 } from "lucide-react";
 import {
  Dialog,
@@ -30,6 +31,7 @@ import type { PackageRow } from "./package-columns";
 import { SendToTelegramButton } from "./send-to-telegram-button";
 import { uploadPackagePreview } from "../actions";
 import { ArchivePreviewPicker } from "./archive-preview-picker";
+import { ImageLightbox } from "./image-lightbox";

 interface FileItem {
  id: string;
@@ -264,6 +266,7 @@ export function PackageFilesDrawer({ pkg, open, onOpenChange, highlightTerm }: P
  const [uploading, setUploading] = useState(false);
  const [localPreviewUrl, setLocalPreviewUrl] = useState<string | null>(null);
  const [showPreviewPicker, setShowPreviewPicker] = useState(false);
+  const [previewLightboxOpen, setPreviewLightboxOpen] = useState(false);
  const fileInputRef = useRef<HTMLInputElement>(null);

  const handlePreviewUpload = useCallback(
@@ -384,9 +387,8 @@ export function PackageFilesDrawer({ pkg, open, onOpenChange, highlightTerm }: P
              <button
                type="button"
                className="relative group h-20 w-20 shrink-0 rounded-lg overflow-hidden bg-muted"
-                onClick={() => fileInputRef.current?.click()}
-                disabled={uploading}
-                title="Click to replace preview image"
+                onClick={() => setPreviewLightboxOpen(true)}
+                title="Click to enlarge"
              >
                <img
                  src={localPreviewUrl ?? `/api/zips/${pkg!.id}/preview`}
@@ -394,11 +396,7 @@ export function PackageFilesDrawer({ pkg, open, onOpenChange, highlightTerm }: P
                  className="h-full w-full object-cover"
                />
                <div className="absolute inset-0 bg-black/50 opacity-0 group-hover:opacity-100 transition-opacity flex items-center justify-center">
-                  {uploading ? (
-                    <Loader2 className="h-5 w-5 text-white animate-spin" />
-                  ) : (
-                    <Upload className="h-5 w-5 text-white" />
-                  )}
+                  <Maximize2 className="h-5 w-5 text-white" />
                </div>
              </button>
            ) : (
@@ -582,6 +580,11 @@ export function PackageFilesDrawer({ pkg, open, onOpenChange, highlightTerm }: P
        }}
      />
    )}
+    <ImageLightbox
+      src={pkg ? (localPreviewUrl ?? `/api/zips/${pkg.id}/preview`) : null}
+      open={previewLightboxOpen}
+      onOpenChange={setPreviewLightboxOpen}
+    />
    </>
  );
 }
@@ -20,6 +20,7 @@ export interface SkippedRow {
  sourceChannel: { id: string; title: string };
  isMultipart: boolean;
  partCount: number;
+  attemptCount: number;
  createdAt: string;
 }

@@ -107,9 +108,22 @@ export function getSkippedColumns({
      ),
      accessorFn: (row) => row.sourceChannel.title,
    },
+    {
+      accessorKey: "attemptCount",
+      header: ({ column }) => <DataTableColumnHeader column={column} title="Attempts" />,
+      cell: ({ row }) => {
+        const count = row.original.attemptCount;
+        const variant = count >= 5 ? "destructive" : count > 1 ? "secondary" : "outline";
+        return (
+          <Badge variant={variant} className="text-[10px]">
+            {count}
+          </Badge>
+        );
+      },
+    },
    {
      accessorKey: "createdAt",
-      header: ({ column }) => <DataTableColumnHeader column={column} title="Skipped" />,
+      header: ({ column }) => <DataTableColumnHeader column={column} title="Last Skipped" />,
      cell: ({ row }) => (
        <span className="text-sm text-muted-foreground">
          {new Date(row.original.createdAt).toLocaleDateString()}
@@ -3,7 +3,8 @@
 import { useState, useCallback, useTransition, useMemo, useRef } from "react";
 import { useRouter, usePathname, useSearchParams } from "next/navigation";
 import { toast } from "sonner";
-import { Search, Layers } from "lucide-react";
+import { Search, Layers, Upload, Send } from "lucide-react";
+import { UploadDialog } from "./upload-dialog";
 import { useDataTable } from "@/hooks/use-data-table";
 import {
  getPackageColumns,
@@ -15,6 +16,7 @@ import {
 import { PackageFilesDrawer } from "./package-files-drawer";
 import { IngestionStatus } from "./ingestion-status";
 import { SkippedPackagesTab } from "./skipped-packages-tab";
+import { CreatorFilter } from "./creator-filter";
 import { DataTable } from "@/components/shared/data-table";
 import { DataTablePagination } from "@/components/shared/data-table-pagination";
 import { DataTableViewOptions } from "@/components/shared/data-table-view-options";
@@ -38,7 +40,7 @@ import {
 } from "@/components/ui/dialog";
 import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
 import { Badge } from "@/components/ui/badge";
-import type { DisplayItem, IngestionAccountStatus } from "@/lib/telegram/types";
+import type { DisplayItem, IngestionAccountStatus, PackageListItem } from "@/lib/telegram/types";
 import type { SkippedRow } from "./skipped-columns";
 import {
  updatePackageCreator,
@@ -48,7 +50,9 @@ import {
  createGroupAction,
  removeFromGroupAction,
  sendAllInGroupAction,
+  sendAllFromCreatorAction,
  updateGroupPreviewAction,
+  mergeGroupsAction,
 } from "../actions";

 interface StlTableProps {
@@ -57,10 +61,14 @@ interface StlTableProps {
  totalCount: number;
  ingestionStatus: IngestionAccountStatus[];
  availableTags: string[];
+  availableCreators: string[];
  searchTerm: string;
  skippedData: SkippedRow[];
  skippedPageCount: number;
  skippedTotalCount: number;
+  ungroupedData: PackageListItem[];
+  ungroupedPageCount: number;
+  ungroupedTotalCount: number;
 }

 export function StlTable({
@@ -69,14 +77,19 @@ export function StlTable({
  totalCount,
  ingestionStatus,
  availableTags,
+  availableCreators,
  searchTerm,
  skippedData,
  skippedPageCount,
  skippedTotalCount,
+  ungroupedData,
+  ungroupedPageCount,
+  ungroupedTotalCount,
 }: StlTableProps) {
  const router = useRouter();
  const pathname = usePathname();
  const searchParams = useSearchParams();
+  const activeCreator = searchParams.get("creator") ?? "";

  const [searchValue, setSearchValue] = useState(searchParams.get("search") ?? "");
  const [viewPkg, setViewPkg] = useState<PackageRow | null>(null);
@@ -96,6 +109,12 @@ export function StlTable({
  const previewInputRef = useRef<HTMLInputElement>(null);
  const [uploadGroupId, setUploadGroupId] = useState<string | null>(null);

+  // Group merge state
+  const [mergeSourceId, setMergeSourceId] = useState<string | null>(null);
+
+  // Upload dialog state
+  const [uploadOpen, setUploadOpen] = useState(false);
+
  const toggleGroup = useCallback((groupId: string) => {
    setExpandedGroups((prev) => {
      const next = new Set(prev);
@@ -193,6 +212,20 @@ export function StlTable({
    [router, pathname, searchParams]
  );

+  const updateCreatorFilter = useCallback(
+    (value: string) => {
+      const params = new URLSearchParams(searchParams.toString());
+      if (value) {
+        params.set("creator", value);
+        params.set("page", "1");
+      } else {
+        params.delete("creator");
+      }
+      router.push(`${pathname}?${params.toString()}`, { scroll: false });
+    },
+    [router, pathname, searchParams]
+  );
+
  const activeTab = searchParams.get("tab") ?? "packages";

  const updateTab = useCallback(
@@ -263,6 +296,23 @@ export function StlTable({
    [router]
  );

+  const handleSendAllFromCreator = useCallback(() => {
+    if (!confirm(`Send all packages from "${activeCreator}" to your Telegram?`)) return;
+    startTransition(async () => {
+      const result = await sendAllFromCreatorAction(activeCreator);
+      if (result.success) {
+        const { queued, skipped } = result.data;
+        toast.success(
+          `Queued ${queued} package${queued === 1 ? "" : "s"} from ${activeCreator}` +
+            (skipped ? ` (${skipped} already queued)` : "")
+        );
+        router.refresh();
+      } else {
+        toast.error(result.error);
+      }
+    });
+  }, [activeCreator, router]);
+
  const handleRemoveFromGroup = useCallback(
    (packageId: string) => {
      startTransition(async () => {
@@ -334,6 +384,35 @@ export function StlTable({
    [uploadGroupId, router]
  );

+  const handleStartMerge = useCallback((groupId: string) => {
+    setMergeSourceId((prev) => {
+      if (prev === groupId) {
+        toast.info("Merge cancelled");
+        return null;
+      }
+      toast.info("Merge source selected — click the merge-here button on the target group");
+      return groupId;
+    });
+  }, []);
+
+  const handleMergeGroups = useCallback(
+    (targetGroupId: string) => {
+      if (!mergeSourceId) return;
+      const sourceId = mergeSourceId;
+      startTransition(async () => {
+        const result = await mergeGroupsAction(targetGroupId, sourceId);
+        if (result.success) {
+          toast.success("Groups merged successfully");
+          setMergeSourceId(null);
+          router.refresh();
+        } else {
+          toast.error(result.error);
+        }
+      });
+    },
+    [mergeSourceId, router]
+  );
+
  const columns = getPackageColumns({
    onViewFiles: (pkg) => setViewPkg(pkg),
    searchTerm,
@@ -375,10 +454,30 @@ export function StlTable({
    onGroupPreviewUpload: handleGroupPreviewUpload,
    selectedPackages,
    onToggleSelect: toggleSelect,
+    mergeSourceId,
+    onStartMerge: handleStartMerge,
+    onCompleteMerge: handleMergeGroups,
  });

  const { table } = useDataTable({ data: tableRows, columns, pageCount });

+  const ungroupedRows: StlTableRow[] = useMemo(
+    () =>
+      ungroupedData.map((pkg) => ({
+        ...pkg,
+        _rowType: "package" as const,
+        _groupId: null,
+        _isGroupMember: false,
+      })),
+    [ungroupedData]
+  );
+
+  const { table: ungroupedTable } = useDataTable({
+    data: ungroupedRows,
+    columns,
+    pageCount: ungroupedPageCount,
+  });
+
  const activeTag = searchParams.get("tag") ?? "";

  return (
@@ -401,6 +500,9 @@ export function StlTable({
              </Badge>
            )}
          </TabsTrigger>
+          {/* "Ungrouped" tab hidden: the STL list is now flat and grouping is
+              no longer surfaced here. The tab content below is kept (unreachable)
+              to avoid churn; remove it and its data fetch if grouping is dropped. */}
        </TabsList>

        <TabsContent value="packages" className="space-y-4">
@@ -429,7 +531,29 @@ export function StlTable({
                </SelectContent>
              </Select>
            )}
+            {availableCreators.length > 0 && (
+              <CreatorFilter
+                creators={availableCreators}
+                value={activeCreator}
+                onChange={updateCreatorFilter}
+              />
+            )}
            <DataTableViewOptions table={table} />
+            <Button variant="outline" size="sm" className="h-9" onClick={() => setUploadOpen(true)}>
+              <Upload className="mr-2 h-4 w-4" />
+              Upload Files
+            </Button>
+            {activeCreator && (
+              <Button
+                variant="outline"
+                size="sm"
+                className="h-9 gap-1.5"
+                onClick={handleSendAllFromCreator}
+              >
+                <Send className="h-3.5 w-3.5" />
+                Send all from {activeCreator}
+              </Button>
+            )}
            {selectedPackages.size >= 2 && (
              <Button
                variant="outline"
@@ -472,6 +596,11 @@ export function StlTable({
            totalCount={skippedTotalCount}
          />
        </TabsContent>
+
+        <TabsContent value="ungrouped" className="space-y-4">
+          <DataTable table={ungroupedTable} emptyMessage="All packages are grouped!" />
+          <DataTablePagination table={ungroupedTable} totalCount={ungroupedTotalCount} />
+        </TabsContent>
      </Tabs>

      <PackageFilesDrawer
@@ -515,6 +644,8 @@ export function StlTable({
        </DialogContent>
      </Dialog>

+      <UploadDialog open={uploadOpen} onOpenChange={setUploadOpen} />
+
      {/* Hidden file input for group preview upload (Task 12) */}
      <input
        ref={previewInputRef}
@@ -0,0 +1,243 @@
+"use client";
+
+import { useState, useRef, useTransition, useEffect } from "react";
+import { Upload, File, X, Loader2, CheckCircle2, AlertCircle } from "lucide-react";
+import { toast } from "sonner";
+import { Button } from "@/components/ui/button";
+import { Input } from "@/components/ui/input";
+import { Label } from "@/components/ui/label";
+import {
+  Dialog,
+  DialogContent,
+  DialogDescription,
+  DialogFooter,
+  DialogHeader,
+  DialogTitle,
+} from "@/components/ui/dialog";
+
+interface UploadDialogProps {
+  open: boolean;
+  onOpenChange: (open: boolean) => void;
+}
+
+function formatSize(bytes: number): string {
+  if (bytes >= 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)} GB`;
+  if (bytes >= 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(0)} MB`;
+  return `${(bytes / 1024).toFixed(0)} KB`;
+}
+
+type UploadStatus = "idle" | "uploading" | "processing" | "done" | "error";
+
+export function UploadDialog({ open, onOpenChange }: UploadDialogProps) {
+  const [files, setFiles] = useState<File[]>([]);
+  const [groupName, setGroupName] = useState("");
+  const [status, setStatus] = useState<UploadStatus>("idle");
+  const [error, setError] = useState<string | null>(null);
+  const [isPending, startTransition] = useTransition();
+  const fileInputRef = useRef<HTMLInputElement>(null);
+  const pollRef = useRef<ReturnType<typeof setInterval> | null>(null);
+
+  useEffect(() => {
+    if (open) {
+      setFiles([]);
+      setGroupName("");
+      setStatus("idle");
+      setError(null);
+    }
+    return () => {
+      if (pollRef.current) clearInterval(pollRef.current);
+    };
+  }, [open]);
+
+  function handleFileChange(e: React.ChangeEvent<HTMLInputElement>) {
+    if (e.target.files) {
+      setFiles(Array.from(e.target.files));
+    }
+  }
+
+  function removeFile(index: number) {
+    setFiles((prev) => prev.filter((_, i) => i !== index));
+  }
+
+  function handleUpload() {
+    if (files.length === 0) return;
+
+    startTransition(async () => {
+      setStatus("uploading");
+      setError(null);
+
+      try {
+        const formData = new FormData();
+        for (const file of files) {
+          formData.append("files", file);
+        }
+        if (groupName.trim()) {
+          formData.append("groupName", groupName.trim());
+        }
+
+        const res = await fetch("/api/uploads", {
+          method: "POST",
+          body: formData,
+        });
+
+        const data = await res.json();
+        if (!res.ok) {
+          setStatus("error");
+          setError(data.error ?? "Upload failed");
+          return;
+        }
+
+        setStatus("processing");
+
+        // Poll for completion
+        pollRef.current = setInterval(async () => {
+          try {
+            const statusRes = await fetch(`/api/uploads/${data.uploadId}`);
+            const statusData = await statusRes.json();
+
+            if (statusData.status === "COMPLETED") {
+              setStatus("done");
+              toast.success(`${files.length} file(s) uploaded and indexed`);
+              if (pollRef.current) clearInterval(pollRef.current);
+            } else if (statusData.status === "FAILED") {
+              setStatus("error");
+              setError(statusData.errorMessage ?? "Processing failed");
+              if (pollRef.current) clearInterval(pollRef.current);
+            }
+          } catch {
+            // Keep polling
+          }
+        }, 3000);
+
+        // Stop polling after 10 minutes
+        setTimeout(() => {
+          if (pollRef.current) {
+            clearInterval(pollRef.current);
+            pollRef.current = null;
+            setStatus((s) => s === "processing" ? "done" : s);
+          }
+        }, 600_000);
+      } catch {
+        setStatus("error");
+        setError("Network error");
+      }
+    });
+  }
+
+  return (
+    <Dialog open={open} onOpenChange={onOpenChange}>
+      <DialogContent className="sm:max-w-lg">
+        <DialogHeader>
+          <DialogTitle>Upload Files</DialogTitle>
+          <DialogDescription>
+            Upload archive files to be processed and indexed. Multiple files will be automatically grouped.
+          </DialogDescription>
+        </DialogHeader>
+
+        {status === "idle" && (
+          <div className="space-y-4">
+            <div
+              className="border-2 border-dashed rounded-lg p-8 text-center cursor-pointer hover:border-primary/50 transition-colors"
+              onClick={() => fileInputRef.current?.click()}
+            >
+              <Upload className="h-8 w-8 mx-auto mb-2 text-muted-foreground" />
+              <p className="text-sm text-muted-foreground">
+                Click to select files or drag & drop
+              </p>
+              <p className="text-xs text-muted-foreground mt-1">
+                ZIP, RAR, 7Z files up to 4GB each
+              </p>
+              <input
+                ref={fileInputRef}
+                type="file"
+                multiple
+                accept=".zip,.rar,.7z,.pdf,.stl"
+                onChange={handleFileChange}
+                className="hidden"
+              />
+            </div>
+
+            {files.length > 0 && (
+              <div className="space-y-2">
+                {files.map((file, i) => (
+                  <div key={i} className="flex items-center gap-2 p-2 rounded bg-muted/30">
+                    <File className="h-4 w-4 shrink-0 text-muted-foreground" />
+                    <span className="text-sm flex-1 truncate">{file.name}</span>
+                    <span className="text-xs text-muted-foreground">{formatSize(file.size)}</span>
+                    <button onClick={() => removeFile(i)} className="p-0.5 hover:text-destructive">
+                      <X className="h-3.5 w-3.5" />
+                    </button>
+                  </div>
+                ))}
+              </div>
+            )}
+
+            {files.length > 1 && (
+              <div>
+                <Label htmlFor="groupName" className="text-sm">Group Name (optional)</Label>
+                <Input
+                  id="groupName"
+                  value={groupName}
+                  onChange={(e) => setGroupName(e.target.value)}
+                  placeholder="Auto-generated from filenames"
+                  className="mt-1"
+                />
+              </div>
+            )}
+          </div>
+        )}
+
+        {(status === "uploading" || status === "processing") && (
+          <div className="flex items-center gap-3 p-6 rounded-lg bg-muted/30 border">
+            <Loader2 className="h-6 w-6 animate-spin text-primary" />
+            <div>
+              <p className="text-sm font-medium">
+                {status === "uploading" ? "Uploading files..." : "Processing & uploading to Telegram..."}
+              </p>
+              <p className="text-xs text-muted-foreground mt-0.5">
+                {status === "uploading"
+                  ? "Sending files to server"
+                  : "Hashing, extracting metadata, uploading to destination channel"}
+              </p>
+            </div>
+          </div>
+        )}
+
+        {status === "done" && (
+          <div className="flex items-center gap-3 p-6 rounded-lg bg-green-500/10 border border-green-500/20">
+            <CheckCircle2 className="h-6 w-6 text-green-500" />
+            <div>
+              <p className="text-sm font-medium text-green-500">Upload complete!</p>
+              <p className="text-xs text-muted-foreground">Files have been indexed and uploaded to Telegram.</p>
+            </div>
+          </div>
+        )}
+
+        {status === "error" && (
+          <div className="flex items-center gap-3 p-6 rounded-lg bg-destructive/10 border border-destructive/20">
+            <AlertCircle className="h-6 w-6 text-destructive" />
+            <div>
+              <p className="text-sm font-medium text-destructive">Upload failed</p>
+              <p className="text-xs text-muted-foreground">{error}</p>
+            </div>
+          </div>
+        )}
+
+        <DialogFooter>
+          {status === "idle" && (
+            <>
+              <Button variant="outline" onClick={() => onOpenChange(false)}>Cancel</Button>
+              <Button onClick={handleUpload} disabled={files.length === 0 || isPending}>
+                <Upload className="h-4 w-4 mr-1" />
+                Upload {files.length > 0 ? `(${files.length})` : ""}
+              </Button>
+            </>
+          )}
+          {(status === "done" || status === "error") && (
+            <Button variant="outline" onClick={() => onOpenChange(false)}>Close</Button>
+          )}
+        </DialogFooter>
+      </DialogContent>
+    </Dialog>
+  );
+}
@@ -10,6 +10,7 @@ import {
  createManualGroup,
  removePackageFromGroup,
  dissolveGroup,
+  mergeGroups,
 } from "@/lib/telegram/queries";

 const ALLOWED_IMAGE_TYPES = [
@@ -185,6 +186,62 @@ export async function setPreviewFromExtract(
  }
 }

+export async function repairPackageAction(
+  packageId: string
+): Promise<ActionResult> {
+  const session = await auth();
+  if (!session?.user?.id) return { success: false, error: "Unauthorized" };
+
+  try {
+    const pkg = await prisma.package.findUnique({
+      where: { id: packageId },
+      select: {
+        id: true,
+        fileName: true,
+        sourceChannelId: true,
+        sourceMessageId: true,
+        destChannelId: true,
+        destMessageId: true,
+      },
+    });
+
+    if (!pkg) return { success: false, error: "Package not found" };
+
+    // Clear the destination info so the worker re-processes it
+    await prisma.package.update({
+      where: { id: packageId },
+      data: {
+        destMessageId: null,
+        destMessageIds: [],
+        destChannelId: null,
+      },
+    });
+
+    // Reset the channel watermark to before this message so worker picks it up
+    await prisma.accountChannelMap.updateMany({
+      where: {
+        channelId: pkg.sourceChannelId,
+        lastProcessedMessageId: { gte: pkg.sourceMessageId },
+      },
+      data: { lastProcessedMessageId: pkg.sourceMessageId - BigInt(1) },
+    });
+
+    // Mark related notifications as read
+    await prisma.systemNotification.updateMany({
+      where: {
+        context: { path: ["packageId"], equals: packageId },
+        isRead: false,
+      },
+      data: { isRead: true },
+    });
+
+    revalidatePath("/stls");
+    return { success: true, data: undefined };
+  } catch {
+    return { success: false, error: "Failed to schedule repair" };
+  }
+}
+
 export async function retrySkippedPackageAction(
  id: string
 ): Promise<ActionResult> {
@@ -435,6 +492,26 @@ export async function updateGroupPreviewAction(
  }
 }

+export async function mergeGroupsAction(
+  targetGroupId: string,
+  sourceGroupId: string
+): Promise<ActionResult> {
+  const session = await auth();
+  if (!session?.user?.id) return { success: false, error: "Unauthorized" };
+
+  if (targetGroupId === sourceGroupId) {
+    return { success: false, error: "Cannot merge a group with itself" };
+  }
+
+  try {
+    await mergeGroups(targetGroupId, sourceGroupId);
+    revalidatePath("/stls");
+    return { success: true, data: undefined };
+  } catch {
+    return { success: false, error: "Failed to merge groups" };
+  }
+}
+
 export async function sendAllInGroupAction(
  groupId: string
 ): Promise<ActionResult> {
@@ -512,3 +589,82 @@ export async function sendAllInGroupAction(
    return { success: false, error: "Failed to send group packages" };
  }
 }
+
+export async function sendAllFromCreatorAction(
+  creatorName: string
+): Promise<ActionResult<{ queued: number; skipped: number }>> {
+  const session = await auth();
+  if (!session?.user?.id) return { success: false, error: "Unauthorized" };
+
+  const creator = creatorName.trim();
+  if (!creator) {
+    return { success: false, error: "No creator specified" };
+  }
+
+  try {
+    const telegramLink = await prisma.telegramLink.findUnique({
+      where: { userId: session.user.id },
+    });
+
+    if (!telegramLink) {
+      return { success: false, error: "No linked Telegram account. Link one in Settings." };
+    }
+
+    const sendablePackages = await prisma.package.findMany({
+      where: {
+        creator,
+        destChannelId: { not: null },
+        destMessageId: { not: null },
+      },
+      select: { id: true },
+    });
+
+    if (sendablePackages.length === 0) {
+      return { success: false, error: "No uploaded packages found for this creator" };
+    }
+
+    let queued = 0;
+    let skipped = 0;
+    for (const pkg of sendablePackages) {
+      // Only create if no existing PENDING/SENDING request for this package+link combo
+      const existing = await prisma.botSendRequest.findFirst({
+        where: {
+          packageId: pkg.id,
+          telegramLinkId: telegramLink.id,
+          status: { in: ["PENDING", "SENDING"] },
+        },
+      });
+
+      if (existing) {
+        skipped++;
+        continue;
+      }
+
+      const sendRequest = await prisma.botSendRequest.create({
+        data: {
+          packageId: pkg.id,
+          telegramLinkId: telegramLink.id,
+          requestedByUserId: session.user.id,
+          status: "PENDING",
+        },
+      });
+
+      // Notify the bot via pg_notify
+      try {
+        await prisma.$queryRawUnsafe(
+          `SELECT pg_notify('bot_send', $1)`,
+          sendRequest.id
+        );
+      } catch {
+        // Best-effort — the bot also polls periodically
+      }
+
+      queued++;
+    }
+
+    revalidatePath("/stls");
+    return { success: true, data: { queued, skipped } };
+  } catch {
+    return { success: false, error: "Failed to send creator packages" };
+  }
+}
@@ -1,6 +1,6 @@
 import { auth } from "@/lib/auth";
 import { redirect } from "next/navigation";
-import { listDisplayItems, searchPackages, getIngestionStatus, getAllPackageTags, listSkippedPackages, countSkippedPackages } from "@/lib/telegram/queries";
+import { listDisplayItems, searchPackages, getIngestionStatus, getAllPackageTags, getAllPackageCreators, listSkippedPackages, countSkippedPackages, listUngroupedPackages, countUngroupedPackages } from "@/lib/telegram/queries";
 import { StlTable } from "./_components/stl-table";
 import type { DisplayItem, PackageListItem } from "@/lib/telegram/types";

@@ -24,7 +24,7 @@ export default async function StlFilesPage({ searchParams }: Props) {
  const tab = (params.tab as string) ?? "packages";

  // Fetch packages, ingestion status, tags, and skipped count in parallel
-  const [result, ingestionStatus, availableTags, skippedCount] = await Promise.all([
+  const [result, ingestionStatus, availableTags, availableCreators, skippedCount, ungroupedCount] = await Promise.all([
    search
      ? searchPackages({
          query: search,
@@ -42,7 +42,9 @@ export default async function StlFilesPage({ searchParams }: Props) {
        }),
    getIngestionStatus(),
    getAllPackageTags(),
+    getAllPackageCreators(),
    countSkippedPackages(),
+    countUngroupedPackages(),
  ]);

  // For search results, wrap as DisplayItem[]; for non-search, already DisplayItem[]
@@ -55,6 +57,11 @@ export default async function StlFilesPage({ searchParams }: Props) {
    ? await listSkippedPackages({ page, limit: perPage })
    : null;

+  // Fetch ungrouped packages only if on that tab
+  const ungroupedResult = tab === "ungrouped"
+    ? await listUngroupedPackages({ page, limit: perPage })
+    : null;
+
  return (
    <StlTable
      data={displayItems}
@@ -62,10 +69,14 @@ export default async function StlFilesPage({ searchParams }: Props) {
      totalCount={result.pagination.total}
      ingestionStatus={ingestionStatus}
      availableTags={availableTags}
+      availableCreators={availableCreators}
      searchTerm={search}
      skippedData={skippedResult?.items ?? []}
      skippedPageCount={skippedResult?.pagination.totalPages ?? 0}
      skippedTotalCount={skippedCount}
+      ungroupedData={ungroupedResult?.items ?? []}
+      ungroupedPageCount={ungroupedResult?.pagination.totalPages ?? 0}
+      ungroupedTotalCount={ungroupedCount}
    />
  );
 }
@@ -9,6 +9,7 @@ import {
  ArrowUpFromLine,
  RefreshCcw,
  Tag,
+  MessagesSquare,
 } from "lucide-react";
 import { Badge } from "@/components/ui/badge";
 import { Button } from "@/components/ui/button";
@@ -27,6 +28,7 @@ interface ChannelColumnsProps {
  onSetType: (id: string, type: "SOURCE" | "DESTINATION") => void;
  onRescan: (id: string) => void;
  onSetCategory: (id: string, category: string | null) => void;
+  onManageTopics: (id: string) => void;
 }

 export function getChannelColumns({
@@ -35,6 +37,7 @@ export function getChannelColumns({
  onSetType,
  onRescan,
  onSetCategory,
+  onManageTopics,
 }: ChannelColumnsProps): ColumnDef<ChannelRow, unknown>[] {
  return [
    {
@@ -147,6 +150,14 @@ export function getChannelColumns({
                Rescan Channel
              </DropdownMenuItem>
            )}
+            {row.original.type === "SOURCE" && row.original.isForum && (
+              <DropdownMenuItem
+                onClick={() => onManageTopics(row.original.id)}
+              >
+                <MessagesSquare className="mr-2 h-3.5 w-3.5" />
+                Topics
+              </DropdownMenuItem>
+            )}
            <DropdownMenuItem
              onClick={() => {
                const cat = prompt("Enter category (e.g. STL, PDF, D&D, Cosplay):", row.original.category ?? "");
@@ -7,6 +7,7 @@ import { getChannelColumns } from "./channel-columns";
 import { DestinationCard } from "./destination-card";
 import { ChannelPickerDialog } from "./channel-picker-dialog";
 import { JoinChannelDialog } from "./join-channel-dialog";
+import { TopicsDrawer } from "./topics-drawer";
 import {
  deleteChannel,
  toggleChannelActive,
@@ -32,6 +33,7 @@ export function ChannelsTab({ channels, globalDestination, accounts }: ChannelsT
  const [rescanId, setRescanId] = useState<string | null>(null);
  const [fetchChannelsAccountId, setFetchChannelsAccountId] = useState<string | null>(null);
  const [joinDialogOpen, setJoinDialogOpen] = useState(false);
+  const [topicsChannelId, setTopicsChannelId] = useState<string | null>(null);

  // Find the first authenticated account for "Fetch Channels"
  const authenticatedAccounts = accounts.filter((a) => a.authState === "AUTHENTICATED" && a.isActive);
@@ -60,6 +62,7 @@ export function ChannelsTab({ channels, globalDestination, accounts }: ChannelsT
        else toast.error(result.error);
      });
    },
+    onManageTopics: (id) => setTopicsChannelId(id),
  });

  const { table } = useDataTable({
@@ -167,6 +170,17 @@ export function ChannelsTab({ channels, globalDestination, accounts }: ChannelsT
        open={joinDialogOpen}
        onOpenChange={setJoinDialogOpen}
      />
+
+      <TopicsDrawer
+        channelId={topicsChannelId}
+        channelTitle={
+          channels.find((c) => c.id === topicsChannelId)?.title
+        }
+        open={!!topicsChannelId}
+        onOpenChange={(open) => {
+          if (!open) setTopicsChannelId(null);
+        }}
+      />
    </div>
  );
 }
@@ -0,0 +1,182 @@
+"use client";
+
+import { useState, useEffect, useCallback, useMemo, useTransition } from "react";
+import { toast } from "sonner";
+import { Loader2, Search } from "lucide-react";
+import {
+  Dialog,
+  DialogContent,
+  DialogDescription,
+  DialogHeader,
+  DialogTitle,
+} from "@/components/ui/dialog";
+import { Input } from "@/components/ui/input";
+import { Switch } from "@/components/ui/switch";
+import { setTopicFetchEnabled } from "../actions";
+
+interface TopicRow {
+  id: string;
+  topicId: string;
+  topicName: string | null;
+  fetchEnabled: boolean;
+  lastScannedAt: string | null;
+}
+
+interface TopicsDrawerProps {
+  channelId: string | null;
+  channelTitle?: string;
+  open: boolean;
+  onOpenChange: (open: boolean) => void;
+}
+
+function topicLabel(t: TopicRow): string {
+  return t.topicName ?? `Topic ${t.topicId}`;
+}
+
+export function TopicsDrawer({
+  channelId,
+  channelTitle,
+  open,
+  onOpenChange,
+}: TopicsDrawerProps) {
+  const [topics, setTopics] = useState<TopicRow[]>([]);
+  const [loading, setLoading] = useState(false);
+  const [filter, setFilter] = useState("");
+  const [, startTransition] = useTransition();
+  const [pendingId, setPendingId] = useState<string | null>(null);
+
+  const fetchTopics = useCallback(async () => {
+    if (!channelId) return;
+    setLoading(true);
+    try {
+      const res = await fetch(`/api/telegram/channels/${channelId}/topics`);
+      if (res.ok) setTopics(await res.json());
+      else toast.error("Failed to load topics");
+    } catch {
+      toast.error("Failed to load topics");
+    }
+    setLoading(false);
+  }, [channelId]);
+
+  useEffect(() => {
+    if (open && channelId) fetchTopics();
+  }, [open, channelId, fetchTopics]);
+
+  const handleOpenChange = (next: boolean) => {
+    // Reset the filter on close so the next open starts clean (the drawer is
+    // opened per-channel, so close-then-open is the normal channel switch).
+    if (!next) setFilter("");
+    onOpenChange(next);
+  };
+
+  const handleToggle = (topic: TopicRow, enabled: boolean) => {
+    // Optimistic update
+    setTopics((prev) =>
+      prev.map((t) => (t.id === topic.id ? { ...t, fetchEnabled: enabled } : t))
+    );
+    setPendingId(topic.id);
+    startTransition(async () => {
+      const result = await setTopicFetchEnabled(topic.id, enabled);
+      if (!result.success) {
+        toast.error(result.error ?? "Failed to update topic");
+        // Revert on failure
+        setTopics((prev) =>
+          prev.map((t) =>
+            t.id === topic.id ? { ...t, fetchEnabled: !enabled } : t
+          )
+        );
+      }
+      setPendingId(null);
+    });
+  };
+
+  const filtered = useMemo(() => {
+    const q = filter.trim().toLowerCase();
+    if (!q) return topics;
+    return topics.filter((t) => topicLabel(t).toLowerCase().includes(q));
+  }, [topics, filter]);
+
+  return (
+    <Dialog open={open} onOpenChange={handleOpenChange}>
+      <DialogContent className="sm:max-w-lg max-h-[80vh] flex flex-col gap-0 p-0">
+        <DialogHeader className="px-6 pt-6 pb-4 border-b border-border space-y-3">
+          <div className="space-y-1">
+            <DialogTitle className="truncate pr-8">
+              Topics{channelTitle ? `: ${channelTitle}` : ""}
+            </DialogTitle>
+            <DialogDescription>
+              Enabled topics are scanned and their files fetched. Disable a topic
+              to stop fetching new files from it — already-fetched files are kept.
+            </DialogDescription>
+          </div>
+          {topics.length > 0 && (
+            <div className="relative">
+              <Search className="absolute left-2.5 top-2.5 h-4 w-4 text-muted-foreground" />
+              <Input
+                placeholder="Filter topics..."
+                value={filter}
+                onChange={(e) => setFilter(e.target.value)}
+                className="pl-9 h-9"
+              />
+            </div>
+          )}
+        </DialogHeader>
+
+        {/* Native overflow scroll: the Radix ScrollArea viewport does not get a
+            bounded height inside this flex-column, max-h, vertically-centred
+            dialog, so the list overflowed instead of scrolling. flex-1 +
+            min-h-0 + overflow-y-auto is the canonical, touch-friendly fix. */}
+        <div className="flex-1 min-h-0 overflow-y-auto">
+          <div className="px-6 py-4 space-y-2">
+            {loading ? (
+              <div className="flex items-center justify-center gap-2 py-12">
+                <Loader2 className="h-5 w-5 animate-spin text-muted-foreground" />
+                <span className="text-sm text-muted-foreground">
+                  Loading topics...
+                </span>
+              </div>
+            ) : topics.length === 0 ? (
+              <p className="py-12 text-center text-sm text-muted-foreground">
+                No topics discovered yet — they&apos;ll appear here after the next
+                scan.
+              </p>
+            ) : filtered.length === 0 ? (
+              <p className="py-12 text-center text-sm text-muted-foreground">
+                No topics match &quot;{filter}&quot;.
+              </p>
+            ) : (
+              filtered.map((topic) => (
+                <div
+                  key={topic.id}
+                  className="flex items-center justify-between gap-3 rounded-md border p-3"
+                >
+                  <div className="min-w-0">
+                    <p className="truncate text-sm font-medium">
+                      {topicLabel(topic)}
+                    </p>
+                    <p className="text-xs text-muted-foreground">
+                      {topic.fetchEnabled
+                        ? "Fetching enabled"
+                        : "Fetching disabled"}
+                      {topic.lastScannedAt
+                        ? ` · last scanned ${new Date(
+                            topic.lastScannedAt
+                          ).toLocaleDateString()}`
+                        : " · not scanned yet"}
+                    </p>
+                  </div>
+                  <Switch
+                    checked={topic.fetchEnabled}
+                    disabled={pendingId === topic.id}
+                    onCheckedChange={(checked) => handleToggle(topic, checked)}
+                    aria-label={`Toggle fetching for ${topicLabel(topic)}`}
+                  />
+                </div>
+              ))
+            )}
+          </div>
+        </div>
+      </DialogContent>
+    </Dialog>
+  );
+}
@@ -9,13 +9,14 @@ import {
  Radio,
  AlertTriangle,
  RefreshCw,
+  SkipForward,
 } from "lucide-react";
 import { Card, CardContent } from "@/components/ui/card";
 import { Badge } from "@/components/ui/badge";
 import { Button } from "@/components/ui/button";
 import { cn } from "@/lib/utils";
 import { toast } from "sonner";
-import { triggerIngestion } from "../actions";
+import { triggerIngestion, disableActiveTopic } from "../actions";
 import type { IngestionAccountStatus } from "@/lib/telegram/types";

 interface WorkerStatusPanelProps {
@@ -218,6 +219,24 @@ function RunningStatus({
 }: {
  run: NonNullable<IngestionAccountStatus["currentRun"]>;
 }) {
+  const [isDisabling, startDisable] = useTransition();
+
+  const handleSkipTopic = () => {
+    const acmId = run.currentAccountChannelMapId;
+    const topicId = run.currentTopicId;
+    if (!acmId || !topicId) return;
+    startDisable(async () => {
+      const result = await disableActiveTopic(acmId, topicId);
+      if (result.success) {
+        toast.success(
+          "Topic disabled — the current file finishes, then the rest is skipped"
+        );
+      } else {
+        toast.error(result.error ?? "Failed to disable topic");
+      }
+    });
+  };
+
  return (
    <div className="space-y-2">
      <div className="flex items-center gap-2">
@@ -273,6 +292,27 @@ function RunningStatus({
          </span>
        )}
      </div>
+
+      {/* Skip & disable the topic currently being processed */}
+      {run.currentTopicId && run.currentAccountChannelMapId && (
+        <div className="pl-6 pt-1">
+          <Button
+            variant="outline"
+            size="sm"
+            className="h-7 gap-1.5 text-xs"
+            onClick={handleSkipTopic}
+            disabled={isDisabling}
+            title="Finish the current file, skip the rest of this topic, and don't fetch it in future runs"
+          >
+            {isDisabling ? (
+              <Loader2 className="h-3 w-3 animate-spin" />
+            ) : (
+              <SkipForward className="h-3 w-3" />
+            )}
+            Skip &amp; disable this topic
+          </Button>
+        </div>
+      )}
    </div>
  );
 }
@@ -291,10 +291,25 @@ export async function setChannelCategory(
  if (!admin.success) return admin;

  try {
+    const existing = await prisma.telegramChannel.findUnique({
+      where: { id },
+      select: { category: true },
+    });
+    if (!existing) return { success: false, error: "Channel not found" };
+
+    const oldCategory = existing.category;
+    const newCategory = category?.trim() || null;
+
    await prisma.telegramChannel.update({
      where: { id },
-      data: { category: category?.trim() || null },
+      data: { category: newCategory },
    });
+
+    // Retroactively re-tag packages from this channel when category changes
+    if (oldCategory !== newCategory && newCategory) {
+      await retagChannelPackages(id, oldCategory, newCategory);
+    }
+
    revalidatePath("/telegram");
    return { success: true, data: undefined };
  } catch {
@@ -302,6 +317,50 @@ export async function setChannelCategory(
  }
 }

+export async function retagChannelPackages(
+  channelId: string,
+  oldCategory: string | null,
+  newCategory: string
+): Promise<ActionResult<{ updated: number }>> {
+  const session = await auth();
+  if (!session?.user?.id) return { success: false, error: "Unauthorized" };
+
+  try {
+    // Find packages from this channel that have the old category tag (or no category tag)
+    const packages = await prisma.package.findMany({
+      where: { sourceChannelId: channelId },
+      select: { id: true, tags: true },
+    });
+
+    let updated = 0;
+    for (const pkg of packages) {
+      const tags = [...pkg.tags];
+      // Remove old category tag if present
+      if (oldCategory) {
+        const idx = tags.indexOf(oldCategory);
+        if (idx !== -1) tags.splice(idx, 1);
+      }
+      // Add new category tag if not already present
+      if (!tags.includes(newCategory)) {
+        tags.push(newCategory);
+      }
+      // Only update if tags actually changed
+      if (JSON.stringify(tags) !== JSON.stringify(pkg.tags)) {
+        await prisma.package.update({
+          where: { id: pkg.id },
+          data: { tags },
+        });
+        updated++;
+      }
+    }
+
+    revalidatePath("/stls");
+    return { success: true, data: { updated } };
+  } catch {
+    return { success: false, error: "Failed to re-tag packages" };
+  }
+}
+
 export async function setChannelType(
  id: string,
  type: "SOURCE" | "DESTINATION"
@@ -377,6 +436,80 @@ export async function rescanChannel(channelId: string): Promise<ActionResult> {
  }
 }

+// ── Topic fetch toggle ──
+
+export async function setTopicFetchEnabled(
+  topicProgressId: string,
+  enabled: boolean
+): Promise<ActionResult> {
+  const admin = await requireAdmin();
+  if (!admin.success) return admin;
+
+  const existing = await prisma.topicProgress.findUnique({
+    where: { id: topicProgressId },
+  });
+  if (!existing) return { success: false, error: "Topic not found" };
+
+  try {
+    await prisma.topicProgress.update({
+      where: { id: topicProgressId },
+      data: { fetchEnabled: enabled },
+    });
+    revalidatePath(REVALIDATE_PATH);
+    return { success: true, data: undefined };
+  } catch {
+    return { success: false, error: "Failed to update topic" };
+  }
+}
+
+/**
+ * Disable the topic currently being processed by the worker, identified by its
+ * account-channel mapping + Telegram topic id (as exposed on the live run
+ * status). Upserts so a disabled row exists even if the worker hasn't persisted
+ * the topic yet. The worker honours this live: it finishes the in-flight file
+ * then skips the rest of the topic, and future runs skip it entirely.
+ */
+export async function disableActiveTopic(
+  accountChannelMapId: string,
+  topicId: string
+): Promise<ActionResult> {
+  const admin = await requireAdmin();
+  if (!admin.success) return admin;
+
+  let topicIdBig: bigint;
+  try {
+    topicIdBig = BigInt(topicId);
+  } catch {
+    return { success: false, error: "Invalid topic id" };
+  }
+  // BigInt("") / BigInt("0") don't throw — reject non-positive ids explicitly
+  // since this action is exported and admin-callable outside the UI button.
+  if (topicIdBig <= BigInt(0)) {
+    return { success: false, error: "Invalid topic id" };
+  }
+
+  try {
+    await prisma.topicProgress.upsert({
+      where: {
+        accountChannelMapId_topicId: {
+          accountChannelMapId,
+          topicId: topicIdBig,
+        },
+      },
+      create: {
+        accountChannelMapId,
+        topicId: topicIdBig,
+        fetchEnabled: false,
+      },
+      update: { fetchEnabled: false },
+    });
+    revalidatePath(REVALIDATE_PATH);
+    return { success: true, data: undefined };
+  } catch {
+    return { success: false, error: "Failed to disable topic" };
+  }
+}
+
 // ── Account-Channel link actions ──

 export async function linkChannel(
@@ -0,0 +1,33 @@
+import { NextResponse } from "next/server";
+import { auth } from "@/lib/auth";
+import {
+  markNotificationRead,
+  markAllNotificationsRead,
+  dismissNotification,
+  clearAllNotifications,
+} from "@/data/notification.queries";
+
+export const dynamic = "force-dynamic";
+
+export async function POST(request: Request) {
+  const session = await auth();
+  if (!session?.user?.id) {
+    return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
+  }
+
+  const body = await request.json().catch(() => ({}));
+  const id = body.id as string | undefined;
+  const action = (body.action as string) ?? "read";
+
+  if (action === "dismiss" && id) {
+    await dismissNotification(id);
+  } else if (action === "clear") {
+    await clearAllNotifications();
+  } else if (id) {
+    await markNotificationRead(id);
+  } else {
+    await markAllNotificationsRead();
+  }
+
+  return NextResponse.json({ success: true });
+}
@@ -0,0 +1,43 @@
+import { NextResponse } from "next/server";
+import { auth } from "@/lib/auth";
+import { prisma } from "@/lib/prisma";
+
+export const dynamic = "force-dynamic";
+
+export async function POST(request: Request) {
+  const session = await auth();
+  if (!session?.user?.id) {
+    return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
+  }
+
+  const body = await request.json().catch(() => ({}));
+  const notificationId = body.notificationId as string;
+  if (!notificationId) {
+    return NextResponse.json({ error: "notificationId required" }, { status: 400 });
+  }
+
+  const notification = await prisma.systemNotification.findUnique({
+    where: { id: notificationId },
+  });
+
+  if (!notification) {
+    return NextResponse.json({ error: "Notification not found" }, { status: 404 });
+  }
+
+  const context = notification.context as Record<string, unknown> | null;
+  const packageId = context?.packageId as string | undefined;
+
+  if (!packageId) {
+    return NextResponse.json({ error: "Notification has no associated package" }, { status: 400 });
+  }
+
+  // Import and call the repair action
+  const { repairPackageAction } = await import("@/app/(app)/stls/actions");
+  const result = await repairPackageAction(packageId);
+
+  if (!result.success) {
+    return NextResponse.json({ error: result.error }, { status: 500 });
+  }
+
+  return NextResponse.json({ success: true });
+}
@@ -0,0 +1,27 @@
+import { NextResponse } from "next/server";
+import { auth } from "@/lib/auth";
+import {
+  getRecentNotifications,
+  getUnreadNotificationCount,
+} from "@/data/notification.queries";
+
+export const dynamic = "force-dynamic";
+
+export async function GET() {
+  const session = await auth();
+  if (!session?.user?.id) {
+    return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
+  }
+
+  const [notifications, unreadCount] = await Promise.all([
+    getRecentNotifications(30),
+    getUnreadNotificationCount(),
+  ]);
+
+  const serialized = notifications.map((n) => ({
+    ...n,
+    createdAt: n.createdAt.toISOString(),
+  }));
+
+  return NextResponse.json({ notifications: serialized, unreadCount });
+}
@@ -0,0 +1,17 @@
+import { NextResponse } from "next/server";
+import { authenticateApiRequest } from "@/lib/telegram/api-auth";
+import { listChannelTopics } from "@/lib/telegram/admin-queries";
+
+export const dynamic = "force-dynamic";
+
+export async function GET(
+  request: Request,
+  { params }: { params: Promise<{ channelId: string }> }
+) {
+  const authResult = await authenticateApiRequest(request, true);
+  if ("error" in authResult) return authResult.error;
+
+  const { channelId } = await params;
+  const topics = await listChannelTopics(channelId);
+  return NextResponse.json(topics);
+}
@@ -0,0 +1,43 @@
+import { NextResponse } from "next/server";
+import { auth } from "@/lib/auth";
+import { prisma } from "@/lib/prisma";
+
+export const dynamic = "force-dynamic";
+
+export async function GET(
+  _request: Request,
+  { params }: { params: Promise<{ id: string }> }
+) {
+  const session = await auth();
+  if (!session?.user?.id) {
+    return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
+  }
+
+  const { id } = await params;
+
+  const upload = await prisma.manualUpload.findUnique({
+    where: { id },
+    include: {
+      files: {
+        select: { id: true, fileName: true, fileSize: true, packageId: true },
+      },
+    },
+  });
+
+  if (!upload || upload.userId !== session.user.id) {
+    return NextResponse.json({ error: "Not found" }, { status: 404 });
+  }
+
+  return NextResponse.json({
+    id: upload.id,
+    status: upload.status,
+    groupName: upload.groupName,
+    errorMessage: upload.errorMessage,
+    files: upload.files.map((f) => ({
+      ...f,
+      fileSize: f.fileSize.toString(),
+    })),
+    createdAt: upload.createdAt.toISOString(),
+    completedAt: upload.completedAt?.toISOString() ?? null,
+  });
+}
@@ -0,0 +1,83 @@
+import { NextResponse } from "next/server";
+import { auth } from "@/lib/auth";
+import { prisma } from "@/lib/prisma";
+import { writeFile, mkdir } from "fs/promises";
+import path from "path";
+
+export const dynamic = "force-dynamic";
+
+const UPLOAD_DIR = process.env.UPLOAD_DIR ?? "/data/uploads";
+const MAX_FILE_SIZE = 4 * 1024 * 1024 * 1024; // 4GB per file
+
+export async function POST(request: Request) {
+  const session = await auth();
+  if (!session?.user?.id) {
+    return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
+  }
+
+  try {
+    const formData = await request.formData();
+    const files = formData.getAll("files") as File[];
+    const groupName = formData.get("groupName") as string | null;
+
+    if (!files.length) {
+      return NextResponse.json({ error: "No files provided" }, { status: 400 });
+    }
+
+    // Create the upload record
+    const upload = await prisma.manualUpload.create({
+      data: {
+        userId: session.user.id,
+        groupName: groupName || (files.length > 1 ? files[0].name.replace(/\.[^.]+$/, "") : null),
+        status: "PENDING",
+      },
+    });
+
+    // Save files to shared volume
+    const uploadDir = path.join(UPLOAD_DIR, upload.id);
+    await mkdir(uploadDir, { recursive: true });
+
+    for (const file of files) {
+      if (file.size > MAX_FILE_SIZE) {
+        return NextResponse.json(
+          { error: `File "${file.name}" exceeds 4GB limit` },
+          { status: 400 }
+        );
+      }
+
+      const filePath = path.join(uploadDir, file.name);
+      const buffer = Buffer.from(await file.arrayBuffer());
+      await writeFile(filePath, buffer);
+
+      await prisma.manualUploadFile.create({
+        data: {
+          uploadId: upload.id,
+          fileName: file.name,
+          filePath,
+          fileSize: BigInt(file.size),
+        },
+      });
+    }
+
+    // Notify worker
+    try {
+      await prisma.$queryRawUnsafe(
+        `SELECT pg_notify('manual_upload', $1)`,
+        upload.id
+      );
+    } catch {
+      // Best-effort
+    }
+
+    return NextResponse.json({
+      uploadId: upload.id,
+      fileCount: files.length,
+      status: "PENDING",
+    });
+  } catch (err) {
+    return NextResponse.json(
+      { error: err instanceof Error ? err.message : "Upload failed" },
+      { status: 500 }
+    );
+  }
+}
@@ -0,0 +1,9 @@
+import { signOut } from "@/lib/auth";
+
+// Server-side sign-out that clears the JWT session cookie and redirects to the
+// login page. Used to recover from a stale session whose user no longer exists
+// in the database (e.g. after a DB reset), which a client-only signOut can't
+// reach because the app crashes before rendering the user menu.
+export async function GET() {
+  await signOut({ redirectTo: "/login" });
+}
@@ -6,6 +6,7 @@ import { Button } from "@/components/ui/button";
 import { Sheet, SheetContent, SheetTrigger } from "@/components/ui/sheet";
 import { UserMenu } from "./user-menu";
 import { MobileSidebar } from "./mobile-sidebar";
+import { NotificationBell } from "./notification-bell";

 const routeTitles: Record<string, string> = {
  "/dashboard": "Dashboard",
@@ -38,7 +39,8 @@ export function Header() {

      <h1 className="text-lg font-semibold">{title}</h1>

-      <div className="ml-auto">
+      <div className="ml-auto flex items-center gap-1">
+        <NotificationBell />
        <UserMenu />
      </div>
    </header>
@@ -0,0 +1,268 @@
+"use client";
+
+import { useState, useEffect, useCallback } from "react";
+import { Bell, AlertTriangle, AlertCircle, Info, CheckCircle2, X, Trash2 } from "lucide-react";
+import { Button } from "@/components/ui/button";
+import { Badge } from "@/components/ui/badge";
+import {
+  Popover,
+  PopoverContent,
+  PopoverTrigger,
+} from "@/components/ui/popover";
+import { ScrollArea } from "@/components/ui/scroll-area";
+import { toast } from "sonner";
+
+interface Notification {
+  id: string;
+  type: string;
+  severity: "INFO" | "WARNING" | "ERROR";
+  title: string;
+  message: string;
+  isRead: boolean;
+  createdAt: string;
+}
+
+const severityIcon = {
+  INFO: Info,
+  WARNING: AlertTriangle,
+  ERROR: AlertCircle,
+};
+
+const severityColor = {
+  INFO: "text-blue-400",
+  WARNING: "text-orange-400",
+  ERROR: "text-red-400",
+};
+
+export function NotificationBell() {
+  const [notifications, setNotifications] = useState<Notification[]>([]);
+  const [unreadCount, setUnreadCount] = useState(0);
+  const [open, setOpen] = useState(false);
+
+  const fetchNotifications = useCallback(async () => {
+    try {
+      const res = await fetch("/api/notifications");
+      if (res.ok) {
+        const data = await res.json();
+        setNotifications(data.notifications ?? []);
+        setUnreadCount(data.unreadCount ?? 0);
+      }
+    } catch {
+      // Ignore fetch errors
+    }
+  }, []);
+
+  // Poll every 30 seconds + on mount
+  useEffect(() => {
+    fetchNotifications();
+    const interval = setInterval(fetchNotifications, 30_000);
+    return () => clearInterval(interval);
+  }, [fetchNotifications]);
+
+  // Refresh when popover opens
+  useEffect(() => {
+    if (open) fetchNotifications();
+  }, [open, fetchNotifications]);
+
+  async function handleMarkAllRead() {
+    try {
+      await fetch("/api/notifications/read", {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({}),
+      });
+      setNotifications((prev) => prev.map((n) => ({ ...n, isRead: true })));
+      setUnreadCount(0);
+    } catch {
+      // Ignore
+    }
+  }
+
+  async function handleMarkRead(id: string) {
+    try {
+      await fetch("/api/notifications/read", {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({ id }),
+      });
+      setNotifications((prev) =>
+        prev.map((n) => (n.id === id ? { ...n, isRead: true } : n))
+      );
+      setUnreadCount((c) => Math.max(0, c - 1));
+    } catch {
+      // Ignore
+    }
+  }
+
+  async function handleDismiss(id: string) {
+    try {
+      await fetch("/api/notifications/read", {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({ id, action: "dismiss" }),
+      });
+      setNotifications((prev) => prev.filter((n) => n.id !== id));
+      setUnreadCount((c) => Math.max(0, c - 1));
+    } catch {
+      // Ignore
+    }
+  }
+
+  async function handleClearAll() {
+    try {
+      await fetch("/api/notifications/read", {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({ action: "clear" }),
+      });
+      setNotifications([]);
+      setUnreadCount(0);
+    } catch {
+      // Ignore
+    }
+  }
+
+  async function handleRepair(notificationId: string) {
+    try {
+      const res = await fetch("/api/notifications/repair", {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({ notificationId }),
+      });
+      if (res.ok) {
+        toast.success("Repair scheduled — package will be re-processed on next cycle");
+        fetchNotifications();
+      }
+    } catch {
+      // Ignore
+    }
+  }
+
+  function formatTime(iso: string): string {
+    const d = new Date(iso);
+    const now = new Date();
+    const diffMs = now.getTime() - d.getTime();
+    const diffMin = Math.floor(diffMs / 60_000);
+    if (diffMin < 1) return "just now";
+    if (diffMin < 60) return `${diffMin}m ago`;
+    const diffHr = Math.floor(diffMin / 60);
+    if (diffHr < 24) return `${diffHr}h ago`;
+    const diffDay = Math.floor(diffHr / 24);
+    return `${diffDay}d ago`;
+  }
+
+  return (
+    <Popover open={open} onOpenChange={setOpen}>
+      <PopoverTrigger asChild>
+        <Button variant="ghost" size="icon" className="relative h-9 w-9">
+          <Bell className="h-4 w-4" />
+          {unreadCount > 0 && (
+            <Badge
+              variant="destructive"
+              className="absolute -top-1 -right-1 h-4 min-w-4 px-1 text-[10px] leading-none"
+            >
+              {unreadCount > 99 ? "99+" : unreadCount}
+            </Badge>
+          )}
+        </Button>
+      </PopoverTrigger>
+      <PopoverContent className="w-96 p-0" align="end">
+        <div className="flex items-center justify-between border-b px-4 py-3">
+          <h3 className="text-sm font-semibold">Notifications</h3>
+          <div className="flex items-center gap-1">
+            {unreadCount > 0 && (
+              <Button
+                variant="ghost"
+                size="sm"
+                className="h-7 text-xs"
+                onClick={handleMarkAllRead}
+              >
+                Mark all read
+              </Button>
+            )}
+            {notifications.length > 0 && (
+              <Button
+                variant="ghost"
+                size="sm"
+                className="h-7 text-xs text-muted-foreground"
+                onClick={handleClearAll}
+              >
+                <Trash2 className="h-3 w-3 mr-1" />
+                Clear
+              </Button>
+            )}
+          </div>
+        </div>
+        <ScrollArea className="max-h-[400px]">
+          {notifications.length === 0 ? (
+            <div className="flex flex-col items-center justify-center py-8 text-muted-foreground">
+              <CheckCircle2 className="h-8 w-8 mb-2 opacity-50" />
+              <p className="text-sm">All clear!</p>
+            </div>
+          ) : (
+            <div className="divide-y">
+              {notifications.map((n) => {
+                const Icon = severityIcon[n.severity] ?? Info;
+                const color = severityColor[n.severity] ?? "text-muted-foreground";
+                return (
+                  <div
+                    key={n.id}
+                    className={`flex w-full gap-3 px-4 py-3 text-left hover:bg-muted/50 transition-colors ${
+                      !n.isRead ? "bg-muted/20" : ""
+                    }`}
+                    role="button"
+                    tabIndex={0}
+                    onClick={() => !n.isRead && handleMarkRead(n.id)}
+                    onKeyDown={(e) => {
+                      if (e.key === "Enter" || e.key === " ") {
+                        if (!n.isRead) handleMarkRead(n.id);
+                      }
+                    }}
+                  >
+                    <Icon className={`h-4 w-4 mt-0.5 shrink-0 ${color}`} />
+                    <div className="flex-1 min-w-0">
+                      <div className="flex items-center gap-2">
+                        <p className={`text-sm truncate ${!n.isRead ? "font-medium" : ""}`}>
+                          {n.title}
+                        </p>
+                        {!n.isRead && (
+                          <span className="h-2 w-2 rounded-full bg-primary shrink-0" />
+                        )}
+                        <button
+                          className="ml-auto shrink-0 p-0.5 rounded hover:bg-muted text-muted-foreground hover:text-foreground"
+                          onClick={(e) => { e.stopPropagation(); handleDismiss(n.id); }}
+                          title="Dismiss"
+                        >
+                          <X className="h-3 w-3" />
+                        </button>
+                      </div>
+                      <p className="text-xs text-muted-foreground line-clamp-2 mt-0.5">
+                        {n.message}
+                      </p>
+                      <p className="text-[10px] text-muted-foreground mt-1">
+                        {formatTime(n.createdAt)}
+                      </p>
+                      {(n.type === "MISSING_PART" || n.type === "HASH_MISMATCH") && (
+                        <Button
+                          variant="outline"
+                          size="sm"
+                          className="h-6 px-2 text-xs mt-1"
+                          onClick={(e) => {
+                            e.stopPropagation();
+                            handleRepair(n.id);
+                          }}
+                        >
+                          Repair
+                        </Button>
+                      )}
+                    </div>
+                  </div>
+                );
+              })}
+            </div>
+          )}
+        </ScrollArea>
+      </PopoverContent>
+    </Popover>
+  );
+}
@@ -0,0 +1,45 @@
+import { prisma } from "@/lib/prisma";
+
+export async function getUnreadNotificationCount(): Promise<number> {
+  return prisma.systemNotification.count({
+    where: { isRead: false },
+  });
+}
+
+export async function getRecentNotifications(limit = 20) {
+  return prisma.systemNotification.findMany({
+    orderBy: { createdAt: "desc" },
+    take: limit,
+    select: {
+      id: true,
+      type: true,
+      severity: true,
+      title: true,
+      message: true,
+      isRead: true,
+      createdAt: true,
+    },
+  });
+}
+
+export async function markNotificationRead(id: string) {
+  return prisma.systemNotification.update({
+    where: { id },
+    data: { isRead: true },
+  });
+}
+
+export async function markAllNotificationsRead() {
+  return prisma.systemNotification.updateMany({
+    where: { isRead: false },
+    data: { isRead: true },
+  });
+}
+
+export async function dismissNotification(id: string) {
+  return prisma.systemNotification.delete({ where: { id } });
+}
+
+export async function clearAllNotifications() {
+  return prisma.systemNotification.deleteMany({});
+}
@@ -1,20 +1,37 @@
+import { Prisma } from "@prisma/client";
 import { prisma } from "@/lib/prisma";

+const DEFAULT_SETTINGS = {
+  lowStockThreshold: 20,
+  currency: "EUR",
+  theme: "dark",
+  units: "metric",
+} as const;
+
 export async function getUserSettings(userId: string) {
  let settings = await prisma.userSettings.findUnique({
    where: { userId },
  });

  if (!settings) {
+    try {
      settings = await prisma.userSettings.create({
-      data: {
-        userId,
-        lowStockThreshold: 20,
-        currency: "EUR",
-        theme: "dark",
-        units: "metric",
-      },
+        data: { userId, ...DEFAULT_SETTINGS },
      });
+    } catch (err) {
+      // The session's user may no longer exist (e.g. a stale JWT cookie after a
+      // database reset). Creating settings then hits a foreign-key violation
+      // (P2003). Don't crash the Server Component render — return unsaved
+      // defaults. The (app) layout guard redirects such stale sessions to
+      // sign-out, so this fallback is only ever momentarily visible.
+      if (
+        err instanceof Prisma.PrismaClientKnownRequestError &&
+        err.code === "P2003"
+      ) {
+        return { id: "", userId, ...DEFAULT_SETTINGS };
+      }
+      throw err;
+    }
  }

  return settings;
@@ -41,6 +41,7 @@ export async function listChannels() {
    telegramId: c.telegramId.toString(),
    title: c.title,
    type: c.type,
+    isForum: c.isForum,
    isActive: c.isActive,
    category: c.category,
    createdAt: c.createdAt.toISOString(),
@@ -140,3 +141,24 @@ export async function getUnlinkedChannels(accountId: string) {
    telegramId: c.telegramId.toString(),
  }));
 }
+
+// ── Channel topic queries ──
+
+export async function listChannelTopics(channelId: string) {
+  const rows = await prisma.topicProgress.findMany({
+    where: { accountChannelMap: { channelId } },
+    orderBy: [{ fetchEnabled: "desc" }, { topicName: "asc" }],
+  });
+
+  return rows.map((r) => ({
+    id: r.id,
+    topicId: r.topicId.toString(),
+    topicName: r.topicName,
+    fetchEnabled: r.fetchEnabled,
+    lastScannedAt: r.lastScannedAt?.toISOString() ?? null,
+  }));
+}
+
+export type ChannelTopicRow = Awaited<
+  ReturnType<typeof listChannelTopics>
+>[number];
@@ -9,6 +9,28 @@ import type {
  PackageGroupRow,
 } from "./types";

+/**
+ * Returns the subset of the given IDs whose `previewData` column is non-null,
+ * WITHOUT transferring the (large) image bytes.
+ *
+ * List views only need a `hasPreview` boolean per row. Selecting `previewData`
+ * directly pulls every JPEG blob (avg ~700 KB, up to 2 MB) into the Node heap
+ * just to compare it against null — under concurrent requests this exhausts the
+ * container memory limit and crashes the process. This keeps the check in SQL.
+ */
+async function fetchPreviewFlags(
+  table: "packages" | "package_groups",
+  ids: string[]
+): Promise<Set<string>> {
+  if (ids.length === 0) return new Set();
+  const placeholders = ids.map((_, i) => `$${i + 1}`).join(", ");
+  const rows = await prisma.$queryRawUnsafe<{ id: string }[]>(
+    `SELECT id FROM ${table} WHERE "previewData" IS NOT NULL AND id IN (${placeholders})`,
+    ...ids
+  );
+  return new Set(rows.map((r) => r.id));
+}
+
 export async function listPackages(options: {
  page: number;
  limit: number;
@@ -40,13 +62,17 @@ export async function listPackages(options: {
        indexedAt: true,
        creator: true,
        tags: true,
-        previewData: true, // check actual image data, not previewMsgId proxy
        sourceChannel: { select: { id: true, title: true } },
      },
    }),
    prisma.package.count({ where }),
  ]);

+  const previewIds = await fetchPreviewFlags(
+    "packages",
+    items.map((p) => p.id)
+  );
+
  const mapped: PackageListItem[] = items.map((pkg) => ({
    id: pkg.id,
    fileName: pkg.fileName,
@@ -55,7 +81,7 @@ export async function listPackages(options: {
    archiveType: pkg.archiveType,
    fileCount: pkg.fileCount,
    isMultipart: pkg.isMultipart,
-    hasPreview: pkg.previewData !== null,
+    hasPreview: previewIds.has(pkg.id),
    creator: pkg.creator,
    tags: pkg.tags,
    indexedAt: pkg.indexedAt.toISOString(),
@@ -109,31 +135,31 @@ export async function listDisplayItems(options: {
  const sortCol = sortBy === "fileName" ? `"fileName"` : sortBy === "fileSize" ? `"fileSize"` : `"indexedAt"`;
  const sortDir = order === "asc" ? "ASC" : "DESC";

-  // Step 1: Count display items
+  // NOTE: The STL list is intentionally FLAT — every package is its own display
+  // row regardless of packageGroupId. Grouping is no longer surfaced in this
+  // view (the creator column/filter organizes the list instead). PackageGroup
+  // rows and the manual grouping actions still exist in the DB/UI; they just
+  // don't drive this list's layout anymore.
+
+  // Step 1: Count display items (one per package)
  const countResult = await prisma.$queryRawUnsafe<[{ count: bigint }]>(
-    `SELECT COUNT(*) AS count FROM (
-      SELECT DISTINCT COALESCE(p."packageGroupId", p."id") AS display_id
-      FROM packages p
-      ${whereClause}
-    ) AS display_items`,
+    `SELECT COUNT(*) AS count FROM packages p ${whereClause}`,
    ...params
  );
  const total = Number(countResult[0].count);

-  // Step 2: Get display item IDs for this page
+  // Step 2: Get package IDs for this page
  const limitParam = paramIdx++;
  const offsetParam = paramIdx++;
  const displayRows = await prisma.$queryRawUnsafe<
    { display_id: string; display_type: string }[]
  >(
    `SELECT
-      COALESCE(p."packageGroupId", p."id") AS display_id,
-      CASE WHEN p."packageGroupId" IS NOT NULL THEN 'group' ELSE 'package' END AS display_type,
-      MAX(p.${sortCol}) AS sort_value
+      p."id" AS display_id,
+      'package' AS display_type,
+      p.${sortCol} AS sort_value
    FROM packages p
    ${whereClause}
-    GROUP BY COALESCE(p."packageGroupId", p."id"),
-             CASE WHEN p."packageGroupId" IS NOT NULL THEN 'group' ELSE 'package' END
    ORDER BY sort_value ${sortDir}
    LIMIT $${limitParam} OFFSET $${offsetParam}`,
    ...params, limit, (page - 1) * limit
@@ -149,7 +175,7 @@ export async function listDisplayItems(options: {
        select: {
          id: true, fileName: true, fileSize: true, contentHash: true,
          archiveType: true, fileCount: true, isMultipart: true,
-          indexedAt: true, creator: true, tags: true, previewData: true,
+          indexedAt: true, creator: true, tags: true,
          sourceChannel: { select: { id: true, title: true } },
        },
      })
@@ -159,13 +185,13 @@ export async function listDisplayItems(options: {
    ? await prisma.packageGroup.findMany({
        where: { id: { in: groupIds } },
        select: {
-          id: true, name: true, previewData: true,
+          id: true, name: true,
          sourceChannel: { select: { id: true, title: true } },
          packages: {
            select: {
              id: true, fileName: true, fileSize: true, contentHash: true,
              archiveType: true, fileCount: true, isMultipart: true,
-              indexedAt: true, creator: true, tags: true, previewData: true,
+              indexedAt: true, creator: true, tags: true,
              sourceChannel: { select: { id: true, title: true } },
            },
            orderBy: { indexedAt: "desc" },
@@ -174,6 +200,16 @@ export async function listDisplayItems(options: {
      })
    : [];

+  // Compute hasPreview flags without transferring image bytes (see fetchPreviewFlags)
+  const allPackageIds = [
+    ...standalonePackages.map((p) => p.id),
+    ...groups.flatMap((g) => g.packages.map((p) => p.id)),
+  ];
+  const [packagePreviewIds, groupPreviewIds] = await Promise.all([
+    fetchPreviewFlags("packages", allPackageIds),
+    fetchPreviewFlags("package_groups", groups.map((g) => g.id)),
+  ]);
+
  // Build DisplayItem array in the original sort order
  const packageMap = new Map(standalonePackages.map((p) => [p.id, p]));
  const groupMap = new Map(groups.map((g) => [g.id, g]));
@@ -191,7 +227,7 @@ export async function listDisplayItems(options: {
          archiveType: pkg.archiveType,
          fileCount: pkg.fileCount,
          isMultipart: pkg.isMultipart,
-          hasPreview: pkg.previewData !== null,
+          hasPreview: packagePreviewIds.has(pkg.id),
          creator: pkg.creator,
          tags: pkg.tags,
          indexedAt: pkg.indexedAt.toISOString(),
@@ -209,7 +245,7 @@ export async function listDisplayItems(options: {
        data: {
          id: grp.id,
          name: grp.name,
-          hasPreview: grp.previewData !== null,
+          hasPreview: groupPreviewIds.has(grp.id),
          totalFileSize: grp.packages.reduce((sum, p) => sum + p.fileSize, BigInt(0)).toString(),
          totalFileCount: grp.packages.reduce((sum, p) => sum + p.fileCount, 0),
          packageCount: grp.packages.length,
@@ -227,7 +263,7 @@ export async function listDisplayItems(options: {
            archiveType: pkg.archiveType,
            fileCount: pkg.fileCount,
            isMultipart: pkg.isMultipart,
-            hasPreview: pkg.previewData !== null,
+            hasPreview: packagePreviewIds.has(pkg.id),
            creator: pkg.creator,
            tags: pkg.tags,
            indexedAt: pkg.indexedAt.toISOString(),
@@ -340,6 +376,30 @@ export async function listPackageFiles(options: {
  };
 }

+async function fullTextSearchPackageIds(query: string, limit: number): Promise<string[]> {
+  // Convert user query to tsquery — handle multi-word by joining with &
+  const tsQuery = query
+    .trim()
+    .split(/\s+/)
+    .filter((w) => w.length >= 2)
+    .map((w) => w.replace(/[^a-zA-Z0-9]/g, ""))
+    .filter(Boolean)
+    .join(" & ");
+
+  if (!tsQuery) return [];
+
+  const results = await prisma.$queryRawUnsafe<{ id: string }[]>(
+    `SELECT id FROM packages
+     WHERE "searchVector" @@ to_tsquery('english', $1)
+     ORDER BY ts_rank("searchVector", to_tsquery('english', $1)) DESC
+     LIMIT $2`,
+    tsQuery,
+    limit
+  );
+
+  return results.map((r) => r.id);
+}
+
 export async function searchPackages(options: {
  query: string;
  page: number;
@@ -366,9 +426,21 @@ export async function searchPackages(options: {
    );
    const fileMatchedIds = fileMatches.map((f) => f.packageId);

+    // Try full-text search first (better ranking, handles word stemming)
+    let ftsPackageNameIds: string[] = [];
+    if (options.searchIn === "both" && q.length >= 3) {
+      try {
+        ftsPackageNameIds = await fullTextSearchPackageIds(q, 200);
+      } catch {
+        // FTS failed — fall back to ILIKE below
+      }
+    }
+
    const packageNameIds =
      options.searchIn === "both"
-        ? (
+        ? ftsPackageNameIds.length > 0
+          ? ftsPackageNameIds
+          : (
              await prisma.package.findMany({
                where: { fileName: { contains: q, mode: "insensitive" } },
                select: { id: true },
@@ -404,13 +476,17 @@ export async function searchPackages(options: {
          indexedAt: true,
          creator: true,
          tags: true,
-          previewData: true,
          sourceChannel: { select: { id: true, title: true } },
        },
      }),
      Promise.resolve(allIds.length),
    ]);

+    const previewIds = await fetchPreviewFlags(
+      "packages",
+      items.map((p) => p.id)
+    );
+
    const mapped: PackageListItem[] = items.map((pkg) => ({
      id: pkg.id,
      fileName: pkg.fileName,
@@ -419,7 +495,7 @@ export async function searchPackages(options: {
      archiveType: pkg.archiveType,
      fileCount: pkg.fileCount,
      isMultipart: pkg.isMultipart,
-      hasPreview: pkg.previewData !== null,
+      hasPreview: previewIds.has(pkg.id),
      creator: pkg.creator,
      tags: pkg.tags,
      indexedAt: pkg.indexedAt.toISOString(),
@@ -458,6 +534,15 @@ export async function getAllPackageTags(): Promise<string[]> {
  return result.map((r) => r.tag);
 }

+export async function getAllPackageCreators(): Promise<string[]> {
+  const result = await prisma.$queryRaw<{ creator: string }[]>`
+    SELECT DISTINCT creator FROM packages
+    WHERE creator IS NOT NULL AND creator <> ''
+    ORDER BY creator
+  `;
+  return result.map((r) => r.creator);
+}
+
 export async function getIngestionStatus(): Promise<IngestionAccountStatus[]> {
  const accounts = await prisma.telegramAccount.findMany({
    orderBy: { createdAt: "asc" },
@@ -514,6 +599,8 @@ export async function getIngestionStatus(): Promise<IngestionAccountStatus[]> {
            totalBytes: currentRun.totalBytes?.toString() ?? null,
            downloadPercent: currentRun.downloadPercent,
            lastActivityAt: currentRun.lastActivityAt?.toISOString() ?? null,
+            currentTopicId: currentRun.currentTopicId?.toString() ?? null,
+            currentAccountChannelMapId: currentRun.currentAccountChannelMapId,
          }
        : null,
    });
@@ -553,6 +640,7 @@ export async function listSkippedPackages(options: {
    sourceMessageId: s.sourceMessageId.toString(),
    isMultipart: s.isMultipart,
    partCount: s.partCount,
+    attemptCount: s.attemptCount,
    createdAt: s.createdAt.toISOString(),
  }));

@@ -571,6 +659,76 @@ export async function countSkippedPackages(): Promise<number> {
  return prisma.skippedPackage.count();
 }

+export async function listUngroupedPackages(options: {
+  page: number;
+  limit: number;
+}) {
+  const { page, limit } = options;
+  const skip = (page - 1) * limit;
+
+  const where = { packageGroupId: null, destMessageId: { not: null } };
+
+  const [items, total] = await Promise.all([
+    prisma.package.findMany({
+      where,
+      orderBy: { indexedAt: "desc" },
+      skip,
+      take: limit,
+      select: {
+        id: true,
+        fileName: true,
+        fileSize: true,
+        archiveType: true,
+        creator: true,
+        fileCount: true,
+        isMultipart: true,
+        partCount: true,
+        tags: true,
+        indexedAt: true,
+        sourceChannel: { select: { id: true, title: true } },
+      },
+    }),
+    prisma.package.count({ where }),
+  ]);
+
+  const previewIds = await fetchPreviewFlags(
+    "packages",
+    items.map((p) => p.id)
+  );
+
+  return {
+    items: items.map((p) => ({
+      id: p.id,
+      fileName: p.fileName,
+      fileSize: p.fileSize.toString(),
+      contentHash: "",
+      archiveType: p.archiveType,
+      creator: p.creator,
+      fileCount: p.fileCount,
+      isMultipart: p.isMultipart,
+      partCount: p.partCount,
+      tags: p.tags,
+      indexedAt: p.indexedAt.toISOString(),
+      hasPreview: previewIds.has(p.id),
+      sourceChannel: p.sourceChannel,
+      matchedFileCount: 0,
+      matchedByContent: false,
+    })),
+    pagination: {
+      total,
+      totalPages: Math.ceil(total / limit),
+      page,
+      limit,
+    },
+  };
+}
+
+export async function countUngroupedPackages(): Promise<number> {
+  return prisma.package.count({
+    where: { packageGroupId: null, destMessageId: { not: null } },
+  });
+}
+
 export async function getPackageGroup(groupId: string) {
  return prisma.packageGroup.findUnique({
    where: { id: groupId },
@@ -630,6 +788,53 @@ export async function createManualGroup(name: string, packageIds: string[]) {
    data: { packageGroupId: group.id },
  });

+  // Learn a grouping rule from the manual override
+  try {
+    const linkedPkgs = await prisma.package.findMany({
+      where: { id: { in: packageIds } },
+      select: { fileName: true, creator: true },
+    });
+
+    // Extract the common filename pattern
+    const fileNames = linkedPkgs.map((p) => p.fileName);
+    let pattern = "";
+    if (fileNames.length > 1) {
+      // Find longest common prefix
+      let prefix = fileNames[0];
+      for (let i = 1; i < fileNames.length; i++) {
+        while (!fileNames[i].startsWith(prefix)) {
+          prefix = prefix.slice(0, -1);
+          if (!prefix) break;
+        }
+      }
+      const trimmed = prefix.replace(/[\s\-_.(]+$/, "");
+      if (trimmed.length >= 4) {
+        pattern = trimmed;
+      }
+    }
+
+    // Fall back to shared creator
+    if (!pattern) {
+      const creators = [...new Set(linkedPkgs.map((p) => p.creator).filter(Boolean))];
+      if (creators.length === 1 && creators[0]) {
+        pattern = creators[0];
+      }
+    }
+
+    if (pattern) {
+      await prisma.groupingRule.create({
+        data: {
+          sourceChannelId: firstPkg.sourceChannelId,
+          pattern,
+          signalType: "MANUAL",
+          createdByGroupId: group.id,
+        },
+      });
+    }
+  } catch {
+    // Best-effort — don't fail the group creation if rule learning fails
+  }
+
  // Clean up empty groups left behind
  await prisma.packageGroup.deleteMany({
    where: { packages: { none: {} }, id: { not: group.id } },
@@ -670,3 +875,13 @@ export async function dissolveGroup(groupId: string) {
  });
  await prisma.packageGroup.delete({ where: { id: groupId } });
 }
+
+export async function mergeGroups(targetGroupId: string, sourceGroupId: string) {
+  // Move all packages from source group to target group
+  await prisma.package.updateMany({
+    where: { packageGroupId: sourceGroupId },
+    data: { packageGroupId: targetGroupId },
+  });
+  // Delete the now-empty source group
+  await prisma.packageGroup.delete({ where: { id: sourceGroupId } });
+}
@@ -55,6 +55,8 @@ export interface SkippedPackageItem {
  sourceMessageId: string;
  isMultipart: boolean;
  partCount: number;
+  /** How many times the worker has tried this source message across cycles. */
+  attemptCount: number;
  createdAt: string;
 }

@@ -121,5 +123,7 @@ export interface IngestionAccountStatus {
    totalBytes: string | null; // BigInt serialized as string
    downloadPercent: number | null;
    lastActivityAt: string | null;
+    currentTopicId: string | null; // BigInt serialized as string
+    currentAccountChannelMapId: string | null;
  } | null;
 }
@@ -6,15 +6,17 @@
  "scripts": {
    "build": "tsc",
    "start": "node dist/index.js",
-    "dev": "tsx watch src/index.ts"
+    "dev": "tsx watch src/index.ts",
+    "test": "vitest run",
+    "test:watch": "vitest"
  },
  "dependencies": {
    "@prisma/adapter-pg": "^7.4.0",
    "@prisma/client": "^7.4.0",
    "pg": "^8.18.0",
    "pino": "^9.6.0",
-    "prebuilt-tdlib": "^0.1008050.0",
-    "tdl": "^8.0.0",
+    "prebuilt-tdlib": "^0.1008064.0",
+    "tdl": "^8.1.0",
    "yauzl": "^3.2.0"
  },
  "devDependencies": {
@@ -23,6 +25,7 @@
    "@types/yauzl": "^2.10.3",
    "prisma": "^7.4.0",
    "tsx": "^4.21.0",
-    "typescript": "^5"
+    "typescript": "^5",
+    "vitest": "^3.2.4"
  }
 }
@@ -0,0 +1,69 @@
+import { describe, it, expect } from "vitest";
+import { parseZipCentralDirectoryFromTail } from "./central-directory.js";
+import { crc32 } from "zlib"; // Node 20+ exposes zlib.crc32
+
+// Build a minimal STORE (no compression) ZIP in-memory with the given files.
+function buildStoreZip(files: { name: string; data: Buffer }[]): Buffer {
+  const chunks: Buffer[] = [];
+  const central: Buffer[] = [];
+  let offset = 0;
+  for (const f of files) {
+    const crc = crc32(f.data) >>> 0;
+    const nameBuf = Buffer.from(f.name, "utf8");
+    const local = Buffer.alloc(30);
+    local.writeUInt32LE(0x04034b50, 0);
+    local.writeUInt16LE(20, 4);              // version needed
+    local.writeUInt16LE(0, 6);               // flags
+    local.writeUInt16LE(0, 8);               // method = store
+    local.writeUInt32LE(crc, 14);
+    local.writeUInt32LE(f.data.length, 18);  // compressed
+    local.writeUInt32LE(f.data.length, 22);  // uncompressed
+    local.writeUInt16LE(nameBuf.length, 26);
+    local.writeUInt16LE(0, 28);              // extra len
+    const localHeader = Buffer.concat([local, nameBuf, f.data]);
+    chunks.push(localHeader);
+
+    const cd = Buffer.alloc(46);
+    cd.writeUInt32LE(0x02014b50, 0);
+    cd.writeUInt16LE(20, 4); cd.writeUInt16LE(20, 6);
+    cd.writeUInt16LE(0, 8); cd.writeUInt16LE(0, 10);
+    cd.writeUInt32LE(crc, 16);
+    cd.writeUInt32LE(f.data.length, 20);
+    cd.writeUInt32LE(f.data.length, 24);
+    cd.writeUInt16LE(nameBuf.length, 28);
+    cd.writeUInt32LE(offset, 42);            // local header offset
+    central.push(Buffer.concat([cd, nameBuf]));
+    offset += localHeader.length;
+  }
+  const cdBuf = Buffer.concat(central);
+  const cdOffset = offset;
+  const eocd = Buffer.alloc(22);
+  eocd.writeUInt32LE(0x06054b50, 0);
+  eocd.writeUInt16LE(files.length, 8);
+  eocd.writeUInt16LE(files.length, 10);
+  eocd.writeUInt32LE(cdBuf.length, 12);
+  eocd.writeUInt32LE(cdOffset, 16);
+  return Buffer.concat([...chunks, cdBuf, eocd]);
+}
+
+describe("parseZipCentralDirectoryFromTail", () => {
+  it("lists entries with correct names, sizes, and crc32", () => {
+    const zip = buildStoreZip([
+      { name: "models/dragon.stl", data: Buffer.from("DRAGON") },
+      { name: "readme.txt", data: Buffer.from("hello world") },
+    ]);
+    const entries = parseZipCentralDirectoryFromTail(zip, 0);
+    expect(entries.map((e) => e.fileName).sort()).toEqual(["dragon.stl", "readme.txt"]);
+    const dragon = entries.find((e) => e.fileName === "dragon.stl")!;
+    expect(dragon.path).toBe("models/dragon.stl");
+    expect(dragon.uncompressedSize).toBe(6n);
+    expect(dragon.crc32).toMatch(/^[0-9a-f]{8}$/);
+  });
+
+  it("throws when the central directory begins before the tail window", () => {
+    const zip = buildStoreZip([{ name: "a.txt", data: Buffer.alloc(100) }]);
+    // Provide only the last 30 bytes but claim they start at offset (len-30):
+    const tail = zip.subarray(zip.length - 30);
+    expect(() => parseZipCentralDirectoryFromTail(tail, zip.length - 30)).toThrow(RangeError);
+  });
+});
@@ -0,0 +1,90 @@
+import path from "path";
+import type { FileEntry } from "./zip-reader.js";
+
+export const MIN_ZIP_TAIL_BYTES = 65_557;
+
+const EOCD_SIG = 0x06054b50;
+const CD_SIG = 0x02014b50;
+
+function extOf(name: string): string | null {
+  const e = path.extname(name).replace(/^\./, "").toLowerCase();
+  return e === "" ? null : e;
+}
+
+/** Parse a ZIP central directory from the tail of an archive. */
+export function parseZipCentralDirectoryFromTail(tail: Buffer, tailStart: number): FileEntry[] {
+  // 1. Find EOCD by scanning backward for its signature.
+  let eocd = -1;
+  for (let i = tail.length - 22; i >= 0; i--) {
+    if (tail.readUInt32LE(i) === EOCD_SIG) { eocd = i; break; }
+  }
+  if (eocd < 0) throw new RangeError("EOCD not found in tail");
+
+  let cdSize = tail.readUInt32LE(eocd + 12);
+  let cdOffset = tail.readUInt32LE(eocd + 16);
+
+  // ZIP64: sizes/offsets of 0xFFFFFFFF mean "see ZIP64 EOCD".
+  if (cdOffset === 0xffffffff || cdSize === 0xffffffff) {
+    const locSig = 0x07064b50;
+    let loc = -1;
+    for (let i = eocd - 20; i >= 0; i--) {
+      if (tail.readUInt32LE(i) === locSig) { loc = i; break; }
+    }
+    if (loc < 0) throw new RangeError("ZIP64 EOCD locator not in tail");
+    const z64Abs = Number(tail.readBigUInt64LE(loc + 8)); // absolute offset of ZIP64 EOCD
+    const z64 = z64Abs - tailStart;
+    if (z64 < 0) throw new RangeError("ZIP64 EOCD before tail window");
+    cdSize = Number(tail.readBigUInt64LE(z64 + 40));
+    cdOffset = Number(tail.readBigUInt64LE(z64 + 48));
+  }
+
+  // 2. Map the absolute central-directory offset into the tail buffer.
+  const cdLocal = cdOffset - tailStart;
+  if (cdLocal < 0 || cdLocal + cdSize > tail.length) {
+    throw new RangeError("Central directory begins before tail window");
+  }
+
+  // 3. Walk central-directory headers.
+  const entries: FileEntry[] = [];
+  let p = cdLocal;
+  const end = cdLocal + cdSize;
+  while (p + 46 <= end && tail.readUInt32LE(p) === CD_SIG) {
+    let crc = tail.readUInt32LE(p + 16) >>> 0;
+    let comp = BigInt(tail.readUInt32LE(p + 20));
+    let uncomp = BigInt(tail.readUInt32LE(p + 24));
+    const nameLen = tail.readUInt16LE(p + 28);
+    const extraLen = tail.readUInt16LE(p + 30);
+    const commentLen = tail.readUInt16LE(p + 32);
+    const name = tail.toString("utf8", p + 46, p + 46 + nameLen);
+
+    // ZIP64 extra field overrides 0xFFFFFFFF sizes.
+    if (comp === 0xffffffffn || uncomp === 0xffffffffn) {
+      let ep = p + 46 + nameLen;
+      const extraEnd = ep + extraLen;
+      while (ep + 4 <= extraEnd) {
+        const id = tail.readUInt16LE(ep);
+        const sz = tail.readUInt16LE(ep + 2);
+        if (id === 0x0001) {
+          let fp = ep + 4;
+          if (uncomp === 0xffffffffn) { uncomp = tail.readBigUInt64LE(fp); fp += 8; }
+          if (comp === 0xffffffffn) { comp = tail.readBigUInt64LE(fp); fp += 8; }
+        }
+        ep += 4 + sz;
+      }
+    }
+
+    const isDir = name.endsWith("/");
+    if (!isDir) {
+      entries.push({
+        path: name,
+        fileName: path.basename(name),
+        extension: extOf(name),
+        compressedSize: comp,
+        uncompressedSize: uncomp,
+        crc32: crc !== 0 ? crc.toString(16).padStart(8, "0") : null,
+      });
+    }
+    p += 46 + nameLen + extraLen + commentLen;
+  }
+  return entries;
+}
@@ -0,0 +1,35 @@
+import { describe, it, expect } from "vitest";
+import { crcFingerprint, fingerprintsMatch } from "./fingerprint.js";
+import type { FileEntry } from "./zip-reader.js";
+
+const fe = (crc: string | null): FileEntry => ({
+  path: "a", fileName: "a", extension: null,
+  compressedSize: 0n, uncompressedSize: 0n, crc32: crc,
+});
+
+describe("crcFingerprint", () => {
+  it("sorts crcs and marks complete", () => {
+    expect(crcFingerprint([fe("00ff"), fe("00aa")])).toEqual({ crcs: ["00aa", "00ff"], complete: true });
+  });
+  it("is incomplete when any crc is null", () => {
+    expect(crcFingerprint([fe("00aa"), fe(null)]).complete).toBe(false);
+  });
+  it("is incomplete when empty", () => {
+    expect(crcFingerprint([]).complete).toBe(false);
+  });
+});
+
+describe("fingerprintsMatch", () => {
+  it("matches identical crc multisets regardless of order", () => {
+    expect(fingerprintsMatch([fe("01"), fe("02")], [fe("02"), fe("01")])).toBe(true);
+  });
+  it("rejects different counts", () => {
+    expect(fingerprintsMatch([fe("01")], [fe("01"), fe("02")])).toBe(false);
+  });
+  it("rejects disjoint sets", () => {
+    expect(fingerprintsMatch([fe("01")], [fe("09")])).toBe(false);
+  });
+  it("rejects when either side is incomplete", () => {
+    expect(fingerprintsMatch([fe("01"), fe(null)], [fe("01"), fe("02")])).toBe(false);
+  });
+});
@@ -0,0 +1,21 @@
+import type { FileEntry } from "./zip-reader.js";
+
+export function crcFingerprint(entries: FileEntry[]): { crcs: string[]; complete: boolean } {
+  if (entries.length === 0) return { crcs: [], complete: false };
+  const crcs: string[] = [];
+  let complete = true;
+  for (const e of entries) {
+    if (e.crc32 == null) { complete = false; continue; }
+    crcs.push(e.crc32.toLowerCase());
+  }
+  crcs.sort();
+  return { crcs, complete };
+}
+
+export function fingerprintsMatch(a: FileEntry[], b: FileEntry[]): boolean {
+  const fa = crcFingerprint(a);
+  const fb = crcFingerprint(b);
+  if (!fa.complete || !fb.complete) return false;
+  if (fa.crcs.length !== fb.crcs.length) return false;
+  return fa.crcs.every((c, i) => c === fb.crcs[i]);
+}
@@ -0,0 +1,128 @@
+import { execFile } from "child_process";
+import { promisify } from "util";
+import { childLogger } from "../util/logger.js";
+
+const execFileAsync = promisify(execFile);
+const log = childLogger("integrity");
+
+export type IntegrityFailureKind = "encrypted" | "corrupt" | "inconclusive";
+
+export type IntegrityResult =
+  | { ok: true }
+  | { ok: false; reason: string; kind: IntegrityFailureKind };
+
+/**
+ * Test that the archive can be read end-to-end without errors, BEFORE we
+ * spend bandwidth uploading it to the destination channel.
+ *
+ * Failures are classified so the caller can react appropriately:
+ *   - "encrypted"    — password-protected; users can't extract it. Actionable.
+ *   - "corrupt"      — genuine CRC / structural error (truncated download, bad
+ *                      central directory, CRC mismatch). Actionable.
+ *   - "inconclusive" — the test tool itself was killed (OOM) or timed out,
+ *                      typically on very large 7z archives in a memory-limited
+ *                      container (exit 137 / SIGKILL). This is a TOOL
+ *                      LIMITATION, not corruption — callers should NOT raise a
+ *                      user-facing alarm for it.
+ *
+ * Returns { ok: true } if the archive is intact, otherwise
+ * { ok: false, reason, kind }.
+ *
+ * For multipart archives, pass the first part. unzip / unrar / 7z all
+ * auto-discover sibling parts.
+ *
+ * archiveType "DOCUMENT" is a pass-through — there's no container to test.
+ */
+export async function testArchiveIntegrity(
+  archiveType: "ZIP" | "RAR" | "SEVEN_Z" | "DOCUMENT",
+  firstPartPath: string
+): Promise<IntegrityResult> {
+  if (archiveType === "DOCUMENT") {
+    return { ok: true };
+  }
+
+  try {
+    if (archiveType === "ZIP") {
+      // -t = test, -qq = very quiet (errors only)
+      const { stderr } = await execFileAsync("unzip", ["-tqq", firstPartPath], {
+        timeout: 300_000, // 5 min for very large archives
+        maxBuffer: 10 * 1024 * 1024,
+      });
+      if (stderr && stderr.trim()) {
+        return { ok: false, kind: "corrupt", reason: `unzip -t reported: ${stderr.slice(0, 500)}` };
+      }
+      return { ok: true };
+    }
+
+    if (archiveType === "RAR") {
+      const { stdout, stderr } = await execFileAsync("unrar", ["t", firstPartPath], {
+        timeout: 300_000,
+        maxBuffer: 10 * 1024 * 1024,
+      });
+      // unrar uses non-zero exit code on errors, which becomes a throw.
+      // If it succeeds, "All OK" is in stdout.
+      const combined = `${stdout}\n${stderr}`;
+      if (/All OK/i.test(combined)) {
+        return { ok: true };
+      }
+      return { ok: false, kind: "corrupt", reason: `unrar t did not report "All OK": ${combined.slice(-500)}` };
+    }
+
+    if (archiveType === "SEVEN_Z") {
+      const { stdout, stderr } = await execFileAsync("7z", ["t", firstPartPath], {
+        timeout: 300_000,
+        maxBuffer: 10 * 1024 * 1024,
+      });
+      const combined = `${stdout}\n${stderr}`;
+      if (/Everything is Ok/i.test(combined)) {
+        return { ok: true };
+      }
+      return { ok: false, kind: "corrupt", reason: `7z t did not report "Everything is Ok": ${combined.slice(-500)}` };
+    }
+
+    return { ok: false, kind: "corrupt", reason: `Unknown archive type: ${archiveType}` };
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    // execFile throws on non-zero exit, on timeout, and when killed by a signal.
+    const e = err as {
+      stderr?: unknown;
+      stdout?: unknown;
+      signal?: string | null;
+      killed?: boolean;
+      code?: number | string | null;
+    };
+    const stderr = typeof e.stderr === "string" ? e.stderr : "";
+    const stdout = typeof e.stdout === "string" ? e.stdout : "";
+    const detail = stderr ? `: ${stderr.slice(0, 500)}` : "";
+    const haystack = `${msg}\n${stdout}\n${stderr}`;
+
+    // Encrypted archives — users can't extract them, so flag clearly.
+    if (/password|encrypted|wrong password|enter password/i.test(haystack)) {
+      return { ok: false, kind: "encrypted", reason: `Archive is encrypted (password protected): ${msg}${detail}` };
+    }
+
+    // Inconclusive — the test tool was killed or timed out rather than
+    // reporting corruption. Common on large 7z in memory-limited containers,
+    // where `7z t` gets OOM-killed (SIGKILL / exit 137) mid-decompression.
+    // That's a tool limitation, not a corrupt archive.
+    const killedBySignal = e.signal === "SIGKILL" || e.signal === "SIGTERM";
+    const killedExitCode = e.code === 137 || e.code === 143; // 128 + SIGKILL/SIGTERM
+    const timedOut = e.killed === true;
+    const maxBufferExceeded = e.code === "ERR_CHILD_PROCESS_STDIO_MAXBUFFER";
+    if (killedBySignal || killedExitCode || timedOut || maxBufferExceeded) {
+      log.debug(
+        { err, archiveType, firstPartPath, signal: e.signal, code: e.code, killed: e.killed },
+        "Archive integrity test inconclusive (tool killed or timed out)"
+      );
+      return {
+        ok: false,
+        kind: "inconclusive",
+        reason: `Integrity test could not complete (tool killed or timed out — likely OOM on a large archive): ${msg}`,
+      };
+    }
+
+    // Genuine failure: a real non-zero exit with error output.
+    log.debug({ err, archiveType, firstPartPath }, "Archive integrity test failed");
+    return { ok: false, kind: "corrupt", reason: `Integrity test failed: ${msg}${detail}` };
+  }
+}
@@ -11,6 +11,11 @@ export interface TelegramMessage {
  fileSize: bigint;
  date: Date;
  mediaAlbumId?: string;
+  replyToMessageId?: bigint;
+  caption?: string;
+  /** TDLib's `remote.unique_id` for the file — stable across reposts of
+   *  the exact same content. Empty string if the message didn't expose it. */
+  remoteUniqueId?: string;
 }

 export interface ArchiveSet {
@@ -0,0 +1,27 @@
+import { describe, it, expect, vi } from "vitest";
+
+// logLevel is required here too (not just maxZipSizeMB/tempDir) because this
+// mock replaces the config module for the whole test-file graph, including
+// util/logger.ts's module-level `pino({ level: config.logLevel })` call —
+// pino throws at import time if level is undefined.
+vi.mock("../../util/config.js", () => ({ config: { maxZipSizeMB: 1, tempDir: "/tmp", logLevel: "info" } }));
+const created: unknown[] = [];
+vi.mock("../../db/client.js", () => ({
+  db: { systemNotification: { create: async (a: unknown) => { created.push(a); } } },
+}));
+
+import { fullDownloadListing } from "./fallback.js";
+
+describe("fullDownloadListing", () => {
+  it("refuses to download over the size cap and records a notification", async () => {
+    const res = await fullDownloadListing({
+      client: {} as never,
+      parts: [{ fileId: "1", fileSize: 2n * 1024n * 1024n * 1024n, fileName: "big.rar" }],
+      archiveType: "RAR",
+      totalSize: 2n * 1024n * 1024n * 1024n,
+      fileName: "big.rar",
+    });
+    expect(res).toBeNull();
+    expect(created).toHaveLength(1);
+  });
+});
@@ -0,0 +1,55 @@
+import { mkdtemp, rm } from "fs/promises";
+import path from "path";
+import type { Client } from "tdl";
+import { config } from "../../util/config.js";
+import { db } from "../../db/client.js";
+import { childLogger } from "../../util/logger.js";
+import { downloadFile } from "../../tdlib/download.js";
+import { read7zContents } from "../sevenz-reader.js";
+import { readRarContents } from "../rar-reader.js";
+import type { FileEntry } from "../zip-reader.js";
+import type { RangedPart } from "./sevenz-ranged.js";
+
+const log = childLogger("ranged-fallback");
+
+export async function fullDownloadListing(args: {
+  client: Client;
+  parts: RangedPart[];
+  archiveType: string;
+  totalSize: bigint;
+  fileName: string;
+}): Promise<FileEntry[] | null> {
+  const capBytes = BigInt(config.maxZipSizeMB) * 1024n * 1024n;
+  if (args.totalSize > capBytes) {
+    await db.systemNotification.create({
+      data: {
+        type: "INTEGRITY_AUDIT",
+        severity: "WARNING",
+        title: `Listing skipped (over size cap): ${args.fileName}`,
+        message: `Ranged listing failed and the archive (${args.totalSize} bytes) exceeds WORKER_MAX_ZIP_SIZE_MB; not downloaded. Inner files left unindexed.`,
+        context: { fileName: args.fileName, archiveType: args.archiveType },
+      },
+    });
+    log.warn({ fileName: args.fileName }, "fallback skipped — over size cap");
+    return null;
+  }
+  const dir = await mkdtemp(path.join(config.tempDir, "fallback-"));
+  const paths: string[] = [];
+  try {
+    for (const p of args.parts) {
+      const dest = path.join(dir, p.fileName);
+      await downloadFile(args.client, p.fileId, dest, p.fileSize, p.fileName, () => {});
+      paths.push(dest);
+    }
+    const entries =
+      args.archiveType === "SEVEN_Z" ? await read7zContents(paths[0])
+      : args.archiveType === "RAR" ? await readRarContents(paths[0])
+      : [];
+    return entries.length > 0 ? entries : null;
+  } catch (err) {
+    log.warn({ err, fileName: args.fileName }, "full-download fallback failed");
+    return null;
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+}
@@ -0,0 +1,14 @@
+import type { Client } from "tdl";
+import { downloadFileRange } from "../../tdlib/range-download.js";
+
+export type RangeReader = (
+  fileId: string,
+  offset: number,
+  length: number,
+  partSize: bigint,
+) => Promise<Buffer>;
+
+export function tdlibRangeReader(client: Client): RangeReader {
+  return (fileId, offset, length, partSize) =>
+    downloadFileRange(client, fileId, offset, length, partSize);
+}
@@ -0,0 +1,150 @@
+import { describe, it, expect } from "vitest";
+import { readVint, detectRarSignature, parseRar5BlockExtent, parseRar4BlockExtent, walkRarVolume, readRarListingRanged } from "./rar-ranged.js";
+import type { RangeReader } from "./range-reader.js";
+
+describe("readVint", () => {
+  it("reads single-byte and multi-byte values (base-128 LE)", () => {
+    expect(readVint(Buffer.from([0x08]), 0)).toEqual({ value: 8, bytes: 1 });
+    // 0x80,0x01 => 0 | (1<<7) = 128
+    expect(readVint(Buffer.from([0x80, 0x01]), 0)).toEqual({ value: 128, bytes: 2 });
+  });
+});
+
+describe("detectRarSignature", () => {
+  it("detects RAR5 and RAR4", () => {
+    expect(detectRarSignature(Buffer.from([0x52,0x61,0x72,0x21,0x1a,0x07,0x01,0x00]))).toEqual({ version: 5, sigLen: 8 });
+    expect(detectRarSignature(Buffer.from([0x52,0x61,0x72,0x21,0x1a,0x07,0x00]))).toEqual({ version: 4, sigLen: 7 });
+    expect(detectRarSignature(Buffer.alloc(8))).toBeNull();
+  });
+});
+
+describe("parseRar5BlockExtent", () => {
+  it("computes header+data extent and flags end-of-archive", () => {
+    // CRC32(4) | HeaderSize vint=5 | Type vint=2 (file) | Flags vint=2 (data present) | DataSize vint=100 | (pad to headerSize)
+    const b = Buffer.concat([
+      Buffer.from([0,0,0,0]),      // CRC
+      Buffer.from([0x05]),          // HeaderSize = 5 (bytes after this vint)
+      Buffer.from([0x02]),          // Type = 2 (file)
+      Buffer.from([0x02]),          // Flags = 0x02 -> data present
+      Buffer.from([0x64]),          // DataSize = 100
+      Buffer.from([0x00, 0x00]),    // padding to fill HeaderSize(5): Type+Flags+DataSize=3, +2 pad =5
+    ]);
+    const ext = parseRar5BlockExtent(b, 0);
+    // headerBytes = 4 (CRC) + 1 (HeaderSize vint) + 5 (HeaderSize) = 10
+    expect(ext.headerBytes).toBe(10);
+    expect(ext.dataSize).toBe(100);
+    expect(ext.isEnd).toBe(false);
+
+    const endBlk = Buffer.from([0,0,0,0, 0x02, 0x05, 0x00]); // HeaderSize=2, Type=5(end), Flags=0
+    const e2 = parseRar5BlockExtent(endBlk, 0);
+    expect(e2.isEnd).toBe(true);
+  });
+});
+
+describe("parseRar4BlockExtent", () => {
+  it("computes extent with ADD_SIZE when flag 0x8000 is set", () => {
+    // CRC(2) TYPE(1)=0x74 FLAGS(2)=0x8000 HEAD_SIZE(2)=11 ADD_SIZE(4)=200
+    const b = Buffer.alloc(11);
+    b.writeUInt8(0x74, 2);
+    b.writeUInt16LE(0x8000, 3);
+    b.writeUInt16LE(11, 5);
+    b.writeUInt32LE(200, 7);
+    const ext = parseRar4BlockExtent(b, 0);
+    expect(ext.headerBytes).toBe(11);
+    expect(ext.dataSize).toBe(200);
+    expect(ext.isEnd).toBe(false);
+  });
+});
+
+// Build a synthetic RAR5 volume: signature + main header + 2 file blocks (each
+// with data) + end block. We only need extents to be walkable.
+function buildRar5Volume(): Buffer {
+  const sig = Buffer.from([0x52,0x61,0x72,0x21,0x1a,0x07,0x01,0x00]);
+  const block = (type: number, flags: number, dataSize: number, pad = 0) => {
+    const body = [Buffer.from([type]), Buffer.from([flags])];
+    if (flags & 0x0002) body.push(Buffer.from([dataSize])); // DataSize (<=127 for test)
+    if (pad) body.push(Buffer.alloc(pad));
+    const bodyBuf = Buffer.concat(body);
+    const hs = Buffer.from([bodyBuf.length]); // HeaderSize vint (<=127)
+    const header = Buffer.concat([Buffer.alloc(4), hs, bodyBuf]); // CRC(4)+HeaderSize+body
+    const data = Buffer.alloc(flags & 0x0002 ? dataSize : 0, 0xEE);
+    return Buffer.concat([header, data]);
+  };
+  const main = block(1, 0, 0);       // main archive header, no data
+  const f1 = block(2, 0x02, 20);     // file header + 20 bytes data
+  const f2 = block(2, 0x02, 30);     // file header + 30 bytes data
+  const end = block(5, 0, 0);        // end of archive
+  return Buffer.concat([sig, main, f1, f2, end]);
+}
+
+describe("walkRarVolume", () => {
+  it("harvests every block header and stops at end-of-archive", async () => {
+    const vol = buildRar5Volume();
+    const read: RangeReader = async (_id, offset, length) => vol.subarray(offset, offset + length);
+    const regions = await walkRarVolume(read, { fileId: "1", fileSize: BigInt(vol.length), fileName: "a.rar" }, 5, 8);
+    expect(regions).not.toBeNull();
+    // main + 2 files + end = 4 header regions
+    expect(regions!).toHaveLength(4);
+    // First region starts right after the 8-byte signature
+    expect(regions![0].offset).toBe(8);
+  });
+
+  it("returns null when a block claims an absurd header size (corrupt/desynced)", async () => {
+    // RAR5 block with HeaderSize vint encoding a value > 8MB.
+    // Encode 9_000_000 as RAR vint: bytes little-endian 7-bit groups with continuation bit.
+    function encodeVint(n: number): number[] {
+      const out: number[] = [];
+      while (n >= 0x80) {
+        out.push((n & 0x7f) | 0x80);
+        n = Math.floor(n / 128);
+      }
+      out.push(n);
+      return out;
+    }
+    const sig = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x01, 0x00]); // RAR5 signature
+    const hsVint = encodeVint(9_000_000);
+    // Block = CRC(4) + HeaderSize vint(9MB) + Type(1 byte) + Flags(1 byte)
+    const block = Buffer.concat([Buffer.alloc(4), Buffer.from(hsVint), Buffer.from([0x02, 0x00])]);
+    const vol = Buffer.concat([sig, block]);
+    const size = 20 * 1024 * 1024;
+    const read = async (_id: string, offset: number, length: number) => {
+      if (offset >= vol.length) return Buffer.alloc(0);
+      return vol.subarray(offset, Math.min(offset + length, vol.length));
+    };
+    const regions = await walkRarVolume(read, { fileId: "1", fileSize: BigInt(size), fileName: "c.rar" }, 5, 8);
+    expect(regions).toBeNull();
+  });
+});
+
+describe("readRarListingRanged (single part)", () => {
+  it("returns null cleanly when the reconstructed file isn't a real RAR", async () => {
+    const vol = buildRar5Volume();
+    const read: RangeReader = async (_id, offset, length) => vol.subarray(offset, offset + length);
+    const res = await readRarListingRanged([{ fileId: "1", fileSize: BigInt(vol.length), fileName: "a.rar" }], read);
+    expect(res === null || Array.isArray(res)).toBe(true); // real unrar parse covered live
+  });
+});
+
+describe("readRarListingRanged (multipart)", () => {
+  it("walks each volume from its own signature and reconstructs all parts", async () => {
+    const vol = buildRar5Volume(); // reuse from Task 6 test
+    // Two volumes with identical structure; each RangeReader read is scoped by fileId.
+    const byId: Record<string, Buffer> = { p1: vol, p2: vol };
+    const reads: Record<string, number> = { p1: 0, p2: 0 };
+    const read: RangeReader = async (fileId, offset, length) => {
+      reads[fileId]++;
+      return byId[fileId].subarray(offset, offset + length);
+    };
+    const res = await readRarListingRanged(
+      [
+        { fileId: "p1", fileSize: BigInt(vol.length), fileName: "x.part1.rar" },
+        { fileId: "p2", fileSize: BigInt(vol.length), fileName: "x.part2.rar" },
+      ],
+      read,
+    );
+    // Both volumes were walked (each read at least its signature + blocks).
+    expect(reads.p1).toBeGreaterThan(0);
+    expect(reads.p2).toBeGreaterThan(0);
+    expect(res === null || Array.isArray(res)).toBe(true);
+  });
+});
@@ -0,0 +1,110 @@
+const RAR4_SIG = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x00]);
+const RAR5_SIG = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x01, 0x00]);
+
+export function readVint(buf: Buffer, pos: number): { value: number; bytes: number } {
+  let value = 0, shift = 0, bytes = 0;
+  while (pos + bytes < buf.length) {
+    const b = buf[pos + bytes];
+    value += (b & 0x7f) * Math.pow(2, shift); // Math.pow keeps >32-bit sizes exact up to 2^53
+    bytes++;
+    if ((b & 0x80) === 0) return { value, bytes };
+    shift += 7;
+    if (shift > 63) break;
+  }
+  throw new RangeError("incomplete RAR vint");
+}
+
+export function detectRarSignature(buf: Buffer): { version: 4 | 5; sigLen: number } | null {
+  if (buf.length >= 8 && buf.subarray(0, 8).equals(RAR5_SIG)) return { version: 5, sigLen: 8 };
+  if (buf.length >= 7 && buf.subarray(0, 7).equals(RAR4_SIG)) return { version: 4, sigLen: 7 };
+  return null;
+}
+
+export interface BlockExtent { headerBytes: number; dataSize: number; isEnd: boolean }
+
+// RAR5: CRC32(4) | HeaderSize(vint) | HeaderType(vint) | HeaderFlags(vint)
+//       [ExtraAreaSize(vint) if flags&0x0001] [DataSize(vint) if flags&0x0002] ...
+export function parseRar5BlockExtent(buf: Buffer, pos: number): BlockExtent {
+  let p = pos + 4; // skip CRC32
+  const hs = readVint(buf, p); p += hs.bytes;
+  const headerBytes = 4 + hs.bytes + hs.value; // CRC + HeaderSize-vint + HeaderSize
+  const type = readVint(buf, p); p += type.bytes;
+  const flags = readVint(buf, p); p += flags.bytes;
+  if (flags.value & 0x0001) { const ea = readVint(buf, p); p += ea.bytes; } // extra area size (skip)
+  let dataSize = 0;
+  if (flags.value & 0x0002) { const ds = readVint(buf, p); p += ds.bytes; dataSize = ds.value; }
+  return { headerBytes, dataSize, isEnd: type.value === 5 };
+}
+
+// RAR4: HEAD_CRC(2) | HEAD_TYPE(1) | HEAD_FLAGS(2) | HEAD_SIZE(2) [ADD_SIZE(4) if flags&0x8000]
+export function parseRar4BlockExtent(buf: Buffer, pos: number): BlockExtent {
+  const type = buf.readUInt8(pos + 2);
+  const flags = buf.readUInt16LE(pos + 3);
+  const headSize = buf.readUInt16LE(pos + 5);
+  const dataSize = (flags & 0x8000) ? buf.readUInt32LE(pos + 7) : 0;
+  return { headerBytes: headSize, dataSize, isEnd: type === 0x7b };
+}
+
+import type { FileEntry } from "../zip-reader.js";
+import type { RangeReader } from "./range-reader.js";
+import type { RangedPart } from "./sevenz-ranged.js";
+import { listFromSparse, type SparsePart } from "./sparse-list.js";
+import { readRarContents } from "../rar-reader.js";
+import { childLogger } from "../../util/logger.js";
+
+const rlog = childLogger("rar-ranged");
+const MAX_RAR_BLOCKS = 50000;
+const MAX_RAR_HEADER_BYTES = 8 * 1024 * 1024; // 8 MB — real RAR block headers are far smaller; guards against a corrupt/desynced HeaderSize
+const HEADER_CHUNK = 8192;
+
+export async function walkRarVolume(
+  read: RangeReader,
+  part: RangedPart,
+  version: 4 | 5,
+  sigLen: number,
+): Promise<{ offset: number; bytes: Buffer }[] | null> {
+  const size = Number(part.fileSize);
+  const regions: { offset: number; bytes: Buffer }[] = [];
+  let pos = sigLen;
+  let blocks = 0;
+  try {
+    while (pos < size) {
+      if (++blocks > MAX_RAR_BLOCKS) return null;
+      const chunkLen = Math.min(HEADER_CHUNK, size - pos);
+      let chunk = await read(part.fileId, pos, chunkLen, part.fileSize);
+      const ext = version === 5 ? parseRar5BlockExtent(chunk, 0) : parseRar4BlockExtent(chunk, 0);
+      if (ext.headerBytes > MAX_RAR_HEADER_BYTES) return null;
+      // Ensure we have the full header bytes to harvest (long filenames).
+      let headerBuf = chunk;
+      if (ext.headerBytes > chunk.length) {
+        headerBuf = await read(part.fileId, pos, Math.min(ext.headerBytes, size - pos), part.fileSize);
+      }
+      regions.push({ offset: pos, bytes: headerBuf.subarray(0, Math.min(ext.headerBytes, size - pos)) });
+      if (ext.isEnd) break;
+      const advance = ext.headerBytes + ext.dataSize;
+      if (advance <= 0) return null;
+      if (pos + advance > size) break; // data clamped at the volume boundary (multipart continuation)
+      pos += advance;
+    }
+    return regions;
+  } catch (err) {
+    rlog.warn({ err, fileId: part.fileId }, "RAR volume walk failed");
+    return null;
+  }
+}
+
+export async function readRarListingRanged(
+  parts: RangedPart[],
+  read: RangeReader,
+): Promise<FileEntry[] | null> {
+  const sparseParts: SparsePart[] = [];
+  for (const part of parts) {
+    const head = await read(part.fileId, 0, 16, part.fileSize);
+    const sig = detectRarSignature(head);
+    if (!sig) return null;
+    const regions = await walkRarVolume(read, part, sig.version, sig.sigLen);
+    if (!regions) return null;
+    sparseParts.push({ fileName: part.fileName, size: Number(part.fileSize), regions });
+  }
+  return listFromSparse(sparseParts, readRarContents);
+}
@@ -0,0 +1,133 @@
+import { describe, it, expect } from "vitest";
+import { parseSevenZSignatureHeader } from "./sevenz-ranged.js";
+
+const MAGIC = Buffer.from([0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c]);
+
+function buildSignatureHeader(nextOffset: bigint, nextSize: bigint): Buffer {
+  const buf = Buffer.alloc(32);
+  MAGIC.copy(buf, 0);
+  buf.writeUInt8(0, 6); buf.writeUInt8(4, 7);  // version 0.4
+  buf.writeUInt32LE(0, 8);                      // StartHeaderCRC (unused here)
+  buf.writeBigUInt64LE(nextOffset, 12);
+  buf.writeBigUInt64LE(nextSize, 20);
+  buf.writeUInt32LE(0, 28);                     // NextHeaderCRC (unused here)
+  return buf;
+}
+
+describe("parseSevenZSignatureHeader", () => {
+  it("reads NextHeaderOffset and NextHeaderSize", () => {
+    const buf = buildSignatureHeader(1_000_000n, 4096n);
+    expect(parseSevenZSignatureHeader(buf)).toEqual({ nextHeaderOffset: 1_000_000, nextHeaderSize: 4096 });
+  });
+
+  it("returns null on bad magic", () => {
+    expect(parseSevenZSignatureHeader(Buffer.alloc(32))).toBeNull();
+  });
+
+  it("returns null when shorter than 32 bytes", () => {
+    expect(parseSevenZSignatureHeader(MAGIC)).toBeNull();
+  });
+});
+
+import { readSevenZListingRanged } from "./sevenz-ranged.js";
+import type { RangeReader } from "./range-reader.js";
+
+describe("readSevenZListingRanged", () => {
+  it("reads the signature + end-header regions and reconstructs for 7z l", async () => {
+    const size = 5_000_000;
+    const endHeaderOffset = 4_900_000; // absolute
+    const nextHeaderOffset = endHeaderOffset - 32;
+    const sig = Buffer.alloc(32);
+    Buffer.from([0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c]).copy(sig, 0);
+    sig.writeBigUInt64LE(BigInt(nextHeaderOffset), 12);
+    sig.writeBigUInt64LE(100n, 20);
+
+    const reads: { offset: number; length: number }[] = [];
+    const read: RangeReader = async (_id, offset, length) => {
+      reads.push({ offset, length });
+      if (offset === 0) return sig.subarray(0, length);
+      return Buffer.alloc(length, 0xAB); // stand-in end-header bytes
+    };
+
+    // Inject a fake lister via the module boundary: readSevenZListingRanged
+    // calls listFromSparse(parts, read7zContents). We assert the ranged reads
+    // it issued; the sparse file + real 7z is covered by live verification.
+    const entries = await readSevenZListingRanged(
+      [{ fileId: "1", fileSize: BigInt(size), fileName: "a.7z" }],
+      read,
+    );
+    // entries may be null here because the stand-in bytes aren't a real 7z;
+    // the contract under test is the ranged-read offsets:
+    expect(reads[0]).toEqual({ offset: 0, length: 32 });
+    expect(reads[1]).toEqual({ offset: endHeaderOffset, length: 100 });
+    expect(entries === null || Array.isArray(entries)).toBe(true);
+  });
+});
+
+import { read7zNumber, locate7zEncodedHeaderPack } from "./sevenz-ranged.js";
+
+describe("read7zNumber", () => {
+  it("reads a single-byte number", () => {
+    expect(read7zNumber(Buffer.from([0x2a]), 0)).toEqual({ value: 42, next: 1 });
+  });
+  it("reads a two-byte number (first-byte length mask + LE trailing byte)", () => {
+    // 1000 = 0x03E8: low byte 0xE8, high nibble 0x03 -> first = 0x80|0x03 = 0x83, trailing 0xE8
+    expect(read7zNumber(Buffer.from([0x83, 0xe8]), 0)).toEqual({ value: 1000, next: 2 });
+    // 500 = 0x01F4 -> first 0x81, trailing 0xF4
+    expect(read7zNumber(Buffer.from([0x81, 0xf4]), 0)).toEqual({ value: 500, next: 2 });
+  });
+  it("throws when pos starts past the buffer end (short read)", () => {
+    expect(() => read7zNumber(Buffer.from([0x2a]), 5)).toThrow(RangeError);
+    expect(() => read7zNumber(Buffer.alloc(0), 0)).toThrow(RangeError);
+  });
+});
+
+describe("locate7zEncodedHeaderPack", () => {
+  it("parses PackPos and summed PackSize from an encoded header", () => {
+    // kEncodedHeader, kPackInfo, PackPos=1000([0x83,0xe8]), NumStreams=1([0x01]),
+    // kSize, PackSize=500([0x81,0xf4])
+    const enc = Buffer.from([0x17, 0x06, 0x83, 0xe8, 0x01, 0x09, 0x81, 0xf4]);
+    expect(locate7zEncodedHeaderPack(enc)).toEqual({ packPos: 1000, packSize: 500 });
+  });
+  it("returns null for a plain (kHeader 0x01) header", () => {
+    expect(locate7zEncodedHeaderPack(Buffer.from([0x01, 0x04]))).toBeNull();
+  });
+  it("sums multiple pack streams", () => {
+    // PackPos=0([0x00]), NumStreams=2([0x02]), kSize, sizes 10([0x0a]) + 20([0x14])
+    const enc = Buffer.from([0x17, 0x06, 0x00, 0x02, 0x09, 0x0a, 0x14]);
+    expect(locate7zEncodedHeaderPack(enc)).toEqual({ packPos: 0, packSize: 30 });
+  });
+});
+
+describe("readSevenZListingRanged (encoded header)", () => {
+  it("fetches the mid-file packed-header region as a third read", async () => {
+    const size = 5_000_000;
+    const nextHeaderOffset = 4_000_000;      // relative to end of 32-byte sig header
+    const endStart = 32 + nextHeaderOffset;  // absolute
+    const packPos = 1000;                    // relative to end of sig header
+    const packStart = 32 + packPos;          // absolute
+    const packSize = 500;
+    const sig = Buffer.alloc(32);
+    Buffer.from([0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c]).copy(sig, 0);
+    sig.writeBigUInt64LE(BigInt(nextHeaderOffset), 12);
+    sig.writeBigUInt64LE(8n, 20); // NextHeaderSize = 8 (the encoded-header descriptor below)
+    const encHeader = Buffer.from([0x17, 0x06, 0x83, 0xe8, 0x01, 0x09, 0x81, 0xf4]);
+
+    const reads: { offset: number; length: number }[] = [];
+    const read = async (_id: string, offset: number, length: number) => {
+      reads.push({ offset, length });
+      if (offset === 0) return sig.subarray(0, length);
+      if (offset === endStart) return encHeader.subarray(0, length);
+      return Buffer.alloc(length, 0xcd); // stand-in packed-header bytes
+    };
+
+    const entries = await readSevenZListingRanged(
+      [{ fileId: "1", fileSize: BigInt(size), fileName: "a.7z" }],
+      read,
+    );
+    expect(reads[0]).toEqual({ offset: 0, length: 32 });
+    expect(reads[1]).toEqual({ offset: endStart, length: 8 });
+    expect(reads[2]).toEqual({ offset: packStart, length: packSize });
+    expect(entries === null || Array.isArray(entries)).toBe(true);
+  });
+});
@@ -0,0 +1,112 @@
+import type { FileEntry } from "../zip-reader.js";
+import { read7zContents } from "../sevenz-reader.js";
+import { listFromSparse } from "./sparse-list.js";
+import type { RangeReader } from "./range-reader.js";
+import { childLogger } from "../../util/logger.js";
+
+const log = childLogger("sevenz-ranged");
+
+const SEVENZ_MAGIC = Buffer.from([0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c]);
+
+const K_HEADER = 0x01;
+const K_ENCODED_HEADER = 0x17;
+const K_PACK_INFO = 0x06;
+const K_SIZE = 0x09;
+
+/** Read a 7z variable-length number: first byte is a length mask, followed by
+ *  little-endian bytes. Math.pow keeps values exact above 2^31. */
+export function read7zNumber(buf: Buffer, pos: number): { value: number; next: number } {
+  if (pos >= buf.length) throw new RangeError("7z number reads past buffer end");
+  const first = buf[pos];
+  let mask = 0x80;
+  let value = 0;
+  let p = pos + 1;
+  for (let i = 0; i < 8; i++) {
+    if ((first & mask) === 0) {
+      value += (first & (mask - 1)) * Math.pow(2, 8 * i);
+      return { value, next: p };
+    }
+    if (p >= buf.length) throw new RangeError("7z number overruns buffer");
+    value += buf[p] * Math.pow(2, 8 * i);
+    p++;
+    mask >>= 1;
+  }
+  return { value, next: p };
+}
+
+/** For an encoded (kEncodedHeader) 7z next-header, return the absolute-ish
+ *  location of the packed header stream(s): PackPos (relative to end of the
+ *  32-byte signature header) and the summed PackSize. Null if not encoded or
+ *  the StreamsInfo doesn't start with PackInfo as expected. */
+export function locate7zEncodedHeaderPack(
+  nextHeader: Buffer,
+): { packPos: number; packSize: number } | null {
+  let p = 0;
+  if (nextHeader[p] !== K_ENCODED_HEADER) return null;
+  p++;
+  if (nextHeader[p] !== K_PACK_INFO) return null;
+  p++;
+  const packPos = read7zNumber(nextHeader, p); p = packPos.next;
+  const numStreams = read7zNumber(nextHeader, p); p = numStreams.next;
+  if (nextHeader[p] !== K_SIZE) return null;
+  p++;
+  let total = 0;
+  for (let i = 0; i < numStreams.value; i++) {
+    const s = read7zNumber(nextHeader, p); p = s.next;
+    total += s.value;
+  }
+  return { packPos: packPos.value, packSize: total };
+}
+
+export function parseSevenZSignatureHeader(
+  buf: Buffer,
+): { nextHeaderOffset: number; nextHeaderSize: number } | null {
+  if (buf.length < 32) return null;
+  if (!buf.subarray(0, 6).equals(SEVENZ_MAGIC)) return null;
+  return {
+    nextHeaderOffset: Number(buf.readBigUInt64LE(12)),
+    nextHeaderSize: Number(buf.readBigUInt64LE(20)),
+  };
+}
+
+export interface RangedPart { fileId: string; fileSize: bigint; fileName: string }
+
+export async function readSevenZListingRanged(
+  parts: RangedPart[],
+  read: RangeReader,
+): Promise<FileEntry[] | null> {
+  const part = parts[0];
+  if (!part) return null;
+  const size = Number(part.fileSize);
+  try {
+    const sig = await read(part.fileId, 0, 32, part.fileSize);
+    const parsed = parseSevenZSignatureHeader(sig);
+    if (!parsed) return null;
+    const endStart = 32 + parsed.nextHeaderOffset;
+    if (endStart < 0 || endStart + parsed.nextHeaderSize > size) return null;
+    const endHeader = await read(part.fileId, endStart, parsed.nextHeaderSize, part.fileSize);
+
+    const regions = [
+      { offset: 0, bytes: sig },
+      { offset: endStart, bytes: endHeader },
+    ];
+
+    const headerType = endHeader[0];
+    if (headerType === K_ENCODED_HEADER) {
+      // Compressed header: its packed bytes live mid-file, not at EOF. Fetch them.
+      const pack = locate7zEncodedHeaderPack(endHeader);
+      if (!pack) return null;
+      const packStart = 32 + pack.packPos;
+      if (packStart < 0 || packStart + pack.packSize > size) return null;
+      const packBytes = await read(part.fileId, packStart, pack.packSize, part.fileSize);
+      regions.push({ offset: packStart, bytes: packBytes });
+    } else if (headerType !== K_HEADER) {
+      return null; // unknown next-header type
+    }
+
+    return listFromSparse([{ fileName: part.fileName, size, regions }], read7zContents);
+  } catch (err) {
+    log.warn({ err, fileId: part.fileId }, "ranged 7z listing failed");
+    return null;
+  }
+}
@@ -0,0 +1,41 @@
+import { describe, it, expect } from "vitest";
+import { open } from "fs/promises";
+import { listFromSparse } from "./sparse-list.js";
+
+describe("listFromSparse", () => {
+  it("writes each region at its offset into a sparse file and passes the path to the lister", async () => {
+    const size = 1_000_000;
+    const regions = [
+      { offset: 0, bytes: Buffer.from("HEAD") },
+      { offset: size - 4, bytes: Buffer.from("TAIL") },
+    ];
+    let seenPath = "";
+    const entries = await listFromSparse(
+      [{ fileName: "sample.7z", size, regions }],
+      async (firstPartPath) => {
+        seenPath = firstPartPath;
+        const fh = await open(firstPartPath, "r");
+        try {
+          const head = Buffer.alloc(4); await fh.read(head, 0, 4, 0);
+          const tail = Buffer.alloc(4); await fh.read(tail, 0, 4, size - 4);
+          const hole = Buffer.alloc(4); await fh.read(hole, 0, 4, 500_000);
+          expect(head.toString()).toBe("HEAD");
+          expect(tail.toString()).toBe("TAIL");
+          expect(hole.equals(Buffer.alloc(4))).toBe(true); // gap is zero
+        } finally { await fh.close(); }
+        return [{ path: "a/b.stl", fileName: "b.stl", extension: "stl", compressedSize: 1n, uncompressedSize: 1n, crc32: null }];
+      },
+    );
+    expect(seenPath.endsWith("sample.7z")).toBe(true);
+    expect(entries).not.toBeNull();
+    expect(entries!).toHaveLength(1);
+  });
+
+  it("returns null when the lister yields no entries", async () => {
+    const res = await listFromSparse(
+      [{ fileName: "x.7z", size: 100, regions: [{ offset: 0, bytes: Buffer.from("A") }] }],
+      async () => [],
+    );
+    expect(res).toBeNull();
+  });
+});
@@ -0,0 +1,52 @@
+import { mkdtemp, open, rm } from "fs/promises";
+import path from "path";
+import { config } from "../../util/config.js";
+import { childLogger } from "../../util/logger.js";
+import type { FileEntry } from "../zip-reader.js";
+
+const log = childLogger("sparse-list");
+
+export interface SparsePart {
+  fileName: string;
+  size: number;
+  regions: { offset: number; bytes: Buffer }[];
+}
+
+export type SparseLister = (firstPartPath: string) => Promise<FileEntry[]>;
+
+/**
+ * Reconstruct archive header bytes into sparse temp files (data areas left as
+ * zero holes), run `lister` on the first part, return its entries.
+ * Returns null on any error or when the lister finds nothing.
+ */
+export async function listFromSparse(
+  parts: SparsePart[],
+  lister: SparseLister,
+): Promise<FileEntry[] | null> {
+  if (parts.length === 0) return null;
+  const dir = await mkdtemp(path.join(config.tempDir, "ranged-"));
+  try {
+    let firstPath = "";
+    for (let i = 0; i < parts.length; i++) {
+      const p = parts[i];
+      const filePath = path.join(dir, p.fileName);
+      if (i === 0) firstPath = filePath;
+      const fh = await open(filePath, "w");
+      try {
+        await fh.truncate(p.size); // create the sparse hole
+        for (const r of p.regions) {
+          await fh.write(r.bytes, 0, r.bytes.length, r.offset);
+        }
+      } finally {
+        await fh.close();
+      }
+    }
+    const entries = await lister(firstPath);
+    return entries.length > 0 ? entries : null;
+  } catch (err) {
+    log.warn({ err }, "sparse listing failed");
+    return null;
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+}
@@ -8,83 +8,122 @@ const execFileAsync = promisify(execFile);
 const log = childLogger("rar-reader");

 /**
- * Parse output of `unrar l -v <file>` to extract file metadata.
- * unrar automatically discovers sibling parts when they're co-located.
+ * Parse output of `unrar lt <file>` to extract file metadata.
+ *
+ * `lt` (list technical) emits one block per archived file with key:value
+ * lines — far more reliable than the column-based default `l -v` output,
+ * which has changed format twice across unrar versions.
+ *
+ * unrar automatically discovers sibling multipart files when they're
+ * co-located (e.g. *.part1.rar + *.part2.rar in the same directory).
+ *
+ * Returns [] on any failure (best-effort: ingestion still succeeds with
+ * an empty file list rather than failing the whole archive).
 */
 export async function readRarContents(
  firstPartPath: string
 ): Promise<FileEntry[]> {
  try {
-    const { stdout } = await execFileAsync("unrar", ["l", "-v", firstPartPath], {
-      timeout: 30000,
-      maxBuffer: 10 * 1024 * 1024, // 10MB for very large archives
+    const { stdout } = await execFileAsync("unrar", ["lt", firstPartPath], {
+      timeout: 60_000,
+      maxBuffer: 50 * 1024 * 1024, // 50MB for archives with very many files
    });

-    return parseUnrarOutput(stdout);
+    const entries = parseUnrarTechnical(stdout);
+    if (entries.length === 0) {
+      // Log a sample of the output so we can diagnose format changes
+      log.warn(
+        { file: firstPartPath, sample: stdout.slice(0, 500) },
+        "unrar lt returned no parseable entries"
+      );
+    }
+    return entries;
  } catch (err) {
    log.warn({ err, file: firstPartPath }, "Failed to read RAR contents");
-    return []; // Fallback: return empty on error
+    return [];
  }
 }

 /**
- * Parse the tabular output of `unrar l -v`.
+ * Parse `unrar lt` output: header followed by per-file key:value blocks
+ * separated by blank lines.
 *
- * Example output format:
- *  Archive: test.rar
- *  Details: RAR 5
+ * Example block:
 *
- *   Attributes      Size     Packed Ratio   Date   Time   CRC-32  Name
- *  ----------- ---------  --------- ----- -------- ----- --------  ----
- *   ...A....      12345      10234  83%  2024-01-15 10:30 DEADBEEF  folder/file.stl
- *  ----------- ---------  --------- ----- -------- ----- --------  ----
+ *         Name: folder/file.stl
+ *         Type: File
+ *         Size: 12345
+ *  Packed size: 10234
+ *        Ratio: 83%
+ *        mtime: 2024-01-15 10:30:00,000000000
+ *   Attributes: ..A....
+ *        CRC32: DEADBEEF
+ *      Host OS: Windows
+ *  Compression: RAR 5.0(v50) -m3 -md=32M
 */
-function parseUnrarOutput(output: string): FileEntry[] {
+function parseUnrarTechnical(output: string): FileEntry[] {
  const entries: FileEntry[] = [];
-  const lines = output.split("\n");
+  // Split into blocks on blank lines, then on each block read key:value pairs.
+  const blocks = output.split(/\r?\n\s*\r?\n/);

-  let inFileList = false;
-  let separatorCount = 0;
+  for (const block of blocks) {
+    const fields = parseBlock(block);
+    if (!fields) continue;

-  for (const line of lines) {
-    const trimmed = line.trim();
-
-    // Detect separator lines (------- pattern)
-    if (/^-{5,}/.test(trimmed)) {
-      separatorCount++;
-      if (separatorCount === 1) {
-        inFileList = true;
-      } else if (separatorCount >= 2) {
-        inFileList = false;
-      }
-      continue;
-    }
-
-    if (!inFileList) continue;
-
-    // Parse file entry line
-    // Format: Attributes Size Packed Ratio Date Time CRC Name
-    const match = trimmed.match(
-      /^\S+\s+(\d+)\s+(\d+)\s+\d+%\s+\S+\s+\S+\s+([0-9A-Fa-f]+)\s+(.+)$/
-    );
-
-    if (match) {
-      const [, uncompressedStr, compressedStr, crc32, filePath] = match;
-
-      // Skip directory entries (typically end with / or have size 0 with dir attributes)
-      if (filePath.endsWith("/") || filePath.endsWith("\\")) continue;
+    // Only File entries (skip Directory, and anything missing the basics)
+    if (fields.type && fields.type.toLowerCase() !== "file") continue;
+    if (!fields.name || fields.size === undefined) continue;

+    const filePath = fields.name;
    const ext = path.extname(filePath).toLowerCase();
    entries.push({
      path: filePath,
      fileName: path.basename(filePath),
      extension: ext ? ext.slice(1) : null,
-        compressedSize: BigInt(compressedStr),
-        uncompressedSize: BigInt(uncompressedStr),
-        crc32: crc32.toLowerCase(),
+      uncompressedSize: BigInt(fields.size),
+      compressedSize: fields.packedSize !== undefined
+        ? BigInt(fields.packedSize)
+        : BigInt(fields.size),
+      crc32: fields.crc32 ? fields.crc32.toLowerCase() : null,
    });
  }
-  }

  return entries;
 }
+
+interface BlockFields {
+  name?: string;
+  type?: string;
+  size?: string;
+  packedSize?: string;
+  crc32?: string;
+}
+
+function parseBlock(block: string): BlockFields | null {
+  // Skip the archive-header block (contains "Archive:" / "Details:" lines
+  // and lacks a Name field).
+  if (!/^\s*Name:/m.test(block)) return null;
+
+  const fields: BlockFields = {};
+  const lines = block.split(/\r?\n/);
+
+  for (const line of lines) {
+    // Match "  key: value" with arbitrary leading whitespace and a multi-word
+    // key (e.g. "Packed size", "Host OS").
+    const m = line.match(/^\s*([A-Za-z][A-Za-z0-9 ]*?)\s*:\s*(.*)$/);
+    if (!m) continue;
+    const key = m[1].trim().toLowerCase();
+    const value = m[2].trim();
+    if (key === "name") fields.name = value;
+    else if (key === "type") fields.type = value;
+    else if (key === "size") fields.size = value;
+    else if (key === "packed size") fields.packedSize = value;
+    else if (key === "crc32" || key === "blake2sp" || key === "checksum") {
+      // unrar may report BLAKE2sp for newer archives instead of CRC32.
+      // Either way we just store it as a hex string in our crc32 field.
+      fields.crc32 = value;
+    }
+  }
+
+  return fields;
+}
@@ -0,0 +1,58 @@
+import type { FileEntry } from "./zip-reader.js";
+
+/**
+ * Mapping from file extensions to slicer tags. Each tag groups a family of
+ * extensions that mean the same thing for end users — "this archive contains
+ * files I can open in <slicer>".
+ *
+ * Extensions are matched case-insensitively without the leading dot.
+ */
+const SLICER_EXTENSION_MAP: Record<string, string> = {
+  // Lychee Slicer
+  lys: "lychee",
+  lyt: "lychee",
+  lyc: "lychee",
+
+  // ChituBox / Anycubic / Phrozen / Elegoo (resin printers)
+  chitubox: "chitubox",
+  ctb: "chitubox",
+  cbddlp: "chitubox",
+
+  // Anycubic Photon family
+  photon: "anycubic",
+  pwmo: "anycubic",
+  pwmx: "anycubic",
+  pwmb: "anycubic",
+  pwma: "anycubic",
+  pws: "anycubic",
+  pwsq: "anycubic",
+  phz: "anycubic",
+
+  // Bambu / Prusa
+  "3mf": "bambu",
+  bgcode: "bambu",
+
+  // FDM gcode (generic)
+  gcode: "fdm",
+
+  // Mango / generic resin formats sometimes seen in releases
+  mfp: "mango",
+  mfpv: "mango",
+  osla: "mango",
+};
+
+/**
+ * Derive a deduplicated list of slicer tags from an archive's file listing.
+ * Returns an empty array if no recognised slicer-specific files are present
+ * (e.g., the archive is just STLs without pre-supports).
+ */
+export function extractSlicerTags(entries: FileEntry[]): string[] {
+  const tags = new Set<string>();
+  for (const entry of entries) {
+    if (!entry.extension) continue;
+    const ext = entry.extension.toLowerCase();
+    const tag = SLICER_EXTENSION_MAP[ext];
+    if (tag) tags.add(tag);
+  }
+  return [...tags].sort();
+}
@@ -18,20 +18,22 @@ const log = childLogger("split");
 const MAX_PART_SIZE = BigInt(config.maxPartSizeMB) * 1024n * 1024n;

 /**
- * Split a file into ≤2GB parts using byte-level splitting.
- * Returns paths to the split parts. If the file is already ≤2GB, returns the original path.
+ * Split a file into parts using byte-level splitting.
+ * Returns paths to the split parts. If the file fits in one part, returns the original path.
+ * Pass maxPartSize to override the global default (e.g., 3950 MiB for Premium accounts).
 */
-export async function byteLevelSplit(filePath: string): Promise<string[]> {
+export async function byteLevelSplit(filePath: string, maxPartSize?: bigint): Promise<string[]> {
+  const effectiveMax = maxPartSize ?? MAX_PART_SIZE;
  const stats = await stat(filePath);
  const fileSize = BigInt(stats.size);

-  if (fileSize <= MAX_PART_SIZE) {
+  if (fileSize <= effectiveMax) {
    return [filePath];
  }

  const dir = path.dirname(filePath);
  const baseName = path.basename(filePath);
-  const partSize = Number(MAX_PART_SIZE);
+  const partSize = Number(effectiveMax);
  const totalParts = Math.ceil(Number(fileSize) / partSize);
  const parts: string[] = [];

@@ -0,0 +1,119 @@
+import { db } from "./db/client.js";
+import { childLogger } from "./util/logger.js";
+
+const log = childLogger("audit");
+
+/**
+ * Periodic integrity audit: checks all packages for consistency.
+ * Creates SystemNotification records for any issues found.
+ *
+ * Checks performed:
+ * 1. Multipart completeness: destMessageIds.length should match partCount
+ * 2. Missing destination: packages with destChannelId but no destMessageId
+ */
+export async function runIntegrityAudit(): Promise<{ checked: number; issues: number }> {
+  log.info("Starting integrity audit");
+
+  let checked = 0;
+  let issues = 0;
+
+  // Check 1: Multipart packages with wrong number of destination message IDs
+  const multipartPackages = await db.package.findMany({
+    where: {
+      isMultipart: true,
+      partCount: { gt: 1 },
+      destMessageId: { not: null },
+    },
+    select: {
+      id: true,
+      fileName: true,
+      partCount: true,
+      destMessageIds: true,
+      sourceChannelId: true,
+      sourceChannel: { select: { title: true } },
+    },
+  });
+
+  checked += multipartPackages.length;
+
+  for (const pkg of multipartPackages) {
+    const actualParts = pkg.destMessageIds.length;
+    // Only flag when we have >1 stored IDs but count doesn't match.
+    // Packages with exactly 1 ID are legacy (backfilled from single destMessageId) — not actionable.
+    if (actualParts > 1 && actualParts !== pkg.partCount) {
+      issues++;
+
+      // Check if we already have a notification for this
+      const existing = await db.systemNotification.findFirst({
+        where: {
+          type: "MISSING_PART",
+          context: { path: ["packageId"], equals: pkg.id },
+        },
+        select: { id: true },
+      });
+
+      if (!existing) {
+        await db.systemNotification.create({
+          data: {
+            type: "MISSING_PART",
+            severity: "WARNING",
+            title: `Incomplete multipart: ${pkg.fileName}`,
+            message: `Expected ${pkg.partCount} parts but only ${actualParts} destination message IDs stored`,
+            context: {
+              packageId: pkg.id,
+              fileName: pkg.fileName,
+              expectedParts: pkg.partCount,
+              actualParts,
+              sourceChannelId: pkg.sourceChannelId,
+              channelTitle: pkg.sourceChannel.title,
+            },
+          },
+        });
+
+        log.warn(
+          { packageId: pkg.id, fileName: pkg.fileName, expected: pkg.partCount, actual: actualParts },
+          "Multipart package has mismatched part count"
+        );
+      }
+    }
+  }
+
+  // Check 2: Packages with dest channel but no dest message (orphaned index)
+  const orphanedCount = await db.package.count({
+    where: {
+      destChannelId: { not: null },
+      destMessageId: null,
+    },
+  });
+
+  if (orphanedCount > 0) {
+    issues++;
+
+    const existing = await db.systemNotification.findFirst({
+      where: {
+        type: "INTEGRITY_AUDIT",
+        context: { path: ["check"], equals: "orphaned_index" },
+        createdAt: { gte: new Date(Date.now() - 24 * 60 * 60 * 1000) },
+      },
+      select: { id: true },
+    });
+
+    if (!existing) {
+      await db.systemNotification.create({
+        data: {
+          type: "INTEGRITY_AUDIT",
+          severity: "INFO",
+          title: `${orphanedCount} packages with missing destination message`,
+          message: `Found ${orphanedCount} packages that have a destination channel set but no destination message ID. These may be from interrupted uploads.`,
+          context: {
+            check: "orphaned_index",
+            count: orphanedCount,
+          },
+        },
+      });
+    }
+  }
+
+  log.info({ checked, issues }, "Integrity audit complete");
+  return { checked, issues };
+}
@@ -0,0 +1,343 @@
+import path from "path";
+import { mkdir, rm } from "fs/promises";
+import { db } from "./db/client.js";
+import { config } from "./util/config.js";
+import { childLogger } from "./util/logger.js";
+import { withTdlibMutex } from "./util/mutex.js";
+import { createTdlibClient, closeTdlibClient } from "./tdlib/client.js";
+import { downloadFile } from "./tdlib/download.js";
+import { getActiveAccounts } from "./db/queries.js";
+import { readZipCentralDirectory } from "./archive/zip-reader.js";
+import { readRarContents } from "./archive/rar-reader.js";
+import { read7zContents } from "./archive/sevenz-reader.js";
+import { extractSlicerTags } from "./archive/slicer-tags.js";
+import type { FileEntry } from "./archive/zip-reader.js";
+
+const log = childLogger("backfill");
+
+/**
+ * Re-extract file listings for Packages whose fileCount is 0 — usually
+ * caused by historical bugs in the archive readers (e.g. the RAR parser
+ * that silently returned [] for every archive before 0bdd4ba).
+ *
+ * For each candidate Package:
+ *   1. Download all destMessageIds from the destination channel
+ *   2. Run the appropriate reader (ZIP / RAR / 7Z) on the assembled files
+ *   3. Insert PackageFile rows + update Package.fileCount
+ *   4. Clean up the temp files
+ *
+ * Triggered via pg_notify "backfill_filelists" with optional payload
+ * `{"limit": N, "archiveType": "RAR"}` — both fields optional, defaults
+ * are limit=100, archiveType=any.
+ */
+export async function processBackfillRequest(payloadJson: string): Promise<void> {
+  let limit = 100;
+  let archiveTypeFilter: "ZIP" | "RAR" | "SEVEN_Z" | undefined;
+  try {
+    const parsed = JSON.parse(payloadJson) as { limit?: number; archiveType?: string };
+    if (typeof parsed.limit === "number" && parsed.limit > 0) limit = parsed.limit;
+    if (parsed.archiveType === "ZIP" || parsed.archiveType === "RAR" || parsed.archiveType === "SEVEN_Z") {
+      archiveTypeFilter = parsed.archiveType;
+    }
+  } catch {
+    // Empty / invalid payload — use defaults
+  }
+
+  const candidates = await db.package.findMany({
+    where: {
+      fileCount: 0,
+      destChannelId: { not: null },
+      destMessageId: { not: null },
+      archiveType: archiveTypeFilter
+        ? archiveTypeFilter
+        : { in: ["ZIP", "RAR", "SEVEN_Z"] },
+    },
+    select: {
+      id: true,
+      fileName: true,
+      fileSize: true,
+      archiveType: true,
+      destChannelId: true,
+      destMessageId: true,
+      destMessageIds: true,
+      isMultipart: true,
+      partCount: true,
+    },
+    orderBy: { createdAt: "asc" },
+    take: limit,
+  });
+
+  if (candidates.length === 0) {
+    log.info({ archiveTypeFilter }, "Backfill: no candidates with fileCount=0");
+    return;
+  }
+
+  log.info(
+    { count: candidates.length, archiveTypeFilter },
+    "Backfill: starting batch"
+  );
+
+  const accounts = await getActiveAccounts();
+  if (accounts.length === 0) {
+    log.warn("Backfill: no authenticated accounts — aborting");
+    return;
+  }
+
+  // Prefer the Premium account if available (faster downloads, larger files)
+  const account = accounts.find((a) => a.isPremium) ?? accounts[0];
+
+  await withTdlibMutex(account.phone, "backfill", async () => {
+    const { client } = await createTdlibClient({ id: account.id, phone: account.phone });
+
+    try {
+      // Load chats so TDLib knows about the destination chat
+      try {
+        await client.invoke({
+          _: "getChats",
+          chat_list: { _: "chatListMain" },
+          limit: 1000,
+        });
+      } catch {
+        // May already be loaded
+      }
+
+      let processed = 0;
+      let succeeded = 0;
+      let failed = 0;
+
+      for (const pkg of candidates) {
+        processed++;
+        const ctx = { packageId: pkg.id, fileName: pkg.fileName };
+
+        try {
+          await processOnePackage(client, pkg, ctx);
+          succeeded++;
+        } catch (err) {
+          failed++;
+          log.warn({ err, ...ctx }, "Backfill failed for package");
+        }
+      }
+
+      log.info(
+        { processed, succeeded, failed, archiveTypeFilter },
+        "Backfill batch complete"
+      );
+    } finally {
+      await closeTdlibClient(client).catch(() => {});
+    }
+  });
+}
+
+interface BackfillPackage {
+  id: string;
+  fileName: string;
+  fileSize: bigint;
+  archiveType: "ZIP" | "RAR" | "SEVEN_Z" | "DOCUMENT" | string;
+  destChannelId: string | null;
+  destMessageId: bigint | null;
+  destMessageIds: bigint[];
+  isMultipart: boolean;
+  partCount: number;
+}
+
+async function processOnePackage(
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  client: any,
+  pkg: BackfillPackage,
+  ctx: { packageId: string; fileName: string }
+): Promise<void> {
+  if (!pkg.destChannelId || !pkg.destMessageId) {
+    log.debug(ctx, "Skipping: no destination channel/message");
+    return;
+  }
+
+  // Look up the destination channel's Telegram ID
+  const destChannel = await db.telegramChannel.findUnique({
+    where: { id: pkg.destChannelId },
+    select: { telegramId: true },
+  });
+  if (!destChannel) {
+    throw new Error("Destination channel not found in DB");
+  }
+  const chatId = Number(destChannel.telegramId);
+
+  // Resolve which message IDs to download. The Package may carry a
+  // single destMessageId or multiple destMessageIds (for multipart).
+  const messageIds: bigint[] =
+    pkg.destMessageIds.length > 0
+      ? pkg.destMessageIds
+      : pkg.destMessageId
+      ? [pkg.destMessageId]
+      : [];
+
+  if (messageIds.length === 0) {
+    throw new Error("Package has no destination message IDs");
+  }
+
+  const tempDir = path.join(config.tempDir, `backfill_${pkg.id}`);
+  await mkdir(tempDir, { recursive: true });
+
+  try {
+    const partPaths: string[] = [];
+
+    for (let i = 0; i < messageIds.length; i++) {
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      const message = (await client.invoke({
+        _: "getMessage",
+        chat_id: chatId,
+        message_id: Number(messageIds[i]),
+      })) as unknown as {
+        content?: { document?: { file_name?: string; document?: { id: number; size: number } } };
+      };
+
+      const doc = message?.content?.document;
+      if (!doc?.document?.id) {
+        throw new Error(`Destination message ${messageIds[i]} has no document`);
+      }
+
+      const fileId = String(doc.document.id);
+      const fileName = doc.file_name ?? `${pkg.id}.part${i + 1}`;
+      const localPath = path.join(tempDir, fileName);
+
+      await downloadFile(
+        client,
+        fileId,
+        localPath,
+        BigInt(doc.document.size),
+        fileName
+      );
+
+      partPaths.push(localPath);
+    }
+
+    // Run the appropriate reader on the assembled file(s)
+    let entries: FileEntry[] = [];
+    if (pkg.archiveType === "ZIP") {
+      entries = await readZipCentralDirectory(partPaths);
+    } else if (pkg.archiveType === "RAR") {
+      // unrar auto-discovers sibling parts when in the same directory
+      entries = await readRarContents(partPaths[0]);
+    } else if (pkg.archiveType === "SEVEN_Z") {
+      entries = await read7zContents(partPaths[0]);
+    } else {
+      log.debug({ ...ctx, archiveType: pkg.archiveType }, "Skipping unsupported archive type");
+      return;
+    }
+
+    if (entries.length === 0) {
+      log.warn(ctx, "Reader returned 0 entries — archive may be encrypted or corrupt");
+      return;
+    }
+
+    // Also derive slicer tags from the file list so the backfilled packages
+    // gain the same search/filter context as newly-ingested ones.
+    const slicerTags = extractSlicerTags(entries);
+
+    // Write everything in a single transaction so a partial backfill never
+    // leaves the Package half-indexed.
+    await db.$transaction(async (tx) => {
+      // Re-check fileCount inside the transaction: another worker might
+      // have backfilled this package between our read and write.
+      const current = await tx.package.findUnique({
+        where: { id: pkg.id },
+        select: { fileCount: true, tags: true },
+      });
+      if (current && current.fileCount > 0) {
+        log.debug({ ...ctx, existingFileCount: current.fileCount }, "Already backfilled by another worker — skipping");
+        return;
+      }
+
+      await tx.packageFile.deleteMany({ where: { packageId: pkg.id } });
+      await tx.packageFile.createMany({
+        data: entries.map((e) => ({
+          packageId: pkg.id,
+          path: e.path,
+          fileName: e.fileName,
+          extension: e.extension,
+          compressedSize: e.compressedSize,
+          uncompressedSize: e.uncompressedSize,
+          crc32: e.crc32,
+        })),
+      });
+
+      // Merge slicer tags with whatever's already on the Package (preserve
+      // channel category, manual tags, etc.).
+      const existingTags = current?.tags ?? [];
+      const mergedTags = [...new Set([...existingTags, ...slicerTags])];
+
+      await tx.package.update({
+        where: { id: pkg.id },
+        data: { fileCount: entries.length, tags: mergedTags },
+      });
+    });
+
+    log.info({ ...ctx, fileCount: entries.length }, "Backfilled file list");
+  } finally {
+    await rm(tempDir, { recursive: true, force: true }).catch(() => {});
+  }
+}
+
+/**
+ * Cheap pure-DB backfill: walk Packages that already have PackageFile rows
+ * but no slicer tags, recompute the tags from their extensions, and merge
+ * with the existing tag list. No downloads, no TDLib.
+ *
+ * Trigger:
+ *   SELECT pg_notify('backfill_slicer_tags', '{"limit":1000}');
+ */
+export async function processSlicerTagBackfill(payloadJson: string): Promise<void> {
+  let limit = 1000;
+  try {
+    const parsed = JSON.parse(payloadJson) as { limit?: number };
+    if (typeof parsed.limit === "number" && parsed.limit > 0) limit = parsed.limit;
+  } catch {
+    // Use default
+  }
+
+  // KNOWN_TAGS = the slicer tags we know how to derive. A Package missing
+  // all of these is a candidate for recompute. extractSlicerTags is safe
+  // to run on every package (returns [] for archives with no slicer files),
+  // but filtering up-front avoids walking the entire DB.
+  const KNOWN_TAGS = ["lychee", "chitubox", "anycubic", "bambu", "fdm", "mango"];
+
+  const candidates = await db.package.findMany({
+    where: {
+      fileCount: { gt: 0 },
+      NOT: { tags: { hasSome: KNOWN_TAGS } },
+    },
+    select: {
+      id: true,
+      tags: true,
+      files: { select: { extension: true } },
+    },
+    orderBy: { createdAt: "asc" },
+    take: limit,
+  });
+
+  if (candidates.length === 0) {
+    log.info("Slicer tag backfill: no candidates");
+    return;
+  }
+
+  log.info({ count: candidates.length }, "Slicer tag backfill: starting");
+
+  let updated = 0;
+  for (const pkg of candidates) {
+    const fileEntries = pkg.files.map((f) => ({
+      path: "",
+      fileName: "",
+      extension: f.extension,
+      compressedSize: 0n,
+      uncompressedSize: 0n,
+      crc32: null as string | null,
+    }));
+    const slicerTags = extractSlicerTags(fileEntries);
+    if (slicerTags.length === 0) continue;
+    const merged = [...new Set([...pkg.tags, ...slicerTags])];
+    if (merged.length === pkg.tags.length) continue;
+    await db.package.update({ where: { id: pkg.id }, data: { tags: merged } });
+    updated++;
+  }
+
+  log.info({ candidates: candidates.length, updated }, "Slicer tag backfill: done");
+}
@@ -5,7 +5,14 @@ import { config } from "../util/config.js";

 const pool = new pg.Pool({
  connectionString: config.databaseUrl,
-  max: 5,
+  // Pool needs headroom for: 2 account advisory locks (held for entire cycle),
+  // up to 2 concurrent hash locks, plus Prisma operations from both accounts.
+  // Previously max=5 caused pool exhaustion and indefinite hangs.
+  max: 15,
+  // Prevent pool.connect() from blocking forever when pool is exhausted.
+  // Throws an error after 30s so the operation can fail and retry instead of
+  // silently hanging for hours (as happened with the Turnbase.7z stall).
+  connectionTimeoutMillis: 30_000,
 });

 const adapter = new PrismaPg(pool);
@@ -79,3 +79,66 @@ export async function releaseLock(accountId: string): Promise<void> {
    client.release();
  }
 }
+
+/**
+ * Derive a lock ID for a content hash. Prefixes with "hash:" so the resulting
+ * 32-bit integer does not collide with account advisory lock IDs.
+ */
+function contentHashToLockId(contentHash: string): number {
+  return hashToLockId(`hash:${contentHash}`);
+}
+
+/**
+ * Acquire a per-content-hash advisory lock before uploading.
+ * Prevents two concurrent workers from uploading the same archive
+ * when both scan a shared source channel.
+ *
+ * Returns true if acquired (proceed with upload).
+ * Returns false if already held (another worker is handling this archive — skip).
+ *
+ * MUST be released via releaseHashLock() after createPackageStub() completes,
+ * including on all error paths (use try/finally).
+ */
+export async function tryAcquireHashLock(contentHash: string): Promise<boolean> {
+  const lockId = contentHashToLockId(contentHash);
+  const client = await pool.connect();
+  try {
+    const result = await client.query<{ pg_try_advisory_lock: boolean }>(
+      "SELECT pg_try_advisory_lock($1)",
+      [lockId]
+    );
+    const acquired = result.rows[0]?.pg_try_advisory_lock ?? false;
+    if (acquired) {
+      heldConnections.set(`hash:${contentHash}`, client);
+      log.debug({ hash: contentHash.slice(0, 16), lockId }, "Hash lock acquired");
+      return true;
+    } else {
+      client.release();
+      log.debug({ hash: contentHash.slice(0, 16), lockId }, "Hash lock held by another worker — skipping");
+      return false;
+    }
+  } catch (err) {
+    client.release();
+    throw err;
+  }
+}
+
+/**
+ * Release the per-content-hash advisory lock.
+ * Call after createPackageStub() completes (or on any error path).
+ */
+export async function releaseHashLock(contentHash: string): Promise<void> {
+  const lockId = contentHashToLockId(contentHash);
+  const client = heldConnections.get(`hash:${contentHash}`);
+  if (!client) {
+    log.warn({ hash: contentHash.slice(0, 16) }, "No held connection for hash lock release");
+    return;
+  }
+  try {
+    await client.query("SELECT pg_advisory_unlock($1)", [lockId]);
+    log.debug({ hash: contentHash.slice(0, 16) }, "Hash lock released");
+  } finally {
+    heldConnections.delete(`hash:${contentHash}`);
+    client.release();
+  }
+}
@@ -1,5 +1,6 @@
 import { db } from "./client.js";
 import type { ArchiveType, FetchStatus } from "@prisma/client";
+import type { FileEntry } from "../archive/zip-reader.js";

 export async function getActiveAccounts() {
  return db.telegramAccount.findMany({
@@ -74,6 +75,108 @@ export async function getUploadedPackageByHash(contentHash: string) {
  });
 }

+export interface CreatePackageStubInput {
+  contentHash: string;
+  fileName: string;
+  fileSize: bigint;
+  archiveType: ArchiveType;
+  sourceChannelId: string;
+  sourceMessageId: bigint;
+  sourceTopicId?: bigint | null;
+  /** TDLib remote.unique_id of the first part — for future dedup. */
+  remoteUniqueId?: string | null;
+  destChannelId: string;
+  destMessageId: bigint;
+  destMessageIds: bigint[];
+  isMultipart: boolean;
+  partCount: number;
+  ingestionRunId: string;
+  creator?: string | null;
+  tags?: string[];
+}
+
+/**
+ * Write a minimal Package record immediately after Telegram confirms the upload.
+ * Call this before preview/metadata extraction so recoverIncompleteUploads() can
+ * detect and verify the package if the worker crashes mid-metadata.
+ *
+ * Follow with updatePackageWithMetadata() once file entries and preview are ready.
+ */
+export async function createPackageStub(
+  input: CreatePackageStubInput
+): Promise<{ id: string }> {
+  const pkg = await db.package.create({
+    data: {
+      contentHash: input.contentHash,
+      fileName: input.fileName,
+      fileSize: input.fileSize,
+      archiveType: input.archiveType,
+      sourceChannelId: input.sourceChannelId,
+      sourceMessageId: input.sourceMessageId,
+      sourceTopicId: input.sourceTopicId ?? undefined,
+      remoteUniqueId: input.remoteUniqueId ?? undefined,
+      destChannelId: input.destChannelId,
+      destMessageId: input.destMessageId,
+      destMessageIds: input.destMessageIds,
+      isMultipart: input.isMultipart,
+      partCount: input.partCount,
+      fileCount: 0,
+      ingestionRunId: input.ingestionRunId,
+      creator: input.creator ?? undefined,
+      tags: input.tags?.length ? input.tags : undefined,
+    },
+    select: { id: true },
+  });
+
+  try {
+    await db.$queryRawUnsafe(
+      `SELECT pg_notify('new_package', $1)`,
+      JSON.stringify({
+        packageId: pkg.id,
+        fileName: input.fileName,
+        creator: input.creator ?? null,
+        tags: input.tags ?? [],
+      })
+    );
+  } catch {
+    // Best-effort
+  }
+
+  return pkg;
+}
+
+/**
+ * Update a stub Package with file entries and preview after metadata extraction.
+ * Called as Phase 2 of the two-phase write after createPackageStub().
+ */
+export async function updatePackageWithMetadata(
+  packageId: string,
+  input: {
+    files: {
+      path: string;
+      fileName: string;
+      extension: string | null;
+      compressedSize: bigint;
+      uncompressedSize: bigint;
+      crc32: string | null;
+    }[];
+    previewData?: Buffer | null;
+    previewMsgId?: bigint | null;
+  }
+): Promise<void> {
+  await db.package.update({
+    where: { id: packageId },
+    data: {
+      fileCount: input.files.length,
+      previewData: input.previewData ? new Uint8Array(input.previewData) : undefined,
+      previewMsgId: input.previewMsgId ?? undefined,
+      files: {
+        create: input.files,
+      },
+    },
+  });
+}
+
 /**
 * Check if a package already exists for a given source message ID
 * AND was successfully uploaded to the destination (destMessageId is set).
@@ -90,6 +193,73 @@ export async function packageExistsBySourceMessage(
  return pkg !== null;
 }

+/**
+ * Strongest pre-download dedup signal: a Package in this channel already
+ * has a matching TDLib remote.unique_id. The unique_id is stable across
+ * reposts of the exact same file content, so a hit is a guaranteed
+ * (lossless) duplicate. No false positives.
+ *
+ * Falls back to the older findRepostedPackage (name + size) for packages
+ * that were ingested before we started capturing remote.unique_id.
+ */
+export async function findPackageByRemoteUniqueId(
+  sourceChannelId: string,
+  remoteUniqueId: string
+): Promise<{
+  id: string;
+  destMessageId: bigint | null;
+  sourceTopicId: bigint | null;
+} | null> {
+  return db.package.findFirst({
+    where: {
+      sourceChannelId,
+      remoteUniqueId,
+      destMessageId: { not: null },
+    },
+    orderBy: { sourceTopicId: { sort: "desc", nulls: "last" } },
+    select: { id: true, destMessageId: true, sourceTopicId: true },
+  });
+}
+
+/**
+ * Detect a likely repost: same source channel + same fileName + same total
+ * fileSize already exists with destMessageId set. Used to skip downloads
+ * when the channel admin re-posts the same file under a new message ID
+ * (which `packageExistsBySourceMessage` cannot catch because the message ID
+ * is different).
+ *
+ * Returns the existing package's destMessageId for logging/observability,
+ * or null if no match. Approximate: same name + same total size is an
+ * extremely strong signal that it's the same content, but theoretically
+ * two unrelated files could collide. If that ever happens, the new file
+ * gets treated as a duplicate and is lost; the user can manually re-link
+ * via the UI by removing the existing Package.
+ */
+export async function findRepostedPackage(
+  sourceChannelId: string,
+  fileName: string,
+  fileSize: bigint
+): Promise<{
+  id: string;
+  destMessageId: bigint | null;
+  sourceTopicId: bigint | null;
+} | null> {
+  return db.package.findFirst({
+    where: {
+      sourceChannelId,
+      fileName,
+      fileSize,
+      destMessageId: { not: null },
+    },
+    // Prefer the existing Package with the most specific (non-NULL)
+    // sourceTopicId, so when the user is re-scanning and the file already
+    // exists in a specific topic, the audit log shows the most informative
+    // match. NULLS LAST in DESC order achieves that for any non-null IDs.
+    orderBy: { sourceTopicId: { sort: "desc", nulls: "last" } },
+    select: { id: true, destMessageId: true, sourceTopicId: true },
+  });
+}
+
 /**
 * Delete orphaned Package rows that have the same content hash but never
 * completed the upload (destMessageId is null). Called before creating a
@@ -119,6 +289,8 @@ export interface CreatePackageInput {
  tags?: string[];
  previewData?: Buffer | null;
  previewMsgId?: bigint | null;
+  sourceCaption?: string | null;
+  replyToMessageId?: bigint | null;
  files: {
    path: string;
    fileName: string;
@@ -150,6 +322,8 @@ export async function createPackageWithFiles(input: CreatePackageInput) {
      tags: input.tags && input.tags.length > 0 ? input.tags : undefined,
      previewData: input.previewData ? new Uint8Array(input.previewData) : undefined,
      previewMsgId: input.previewMsgId ?? undefined,
+      sourceCaption: input.sourceCaption ?? undefined,
+      replyToMessageId: input.replyToMessageId ?? undefined,
      files: {
        create: input.files,
      },
@@ -196,10 +370,13 @@ export interface ActivityUpdate {
  downloadedBytes?: bigint | null;
  totalBytes?: bigint | null;
  downloadPercent?: number | null;
+  currentTopicId?: bigint | null;
+  currentAccountChannelMapId?: string | null;
  messagesScanned?: number;
  zipsFound?: number;
  zipsDuplicate?: number;
  zipsIngested?: number;
+  zipsBackfilled?: number;
 }

 export async function updateRunActivity(
@@ -223,6 +400,11 @@ export async function updateRunActivity(
      ...(activity.zipsFound !== undefined && { zipsFound: activity.zipsFound }),
      ...(activity.zipsDuplicate !== undefined && { zipsDuplicate: activity.zipsDuplicate }),
      ...(activity.zipsIngested !== undefined && { zipsIngested: activity.zipsIngested }),
+      ...(activity.zipsBackfilled !== undefined && { zipsBackfilled: activity.zipsBackfilled }),
+      ...(activity.currentTopicId !== undefined && { currentTopicId: activity.currentTopicId }),
+      ...(activity.currentAccountChannelMapId !== undefined && {
+        currentAccountChannelMapId: activity.currentAccountChannelMapId,
+      }),
    },
  });
 }
@@ -237,6 +419,8 @@ const CLEAR_ACTIVITY = {
  downloadedBytes: null,
  totalBytes: null,
  downloadPercent: null,
+  currentTopicId: null,
+  currentAccountChannelMapId: null,
  lastActivityAt: new Date(),
 };

@@ -247,6 +431,7 @@ export async function completeIngestionRun(
    zipsFound: number;
    zipsDuplicate: number;
    zipsIngested: number;
+    zipsBackfilled: number;
  }
 ) {
  return db.ingestionRun.update({
@@ -282,6 +467,73 @@ export async function updateLastProcessedMessage(
  });
 }

+export interface ScanStateUpdate {
+  /** New watermark to persist. Use the same value the caller would have
+   *  passed to updateLastProcessedMessage / upsertTopicProgress. */
+  lastProcessedMessageId: bigint | null;
+  /** True if the scan found archives OR has retryable SkippedPackages
+   *  pending. The caller computes this via the trulyIdle formula. */
+  lastScanFoundArchives: boolean;
+  /** Pre-incremented value of consecutiveEmptyScans. Caller passes:
+   *    trulyIdle ? prev + 1 : 0
+   *  We do the arithmetic outside the helper so the helper stays a pure
+   *  setter — easier to reason about. */
+  consecutiveEmptyScans: number;
+}
+
+/**
+ * Atomically update an AccountChannelMap's watermark and scan-state fields.
+ * Replaces the older updateLastProcessedMessage for the post-scan write.
+ * Sets lastScannedAt = NOW() server-side.
+ */
+export async function upsertChannelScanState(
+  mappingId: string,
+  update: ScanStateUpdate
+) {
+  return db.accountChannelMap.update({
+    where: { id: mappingId },
+    data: {
+      lastProcessedMessageId: update.lastProcessedMessageId ?? undefined,
+      lastScannedAt: new Date(),
+      lastScanFoundArchives: update.lastScanFoundArchives,
+      consecutiveEmptyScans: update.consecutiveEmptyScans,
+    },
+  });
+}
+
+/**
+ * Atomically upsert a TopicProgress row with the new watermark + scan-state
+ * fields. Same semantics as upsertChannelScanState but for forum topics.
+ */
+export async function upsertTopicScanState(
+  accountChannelMapId: string,
+  topicId: bigint,
+  topicName: string | null,
+  update: ScanStateUpdate
+) {
+  return db.topicProgress.upsert({
+    where: {
+      accountChannelMapId_topicId: { accountChannelMapId, topicId },
+    },
+    create: {
+      accountChannelMapId,
+      topicId,
+      topicName,
+      lastProcessedMessageId: update.lastProcessedMessageId,
+      lastScannedAt: new Date(),
+      lastScanFoundArchives: update.lastScanFoundArchives,
+      consecutiveEmptyScans: update.consecutiveEmptyScans,
+    },
+    update: {
+      topicName,
+      lastProcessedMessageId: update.lastProcessedMessageId ?? undefined,
+      lastScannedAt: new Date(),
+      lastScanFoundArchives: update.lastScanFoundArchives,
+      consecutiveEmptyScans: update.consecutiveEmptyScans,
+    },
+  });
+}
+
 export async function markStaleRunsAsFailed() {
  return db.ingestionRun.updateMany({
    where: { status: "RUNNING" },
@@ -304,6 +556,16 @@ export async function updateAccountAuthState(
  });
 }

+export async function updateAccountPremiumStatus(
+  accountId: string,
+  isPremium: boolean
+): Promise<void> {
+  await db.telegramAccount.update({
+    where: { id: accountId },
+    data: { isPremium },
+  });
+}
+
 export async function getAccountAuthCode(accountId: string) {
  const account = await db.telegramAccount.findUnique({
    where: { id: accountId },
@@ -409,6 +671,45 @@ export async function upsertTopicProgress(
  });
 }

+/**
+ * Ensure a TopicProgress row exists for every discovered topic so the admin
+ * UI can list and toggle them — including brand-new topics. Inserts missing
+ * rows only (fetchEnabled defaults to true); never overwrites watermarks,
+ * scan-state, or the user's fetchEnabled choice on rows that already exist.
+ */
+export async function ensureTopicProgressRows(
+  mappingId: string,
+  topics: { topicId: bigint; name: string | null }[]
+): Promise<void> {
+  if (topics.length === 0) return;
+  await db.topicProgress.createMany({
+    data: topics.map((t) => ({
+      accountChannelMapId: mappingId,
+      topicId: t.topicId,
+      topicName: t.name,
+    })),
+    skipDuplicates: true,
+  });
+}
+
+/**
+ * Read the CURRENT per-topic fetch flag straight from the DB. Returns true
+ * when no row exists yet (default = enabled). Called fresh inside the topic
+ * loop so a mid-run toggle is honoured for topics not yet started.
+ */
+export async function isTopicFetchEnabled(
+  mappingId: string,
+  topicId: bigint
+): Promise<boolean> {
+  const row = await db.topicProgress.findUnique({
+    where: {
+      accountChannelMapId_topicId: { accountChannelMapId: mappingId, topicId },
+    },
+    select: { fetchEnabled: true },
+  });
+  return row?.fetchEnabled ?? true;
+}
+
 // ── Channel fetch requests (DB-mediated communication with web app) ──

 export async function getChannelFetchRequest(requestId: string) {
@@ -512,6 +813,7 @@ export async function upsertSkippedPackage(data: {
      errorMessage: data.errorMessage ?? null,
      fileName: data.fileName,
      fileSize: data.fileSize,
+      attemptCount: { increment: 1 },
      createdAt: new Date(),
    },
    create: {
@@ -529,6 +831,26 @@ export async function upsertSkippedPackage(data: {
  });
 }

+/**
+ * Return source-message IDs in a channel whose SkippedPackage attemptCount has
+ * reached or exceeded the cap — these are treated as "permanently failed for
+ * now" so the watermark can advance past them. The user can manually retry via
+ * the UI to reset the SkippedPackage record.
+ */
+export async function getCappedSkippedMessageIds(
+  sourceChannelId: string,
+  cap: number
+): Promise<Set<bigint>> {
+  const rows = await db.skippedPackage.findMany({
+    where: {
+      sourceChannelId,
+      attemptCount: { gte: cap },
+    },
+    select: { sourceMessageId: true },
+  });
+  return new Set(rows.map((r) => r.sourceMessageId));
+}
+
 export async function deleteSkippedPackage(
  sourceChannelId: string,
  sourceMessageId: bigint
@@ -538,6 +860,63 @@ export async function deleteSkippedPackage(
  });
 }

+/**
+ * Find SkippedPackages for a given account+channel that are still eligible
+ * for auto-retry (attemptCount below the cap). Used at the start of a scan
+ * to pull the watermark back so we don't strand failed messages forever
+ * after the watermark has advanced past them.
+ *
+ * For non-forum channels, pass `topicId: null` to get rows with NULL topic.
+ * For forum channels, pass the topic ID to scope to that topic only.
+ */
+export async function getRetryableSkippedMessageIds(args: {
+  accountId: string;
+  sourceChannelId: string;
+  topicId: bigint | null;
+  cap: number;
+}): Promise<bigint[]> {
+  const rows = await db.skippedPackage.findMany({
+    where: {
+      accountId: args.accountId,
+      sourceChannelId: args.sourceChannelId,
+      sourceTopicId: args.topicId,
+      attemptCount: { lt: args.cap },
+    },
+    select: { sourceMessageId: true },
+    orderBy: { sourceMessageId: "asc" },
+  });
+  return rows.map((r) => r.sourceMessageId);
+}
+
+/**
+ * Update a Package's source topic when a more specific topic context is
+ * discovered for the same content. Used when findRepostedPackage matches
+ * an existing Package whose topic is less specific (e.g., "General") than
+ * the topic we just encountered the file in.
+ *
+ * Also updates the creator if the new topic name is more informative than
+ * the existing creator (i.e., the existing creator was derived from a
+ * less-specific topic name like "General").
+ */
+export async function updatePackageTopicContext(
+  packageId: string,
+  newTopicId: bigint,
+  newTopicName: string | null
+): Promise<void> {
+  await db.package.update({
+    where: { id: packageId },
+    data: {
+      sourceTopicId: newTopicId,
+      // Only overwrite creator if the new topic name is meaningful (non-empty,
+      // non-General). Keeps explicit creator values from filename or admin
+      // input intact.
+      ...(newTopicName && newTopicName !== "General"
+        ? { creator: newTopicName }
+        : {}),
+    },
+  });
+}
+
 export async function createOrFindPackageGroup(input: {
  mediaAlbumId: string;
  sourceChannelId: string;
@@ -587,3 +966,185 @@ export async function linkPackagesToGroup(
    data: { packageGroupId: groupId },
  });
 }
+
+export async function createTimeWindowGroup(input: {
+  sourceChannelId: string;
+  name: string;
+  packageIds: string[];
+}): Promise<string> {
+  const group = await db.packageGroup.create({
+    data: {
+      sourceChannelId: input.sourceChannelId,
+      name: input.name,
+      groupingSource: "AUTO_TIME",
+    },
+  });
+
+  await db.package.updateMany({
+    where: { id: { in: input.packageIds } },
+    data: { packageGroupId: group.id },
+  });
+
+  return group.id;
+}
+
+export async function createAutoGroup(input: {
+  sourceChannelId: string;
+  name: string;
+  packageIds: string[];
+  groupingSource: "ALBUM" | "MANUAL" | "AUTO_TIME" | "AUTO_PATTERN" | "AUTO_ZIP" | "AUTO_CAPTION" | "AUTO_REPLY";
+}): Promise<string> {
+  const group = await db.packageGroup.create({
+    data: {
+      sourceChannelId: input.sourceChannelId,
+      name: input.name,
+      groupingSource: input.groupingSource,
+    },
+  });
+
+  await db.package.updateMany({
+    where: { id: { in: input.packageIds } },
+    data: { packageGroupId: group.id },
+  });
+
+  return group.id;
+}
+
+// ── Provenance backfill ──
+
+export interface PlaceholderCandidate {
+  id: string;
+  archiveType: string;
+  fileName: string;
+  fileCount: number;
+  fileSize: bigint;
+  destMessageId: bigint | null;
+  destMessageIds: bigint[];
+  destChannel: { telegramId: bigint } | null;
+}
+
+/**
+ * Find every placeholder Package matching name+size (oldest first). Package
+ * has no direct `destChannel` relation (only the scalar `destChannelId`), so
+ * each row's destination TelegramChannel telegramId is resolved with a
+ * follow-up lookup rather than a Prisma include.
+ */
+export async function findPlaceholderCandidates(
+  destChannelId: string,
+  fileName: string,
+  fileSize: bigint,
+): Promise<PlaceholderCandidate[]> {
+  const rows = await db.package.findMany({
+    where: {
+      fileName,
+      fileSize,
+      destMessageId: { not: null },
+      // Placeholder provenance (spec §1): manual-upload (source == destination)
+      // OR rebuild record (sourceMessageId == 0 "unknown" sentinel).
+      OR: [
+        { sourceChannelId: destChannelId },
+        { sourceMessageId: 0n },
+      ],
+    },
+    select: {
+      id: true, archiveType: true, fileName: true, fileCount: true, fileSize: true,
+      destMessageId: true, destMessageIds: true, destChannelId: true,
+    },
+    orderBy: { indexedAt: "asc" },
+  });
+  if (rows.length === 0) return [];
+
+  const destChannelIds = [...new Set(rows.map((r) => r.destChannelId).filter((id): id is string => !!id))];
+  const channels = destChannelIds.length
+    ? await db.telegramChannel.findMany({
+        where: { id: { in: destChannelIds } },
+        select: { id: true, telegramId: true },
+      })
+    : [];
+  const telegramIdById = new Map(channels.map((c) => [c.id, c.telegramId]));
+
+  return rows.map((row) => ({
+    id: row.id,
+    archiveType: row.archiveType,
+    fileName: row.fileName,
+    fileCount: row.fileCount,
+    fileSize: row.fileSize,
+    destMessageId: row.destMessageId,
+    destMessageIds: row.destMessageIds,
+    destChannel: row.destChannelId && telegramIdById.has(row.destChannelId)
+      ? { telegramId: telegramIdById.get(row.destChannelId)! }
+      : null,
+  }));
+}
+
+export async function findPlaceholderCandidate(
+  destChannelId: string,
+  fileName: string,
+  fileSize: bigint,
+): Promise<PlaceholderCandidate | null> {
+  return (await findPlaceholderCandidates(destChannelId, fileName, fileSize))[0] ?? null;
+}
+
+export async function getPackageFileCrcs(packageId: string): Promise<(string | null)[]> {
+  const rows = await db.packageFile.findMany({
+    where: { packageId },
+    select: { crc32: true },
+  });
+  return rows.map((r) => r.crc32);
+}
+
+export interface BackfillProvenanceInput {
+  packageId: string;
+  destChannelId: string; // to re-check placeholder status in-txn
+  sourceChannelId: string;
+  sourceMessageId: bigint;
+  sourceTopicId: bigint | null;
+  sourceCaption: string | null;
+  remoteUniqueId: string | null;
+  creator: string | null; // always set (re-derived by caller)
+  entries?: FileEntry[]; // set only if candidate had fileCount === 0
+  previewData?: Buffer | null; // set only if provided and candidate lacks one
+  previewMsgId?: bigint | null;
+}
+
+export async function backfillProvenance(input: BackfillProvenanceInput): Promise<boolean> {
+  return db.$transaction(async (tx) => {
+    const current = await tx.package.findUnique({
+      where: { id: input.packageId },
+      select: { sourceChannelId: true, sourceMessageId: true, previewData: true, fileCount: true },
+    });
+    // Re-check placeholder status inside the txn (another worker may have won).
+    // Placeholder = manual-upload (source==dest) OR rebuild (sourceMessageId==0).
+    const stillPlaceholder =
+      !!current &&
+      (current.sourceChannelId === input.destChannelId || current.sourceMessageId === 0n);
+    if (!stillPlaceholder) return false;
+
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const data: any = {
+      sourceChannelId: input.sourceChannelId,
+      sourceMessageId: input.sourceMessageId,
+      sourceTopicId: input.sourceTopicId,
+      sourceCaption: input.sourceCaption,
+      remoteUniqueId: input.remoteUniqueId,
+      creator: input.creator,
+    };
+    if (input.entries && current.fileCount === 0) {
+      await tx.packageFile.deleteMany({ where: { packageId: input.packageId } });
+      await tx.packageFile.createMany({
+        data: input.entries.map((e) => ({
+          packageId: input.packageId,
+          path: e.path, fileName: e.fileName, extension: e.extension,
+          compressedSize: e.compressedSize, uncompressedSize: e.uncompressedSize, crc32: e.crc32,
+        })),
+      });
+      data.fileCount = input.entries.length;
+    }
+    if (input.previewData && !current.previewData) {
+      data.previewData = input.previewData;
+      data.previewMsgId = input.previewMsgId ?? null;
+    }
+    await tx.package.update({ where: { id: input.packageId }, data });
+    return true;
+  });
+}
@@ -101,16 +101,14 @@ export async function processExtractRequest(requestId: string): Promise<void> {
  try {
    await mkdir(tempDir, { recursive: true });

-    // Wrap the entire TDLib session in the mutex so no other TDLib
-    // operation can run concurrently (TDLib is single-session).
-    await withTdlibMutex("extract", async () => {
    const accounts = await getActiveAccounts();
    if (accounts.length === 0) {
      throw new Error("No authenticated Telegram accounts available");
    }
-
    const account = accounts[0];
-      const client = await createTdlibClient({ id: account.id, phone: account.phone });
+
+    await withTdlibMutex(account.phone, "extract", async () => {
+      const { client } = await createTdlibClient({ id: account.id, phone: account.phone });

      try {
        // Load chat list so TDLib can find the dest channel
@@ -5,6 +5,8 @@ import { withTdlibMutex } from "./util/mutex.js";
 import { processFetchRequest } from "./worker.js";
 import { processExtractRequest } from "./extract-listener.js";
 import { rebuildPackageDatabase } from "./rebuild.js";
+import { processManualUpload } from "./manual-upload.js";
+import { processBackfillRequest, processSlicerTagBackfill } from "./backfill.js";
 import { generateInviteLink, createSupergroup, searchPublicChat } from "./tdlib/chats.js";
 import { createTdlibClient, closeTdlibClient } from "./tdlib/client.js";
 import { triggerImmediateCycle } from "./scheduler.js";
@@ -13,6 +15,7 @@ import {
  getGlobalSetting,
  setGlobalSetting,
  getActiveAccounts,
+  getChannelFetchRequest,
  upsertChannel,
  ensureAccountChannelLink,
  updateFetchRequestStatus,
@@ -55,6 +58,9 @@ async function connectListener(): Promise<void> {
    await pgClient.query("LISTEN join_channel");
    await pgClient.query("LISTEN archive_extract");
    await pgClient.query("LISTEN rebuild_packages");
+    await pgClient.query("LISTEN manual_upload");
+    await pgClient.query("LISTEN backfill_filelists");
+    await pgClient.query("LISTEN backfill_slicer_tags");

    pgClient.on("notification", (msg) => {
      if (msg.channel === "channel_fetch" && msg.payload) {
@@ -71,6 +77,12 @@ async function connectListener(): Promise<void> {
        handleArchiveExtract(msg.payload);
      } else if (msg.channel === "rebuild_packages" && msg.payload) {
        handleRebuildPackages(msg.payload);
+      } else if (msg.channel === "manual_upload" && msg.payload) {
+        handleManualUpload(msg.payload);
+      } else if (msg.channel === "backfill_filelists") {
+        handleBackfillFilelists(msg.payload ?? "{}");
+      } else if (msg.channel === "backfill_slicer_tags") {
+        handleBackfillSlicerTags(msg.payload ?? "{}");
      }
    });

@@ -96,7 +108,7 @@ async function connectListener(): Promise<void> {
      }
    });

-    log.info("Fetch listener started (channel_fetch, generate_invite, create_destination, ingestion_trigger, join_channel, archive_extract, rebuild_packages)");
+    log.info("Fetch listener started (channel_fetch, generate_invite, create_destination, ingestion_trigger, join_channel, archive_extract, rebuild_packages, manual_upload, backfill_filelists, backfill_slicer_tags)");
  } catch (err) {
    log.error({ err }, "Failed to start fetch listener — retrying");
    scheduleReconnect();
@@ -129,7 +141,9 @@ let fetchQueue: Promise<void> = Promise.resolve();
 function handleChannelFetch(requestId: string): void {
  fetchQueue = fetchQueue.then(async () => {
    try {
-      await withTdlibMutex("fetch-channels", () =>
+      const request = await getChannelFetchRequest(requestId);
+      const key = request?.account?.phone ?? "global";
+      await withTdlibMutex(key, "fetch-channels", () =>
        processFetchRequest(requestId)
      );
    } catch (err) {
@@ -143,22 +157,20 @@ function handleChannelFetch(requestId: string): void {
 function handleGenerateInvite(channelId: string): void {
  fetchQueue = fetchQueue.then(async () => {
    try {
-      await withTdlibMutex("generate-invite", async () => {
+      const accounts = await getActiveAccounts();
+      if (accounts.length === 0) {
+        log.warn("No authenticated accounts to generate invite link");
+        return;
+      }
+      const account = accounts[0];
+      await withTdlibMutex(account.phone, "generate-invite", async () => {
        const destChannel = await getGlobalDestinationChannel();
        if (!destChannel || destChannel.id !== channelId) {
          log.warn({ channelId }, "Destination channel mismatch, skipping invite generation");
          return;
        }

-        // Use the first available authenticated account to generate the link
-        const accounts = await getActiveAccounts();
-        if (accounts.length === 0) {
-          log.warn("No authenticated accounts to generate invite link");
-          return;
-        }
-
-        const account = accounts[0];
-        const client = await createTdlibClient({ id: account.id, phone: account.phone });
+        const { client } = await createTdlibClient({ id: account.id, phone: account.phone });

        try {
          const link = await generateInviteLink(client, destChannel.telegramId);
@@ -183,7 +195,13 @@ function handleCreateDestination(payload: string): void {
      const parsed = JSON.parse(payload) as { requestId: string; title: string };
      requestId = parsed.requestId;

-      await withTdlibMutex("create-destination", async () => {
+      const accounts = await getActiveAccounts();
+      if (accounts.length === 0) {
+        throw new Error("No authenticated accounts available to create the group");
+      }
+      const account = accounts[0];
+
+      await withTdlibMutex(account.phone, "create-destination", async () => {
        const { db } = await import("./db/client.js");

        // Mark the request as in-progress
@@ -192,14 +210,7 @@ function handleCreateDestination(payload: string): void {
          data: { status: "IN_PROGRESS" },
        });

-        // Use the first available authenticated account
-        const accounts = await getActiveAccounts();
-        if (accounts.length === 0) {
-          throw new Error("No authenticated accounts available to create the group");
-        }
-
-        const account = accounts[0];
-        const client = await createTdlibClient({ id: account.id, phone: account.phone });
+        const { client } = await createTdlibClient({ id: account.id, phone: account.phone });

        try {
          // Create the supergroup via TDLib
@@ -324,16 +335,16 @@ function handleJoinChannel(payload: string): void {
      const parsed = JSON.parse(payload) as { requestId: string; input: string; accountId: string };
      requestId = parsed.requestId;

-      await withTdlibMutex("join-channel", async () => {
-        await updateFetchRequestStatus(requestId!, "IN_PROGRESS");
-
      const accounts = await getActiveAccounts();
      const account = accounts.find((a) => a.id === parsed.accountId) ?? accounts[0];
      if (!account) {
        throw new Error("No authenticated accounts available");
      }

-        const client = await createTdlibClient({ id: account.id, phone: account.phone });
+      await withTdlibMutex(account.phone, "join-channel", async () => {
+        await updateFetchRequestStatus(requestId!, "IN_PROGRESS");
+
+        const { client } = await createTdlibClient({ id: account.id, phone: account.phone });

        try {
          const linkInfo = parseTelegramInput(parsed.input);
@@ -503,7 +514,12 @@ function handleIngestionTrigger(): void {
 function handleRebuildPackages(requestId: string): void {
  fetchQueue = fetchQueue.then(async () => {
    try {
-      await withTdlibMutex("rebuild-packages", () =>
+      const accounts = await getActiveAccounts();
+      if (accounts.length === 0) {
+        log.warn("No authenticated accounts to rebuild packages");
+        return;
+      }
+      await withTdlibMutex(accounts[0].phone, "rebuild-packages", () =>
        rebuildPackageDatabase(requestId)
      );
    } catch (err) {
@@ -511,3 +527,38 @@ function handleRebuildPackages(requestId: string): void {
    }
  });
 }
+
+// ── Manual upload handler ──
+
+function handleManualUpload(uploadId: string): void {
+  fetchQueue = fetchQueue
+    .then(() => processManualUpload(uploadId))
+    .catch((err) => log.error({ err, uploadId }, "Manual upload processing failed"));
+}
+
+// ── Backfill file-list handler ──
+//
+// Trigger via:
+//   SELECT pg_notify('backfill_filelists', '{"limit":50,"archiveType":"RAR"}');
+//
+// Both fields are optional. archiveType filters to one of ZIP/RAR/SEVEN_Z.
+// Default limit is 100. The handler queues so multiple notifications run
+// sequentially (no concurrent TDLib downloads competing for the mutex).
+function handleBackfillFilelists(payload: string): void {
+  fetchQueue = fetchQueue
+    .then(() => processBackfillRequest(payload))
+    .catch((err) => log.error({ err, payload }, "Backfill request failed"));
+}
+
+// ── Slicer tag backfill handler ──
+//
+// Trigger:
+//   SELECT pg_notify('backfill_slicer_tags', '{"limit":1000}');
+//
+// Pure-DB pass over Packages that have file lists but no slicer tags.
+// No downloads, no TDLib involvement — fast and safe.
+function handleBackfillSlicerTags(payload: string): void {
+  fetchQueue = fetchQueue
+    .then(() => processSlicerTagBackfill(payload))
+    .catch((err) => log.error({ err, payload }, "Slicer tag backfill failed"));
+}
@@ -1,7 +1,8 @@
 import type { Client } from "tdl";
 import type { TelegramPhoto } from "./preview/match.js";
 import { downloadPhotoThumbnail } from "./tdlib/download.js";
-import { createOrFindPackageGroup, linkPackagesToGroup } from "./db/queries.js";
+import { createOrFindPackageGroup, linkPackagesToGroup, createTimeWindowGroup, createAutoGroup } from "./db/queries.js";
+import { config } from "./util/config.js";
 import { childLogger } from "./util/logger.js";
 import { db } from "./db/client.js";

@@ -77,3 +78,591 @@ export async function processAlbumGroups(
    }
  }
 }
+
+/**
+ * Apply learned GroupingRules from manual overrides.
+ * For each rule, find ungrouped packages whose fileName contains the pattern.
+ */
+export async function processRuleBasedGroups(
+  sourceChannelId: string,
+  indexedPackages: IndexedPackageRef[]
+): Promise<void> {
+  const rules = await db.groupingRule.findMany({
+    where: { sourceChannelId },
+    orderBy: { confidence: "desc" },
+  });
+
+  if (rules.length === 0) return;
+
+  const ungrouped = await db.package.findMany({
+    where: {
+      id: { in: indexedPackages.map((p) => p.packageId) },
+      packageGroupId: null,
+    },
+    select: { id: true, fileName: true, creator: true },
+  });
+
+  if (ungrouped.length < 2) return;
+
+  for (const rule of rules) {
+    const matches = ungrouped.filter((pkg) => {
+      const lower = rule.pattern.toLowerCase();
+      return pkg.fileName.toLowerCase().includes(lower) ||
+        (pkg.creator && pkg.creator.toLowerCase().includes(lower));
+    });
+
+    if (matches.length < 2) continue;
+
+    // Check if any are already grouped (by a previous rule in this loop)
+    const stillUngrouped = await db.package.findMany({
+      where: {
+        id: { in: matches.map((m) => m.id) },
+        packageGroupId: null,
+      },
+      select: { id: true },
+    });
+
+    if (stillUngrouped.length < 2) continue;
+
+    try {
+      const groupId = await createAutoGroup({
+        sourceChannelId,
+        name: rule.pattern,
+        packageIds: stillUngrouped.map((m) => m.id),
+        groupingSource: "MANUAL",
+      });
+
+      log.info(
+        { groupId, ruleId: rule.id, pattern: rule.pattern, memberCount: stillUngrouped.length },
+        "Applied learned grouping rule"
+      );
+    } catch (err) {
+      log.warn({ err, ruleId: rule.id }, "Failed to apply grouping rule");
+    }
+  }
+}
+
+/**
+ * After album grouping, cluster remaining ungrouped packages from the same channel
+ * that were posted within a configurable time window.
+ * Only groups packages that were just indexed in this scan cycle (the `indexedPackages` list).
+ */
+export async function processTimeWindowGroups(
+  sourceChannelId: string,
+  indexedPackages: IndexedPackageRef[]
+): Promise<void> {
+  if (config.autoGroupTimeWindowMinutes <= 0) return;
+
+  // Find which of the just-indexed packages are still ungrouped
+  const ungrouped = await db.package.findMany({
+    where: {
+      id: { in: indexedPackages.map((p) => p.packageId) },
+      packageGroupId: null,
+    },
+    orderBy: { sourceMessageId: "asc" },
+    select: {
+      id: true,
+      fileName: true,
+      sourceMessageId: true,
+      indexedAt: true,
+    },
+  });
+
+  if (ungrouped.length < 2) return;
+
+  const windowMs = config.autoGroupTimeWindowMinutes * 60 * 1000;
+
+  // Cluster by time proximity: walk through sorted list, start new cluster when gap > window
+  const clusters: typeof ungrouped[] = [];
+  let current: typeof ungrouped = [ungrouped[0]];
+
+  for (let i = 1; i < ungrouped.length; i++) {
+    const prev = current[current.length - 1];
+    const gap = Math.abs(ungrouped[i].indexedAt.getTime() - prev.indexedAt.getTime());
+
+    if (gap <= windowMs) {
+      current.push(ungrouped[i]);
+    } else {
+      clusters.push(current);
+      current = [ungrouped[i]];
+    }
+  }
+  clusters.push(current);
+
+  // Create groups for clusters with 2+ packages
+  for (const cluster of clusters) {
+    if (cluster.length < 2) continue;
+
+    // Derive group name from common filename prefix
+    const name = findCommonPrefix(cluster.map((p) => p.fileName)) || cluster[0].fileName;
+
+    try {
+      const groupId = await createTimeWindowGroup({
+        sourceChannelId,
+        name,
+        packageIds: cluster.map((p) => p.id),
+      });
+
+      log.info(
+        { groupId, name, memberCount: cluster.length },
+        "Created time-window group"
+      );
+    } catch (err) {
+      log.warn({ err, clusterSize: cluster.length }, "Failed to create time-window group");
+    }
+  }
+}
+
+/**
+ * Group ungrouped packages that share a date pattern (YYYY-MM, YYYY_MM, etc.)
+ * or project slug extracted from their filenames.
+ */
+export async function processPatternGroups(
+  sourceChannelId: string,
+  indexedPackages: IndexedPackageRef[]
+): Promise<void> {
+  const ungrouped = await db.package.findMany({
+    where: {
+      id: { in: indexedPackages.map((p) => p.packageId) },
+      packageGroupId: null,
+    },
+    select: { id: true, fileName: true },
+  });
+
+  if (ungrouped.length < 2) return;
+
+  // Group by extracted pattern
+  const patternMap = new Map<string, typeof ungrouped>();
+  for (const pkg of ungrouped) {
+    const pattern = extractPattern(pkg.fileName);
+    if (!pattern) continue;
+    const group = patternMap.get(pattern) ?? [];
+    group.push(pkg);
+    patternMap.set(pattern, group);
+  }
+
+  for (const [pattern, members] of patternMap) {
+    if (members.length < 2) continue;
+
+    try {
+      const groupId = await createAutoGroup({
+        sourceChannelId,
+        name: pattern,
+        packageIds: members.map((m) => m.id),
+        groupingSource: "AUTO_PATTERN",
+      });
+
+      log.info(
+        { groupId, pattern, memberCount: members.length },
+        "Created pattern-based group"
+      );
+    } catch (err) {
+      log.warn({ err, pattern }, "Failed to create pattern group");
+    }
+  }
+}
+
+/**
+ * Extract a grouping pattern from a filename.
+ * Matches: YYYY-MM, YYYY_MM, "Month Year", or a project prefix before common separators.
+ * Returns null if no usable pattern found.
+ */
+function extractPattern(fileName: string): string | null {
+  // Strip extension for matching
+  const name = fileName.replace(/\.(zip|rar|7z|pdf|stl)(\.\d+)?$/i, "");
+
+  // Match YYYY-MM or YYYY_MM patterns
+  const dateMatch = name.match(/(\d{4})[\-_](\d{2})/);
+  if (dateMatch) {
+    return `${dateMatch[1]}-${dateMatch[2]}`;
+  }
+
+  // Match "Month Year" patterns (e.g., "January 2025", "Jan 2025")
+  const months = "(?:jan(?:uary)?|feb(?:ruary)?|mar(?:ch)?|apr(?:il)?|may|jun(?:e)?|jul(?:y)?|aug(?:ust)?|sep(?:tember)?|oct(?:ober)?|nov(?:ember)?|dec(?:ember)?)";
+  const monthYearMatch = name.match(new RegExp(`(${months})\\s*(\\d{4})`, "i"));
+  if (monthYearMatch) {
+    const monthStr = monthYearMatch[1].toLowerCase().slice(0, 3);
+    const monthNum = ["jan","feb","mar","apr","may","jun","jul","aug","sep","oct","nov","dec"].indexOf(monthStr) + 1;
+    if (monthNum > 0) {
+      return `${monthYearMatch[2]}-${String(monthNum).padStart(2, "0")}`;
+    }
+  }
+
+  // Match project prefix: text before " - ", " – ", or "(". Must be at least 5 chars.
+  const prefixMatch = name.match(/^(.{5,}?)(?:\s*[\-–]\s|\s*\()/);
+  if (prefixMatch) {
+    return prefixMatch[1].trim();
+  }
+
+  return null;
+}
+
+/**
+ * Group ungrouped packages that share the same creator within a channel.
+ * Only groups if there are 3+ packages from the same creator (to avoid
+ * over-grouping when a creator only has a couple files).
+ */
+export async function processCreatorGroups(
+  sourceChannelId: string,
+  indexedPackages: IndexedPackageRef[]
+): Promise<void> {
+  const ungrouped = await db.package.findMany({
+    where: {
+      id: { in: indexedPackages.map((p) => p.packageId) },
+      packageGroupId: null,
+      creator: { not: null },
+    },
+    select: { id: true, fileName: true, creator: true },
+  });
+
+  if (ungrouped.length < 3) return;
+
+  // Group by creator
+  const creatorMap = new Map<string, typeof ungrouped>();
+  for (const pkg of ungrouped) {
+    if (!pkg.creator) continue;
+    const key = pkg.creator.toLowerCase();
+    const group = creatorMap.get(key) ?? [];
+    group.push(pkg);
+    creatorMap.set(key, group);
+  }
+
+  for (const [, members] of creatorMap) {
+    if (members.length < 3) continue;
+
+    const creatorName = members[0].creator!;
+    const name = findCommonPrefix(members.map((m) => m.fileName)) || creatorName;
+
+    try {
+      const groupId = await createAutoGroup({
+        sourceChannelId,
+        name,
+        packageIds: members.map((m) => m.id),
+        groupingSource: "AUTO_PATTERN",
+      });
+
+      log.info(
+        { groupId, creator: creatorName, memberCount: members.length },
+        "Created creator-based group"
+      );
+    } catch (err) {
+      log.warn({ err, creator: creatorName }, "Failed to create creator group");
+    }
+  }
+}
+
+/**
+ * Group ungrouped packages that share the same root folder inside their archives.
+ * E.g., if two packages both contain files under "ProjectX/", they're likely related.
+ * Only considers packages with 3+ files (to avoid false positives from flat archives).
+ */
+export async function processZipPathGroups(
+  sourceChannelId: string,
+  indexedPackages: IndexedPackageRef[]
+): Promise<void> {
+  // Find ungrouped packages that have indexed files
+  const ungrouped = await db.package.findMany({
+    where: {
+      id: { in: indexedPackages.map((p) => p.packageId) },
+      packageGroupId: null,
+      fileCount: { gte: 3 },
+    },
+    select: {
+      id: true,
+      fileName: true,
+      files: {
+        select: { path: true },
+        take: 50,
+      },
+    },
+  });
+
+  if (ungrouped.length < 2) return;
+
+  // Extract the dominant root folder for each package
+  const packageRoots = new Map<string, { id: string; fileName: string }[]>();
+
+  for (const pkg of ungrouped) {
+    const root = extractRootFolder(pkg.files.map((f) => f.path));
+    if (!root) continue;
+
+    const key = root.toLowerCase();
+    const group = packageRoots.get(key) ?? [];
+    group.push({ id: pkg.id, fileName: pkg.fileName });
+    packageRoots.set(key, group);
+  }
+
+  // Create groups for roots shared by 2+ packages
+  for (const [root, members] of packageRoots) {
+    if (members.length < 2) continue;
+
+    try {
+      const groupId = await createAutoGroup({
+        sourceChannelId,
+        name: root,
+        packageIds: members.map((m) => m.id),
+        groupingSource: "AUTO_ZIP",
+      });
+
+      log.info(
+        { groupId, rootFolder: root, memberCount: members.length },
+        "Created ZIP path prefix group"
+      );
+    } catch (err) {
+      log.warn({ err, rootFolder: root }, "Failed to create ZIP path group");
+    }
+  }
+}
+
+/**
+ * Group ungrouped packages that reply to the same root message.
+ * If message B and C both reply to message A, they're grouped together.
+ */
+export async function processReplyChainGroups(
+  sourceChannelId: string,
+  indexedPackages: IndexedPackageRef[]
+): Promise<void> {
+  const ungrouped = await db.package.findMany({
+    where: {
+      id: { in: indexedPackages.map((p) => p.packageId) },
+      packageGroupId: null,
+      replyToMessageId: { not: null },
+    },
+    select: {
+      id: true,
+      fileName: true,
+      replyToMessageId: true,
+    },
+  });
+
+  if (ungrouped.length < 2) return;
+
+  // Group by replyToMessageId
+  const replyMap = new Map<string, typeof ungrouped>();
+  for (const pkg of ungrouped) {
+    if (!pkg.replyToMessageId) continue;
+    const key = pkg.replyToMessageId.toString();
+    const group = replyMap.get(key) ?? [];
+    group.push(pkg);
+    replyMap.set(key, group);
+  }
+
+  for (const [replyId, members] of replyMap) {
+    if (members.length < 2) continue;
+
+    const name = findCommonPrefix(members.map((m) => m.fileName)) || members[0].fileName;
+
+    try {
+      const groupId = await createAutoGroup({
+        sourceChannelId,
+        name,
+        packageIds: members.map((m) => m.id),
+        groupingSource: "AUTO_REPLY" as const,
+      });
+
+      log.info(
+        { groupId, replyToMessageId: replyId, memberCount: members.length },
+        "Created reply-chain group"
+      );
+    } catch (err) {
+      log.warn({ err, replyToMessageId: replyId }, "Failed to create reply-chain group");
+    }
+  }
+}
+
+/**
+ * Group ungrouped packages with similar captions from the same channel.
+ * Uses normalized caption comparison — two captions match if they share
+ * the same significant words (ignoring common words and file extensions).
+ */
+export async function processCaptionGroups(
+  sourceChannelId: string,
+  indexedPackages: IndexedPackageRef[]
+): Promise<void> {
+  const ungrouped = await db.package.findMany({
+    where: {
+      id: { in: indexedPackages.map((p) => p.packageId) },
+      packageGroupId: null,
+      sourceCaption: { not: null },
+    },
+    select: {
+      id: true,
+      fileName: true,
+      sourceCaption: true,
+    },
+  });
+
+  if (ungrouped.length < 2) return;
+
+  // Group by normalized caption key
+  const captionMap = new Map<string, typeof ungrouped>();
+  for (const pkg of ungrouped) {
+    if (!pkg.sourceCaption) continue;
+    const key = normalizeCaptionKey(pkg.sourceCaption);
+    if (!key) continue;
+    const group = captionMap.get(key) ?? [];
+    group.push(pkg);
+    captionMap.set(key, group);
+  }
+
+  for (const [, members] of captionMap) {
+    if (members.length < 2) continue;
+
+    const name = members[0].sourceCaption!.slice(0, 80);
+
+    try {
+      const groupId = await createAutoGroup({
+        sourceChannelId,
+        name,
+        packageIds: members.map((m) => m.id),
+        groupingSource: "AUTO_CAPTION" as const,
+      });
+
+      log.info(
+        { groupId, memberCount: members.length },
+        "Created caption-match group"
+      );
+    } catch (err) {
+      log.warn({ err }, "Failed to create caption group");
+    }
+  }
+}
+
+/**
+ * Normalize a caption for grouping: lowercase, strip extensions and numbers,
+ * extract significant words (3+ chars), sort, and join.
+ * Two captions with the same key are considered a match.
+ */
+function normalizeCaptionKey(caption: string): string | null {
+  const stripped = caption
+    .toLowerCase()
+    .replace(/\.(zip|rar|7z|stl|pdf|obj|gcode)(\.\d+)?/gi, "")
+    .replace(/[^a-z0-9\s]/g, " ");
+
+  const words = stripped
+    .split(/\s+/)
+    .filter((w) => w.length >= 3)
+    .filter((w) => !["the", "and", "for", "with", "from", "part", "file", "files"].includes(w));
+
+  if (words.length < 2) return null;
+
+  return words.sort().join(" ");
+}
+
+/**
+ * Extract the dominant root folder from a list of archive file paths.
+ * Returns the first path segment that appears in >50% of files.
+ * Returns null for flat archives or archives with no common root.
+ */
+function extractRootFolder(paths: string[]): string | null {
+  if (paths.length === 0) return null;
+
+  // Count first path segments
+  const segmentCounts = new Map<string, number>();
+  for (const p of paths) {
+    // Normalize separators and get first segment
+    const normalized = p.replace(/\\/g, "/");
+    const firstSlash = normalized.indexOf("/");
+    if (firstSlash <= 0) continue; // Skip root-level files
+    const segment = normalized.slice(0, firstSlash);
+    // Skip common noise folders
+    if (segment === "__MACOSX" || segment === ".DS_Store" || segment === "Thumbs.db") continue;
+    segmentCounts.set(segment, (segmentCounts.get(segment) ?? 0) + 1);
+  }
+
+  if (segmentCounts.size === 0) return null;
+
+  // Find the most common segment
+  let maxSegment = "";
+  let maxCount = 0;
+  for (const [seg, count] of segmentCounts) {
+    if (count > maxCount) {
+      maxSegment = seg;
+      maxCount = count;
+    }
+  }
+
+  // Must appear in >50% of files and be at least 3 chars
+  if (maxCount < paths.length * 0.5 || maxSegment.length < 3) return null;
+
+  return maxSegment;
+}
+
+/**
+ * Detect packages that could have been grouped differently.
+ * Checks if any grouped package's filename matches a GroupingRule
+ * that would place it in a different group.
+ */
+export async function detectGroupingConflicts(
+  sourceChannelId: string,
+  indexedPackages: IndexedPackageRef[]
+): Promise<void> {
+  const rules = await db.groupingRule.findMany({
+    where: { sourceChannelId },
+  });
+  if (rules.length === 0) return;
+
+  const grouped = await db.package.findMany({
+    where: {
+      id: { in: indexedPackages.map((p) => p.packageId) },
+      packageGroupId: { not: null },
+    },
+    select: {
+      id: true,
+      fileName: true,
+      packageGroupId: true,
+      packageGroup: { select: { name: true, groupingSource: true } },
+    },
+  });
+
+  for (const pkg of grouped) {
+    for (const rule of rules) {
+      if (pkg.fileName.toLowerCase().includes(rule.pattern.toLowerCase())) {
+        // Check if the rule's source group is different from current group
+        if (rule.createdByGroupId && rule.createdByGroupId !== pkg.packageGroupId) {
+          try {
+            await db.systemNotification.create({
+              data: {
+                type: "GROUPING_CONFLICT",
+                severity: "INFO",
+                title: `Potential grouping conflict: ${pkg.fileName}`,
+                message: `Grouped by ${pkg.packageGroup?.groupingSource ?? "unknown"} into "${pkg.packageGroup?.name}", but also matches rule "${rule.pattern}" from a different manual group`,
+                context: {
+                  packageId: pkg.id,
+                  fileName: pkg.fileName,
+                  currentGroupId: pkg.packageGroupId,
+                  matchedRuleId: rule.id,
+                  matchedPattern: rule.pattern,
+                },
+              },
+            });
+          } catch {
+            // Best-effort
+          }
+          break; // One notification per package
+        }
+      }
+    }
+  }
+}
+
+/**
+ * Find the longest common prefix among a list of filenames,
+ * trimming trailing separators and partial words.
+ */
+function findCommonPrefix(names: string[]): string {
+  if (names.length === 0) return "";
+  if (names.length === 1) return names[0];
+
+  let prefix = names[0];
+  for (let i = 1; i < names.length; i++) {
+    while (!names[i].startsWith(prefix)) {
+      prefix = prefix.slice(0, -1);
+      if (prefix.length === 0) return "";
+    }
+  }
+
+  // Trim trailing separators and partial words
+  const trimmed = prefix.replace(/[\s\-_.(]+$/, "");
+  return trimmed.length >= 3 ? trimmed : "";
+}
@@ -27,6 +27,33 @@ async function main(): Promise<void> {
  await cleanupTempDir();
  await markStaleRunsAsFailed();

+  // Release any advisory locks orphaned by a previous worker instance.
+  // When Docker kills a container, PostgreSQL may keep the session alive
+  // (zombie connections), holding advisory locks that block the new worker.
+  try {
+    const result = await pool.query(`
+      SELECT pid, state, left(query, 80) as query, age(clock_timestamp(), state_change) as idle_time
+      FROM pg_stat_activity
+      WHERE datname = current_database()
+        AND pid != pg_backend_pid()
+        AND state = 'idle'
+        AND query LIKE '%pg_try_advisory_lock%'
+        AND state_change < clock_timestamp() - interval '5 minutes'
+    `);
+    for (const row of result.rows) {
+      log.warn(
+        { pid: row.pid, idleTime: row.idle_time, query: row.query },
+        "Terminating stale advisory lock session from previous worker"
+      );
+      await pool.query("SELECT pg_terminate_backend($1)", [row.pid]);
+    }
+    if (result.rows.length > 0) {
+      log.info({ terminated: result.rows.length }, "Cleaned up stale advisory lock sessions");
+    }
+  } catch (err) {
+    log.warn({ err }, "Failed to clean up stale advisory locks (non-fatal)");
+  }
+
  // Verify destination messages exist for all "uploaded" packages.
  // Resets any packages whose dest message is missing so they get re-processed.
  await recoverIncompleteUploads();
@@ -0,0 +1,211 @@
+import path from "path";
+import { rm } from "fs/promises";
+import { db } from "./db/client.js";
+import { childLogger } from "./util/logger.js";
+import { config } from "./util/config.js";
+import { hashParts } from "./archive/hash.js";
+import { byteLevelSplit } from "./archive/split.js";
+import { uploadToChannel } from "./upload/channel.js";
+import { createTdlibClient, closeTdlibClient } from "./tdlib/client.js";
+import { readZipCentralDirectory } from "./archive/zip-reader.js";
+import { readRarContents } from "./archive/rar-reader.js";
+import { read7zContents } from "./archive/sevenz-reader.js";
+import { getActiveAccounts } from "./db/queries.js";
+
+const log = childLogger("manual-upload");
+
+export async function processManualUpload(uploadId: string): Promise<void> {
+  log.info({ uploadId }, "Processing manual upload");
+
+  const upload = await db.manualUpload.findUnique({
+    where: { id: uploadId },
+    include: { files: true },
+  });
+
+  if (!upload || upload.status !== "PENDING") {
+    log.warn({ uploadId }, "Manual upload not found or not pending");
+    return;
+  }
+
+  await db.manualUpload.update({
+    where: { id: uploadId },
+    data: { status: "PROCESSING" },
+  });
+
+  try {
+    // Get destination channel
+    const destSetting = await db.globalSetting.findUnique({
+      where: { key: "destination_channel_id" },
+    });
+    if (!destSetting) throw new Error("No destination channel configured");
+
+    const destChannel = await db.telegramChannel.findFirst({
+      where: { id: destSetting.value, type: "DESTINATION", isActive: true },
+    });
+    if (!destChannel) throw new Error("Destination channel not found or inactive");
+
+    // Get a TDLib client (use first active account)
+    const accounts = await getActiveAccounts();
+    const account = accounts[0];
+    if (!account) throw new Error("No authenticated Telegram account available");
+
+    const { client } = await createTdlibClient({ id: account.id, phone: account.phone });
+
+    try {
+      const packageIds: string[] = [];
+
+      for (const file of upload.files) {
+        try {
+          const filePath = file.filePath;
+          const fileName = file.fileName;
+          const fileSize = file.fileSize;
+
+          log.info({ fileName, fileSize: Number(fileSize) }, "Processing file");
+
+          // Determine archive type
+          let archiveType: "ZIP" | "RAR" | "SEVEN_Z" | "DOCUMENT" = "DOCUMENT";
+          const ext = fileName.toLowerCase();
+          if (ext.endsWith(".zip")) archiveType = "ZIP";
+          else if (ext.endsWith(".rar")) archiveType = "RAR";
+          else if (ext.endsWith(".7z")) archiveType = "SEVEN_Z";
+
+          // Hash the file
+          const contentHash = await hashParts([filePath]);
+
+          // Check for duplicates
+          const existing = await db.package.findFirst({
+            where: { contentHash, destMessageId: { not: null } },
+            select: { id: true },
+          });
+
+          if (existing) {
+            log.info({ fileName, contentHash }, "Duplicate file, skipping upload");
+            await db.manualUploadFile.update({
+              where: { id: file.id },
+              data: { packageId: existing.id },
+            });
+            packageIds.push(existing.id);
+            continue;
+          }
+
+          // Read archive metadata
+          let entries: {
+            path: string;
+            fileName: string;
+            extension: string | null;
+            compressedSize: bigint;
+            uncompressedSize: bigint;
+            crc32: string | null;
+          }[] = [];
+          try {
+            if (archiveType === "ZIP") entries = await readZipCentralDirectory([filePath]);
+            else if (archiveType === "RAR") entries = await readRarContents(filePath);
+            else if (archiveType === "SEVEN_Z") entries = await read7zContents(filePath);
+          } catch {
+            log.debug({ fileName }, "Could not read archive metadata");
+          }
+
+          // Split if needed
+          const MAX_UPLOAD_SIZE = BigInt(config.maxPartSizeMB) * 1024n * 1024n;
+          let uploadPaths = [filePath];
+          if (fileSize > MAX_UPLOAD_SIZE) {
+            uploadPaths = await byteLevelSplit(filePath);
+          }
+
+          // Upload to Telegram
+          const destResult = await uploadToChannel(
+            client,
+            destChannel.telegramId,
+            uploadPaths
+          );
+
+          // Create package record
+          const pkg = await db.package.create({
+            data: {
+              contentHash,
+              fileName,
+              fileSize,
+              archiveType,
+              sourceChannelId: destChannel.id,
+              sourceMessageId: destResult.messageId,
+              destChannelId: destChannel.id,
+              destMessageId: destResult.messageId,
+              destMessageIds: destResult.messageIds,
+              isMultipart: uploadPaths.length > 1,
+              partCount: uploadPaths.length,
+              fileCount: entries.length,
+              files: entries.length > 0 ? { create: entries } : undefined,
+            },
+          });
+
+          await db.manualUploadFile.update({
+            where: { id: file.id },
+            data: { packageId: pkg.id },
+          });
+
+          packageIds.push(pkg.id);
+          log.info({ fileName, packageId: pkg.id }, "File processed and uploaded");
+
+          // Clean up split files (but not the original)
+          if (uploadPaths.length > 1) {
+            for (const splitPath of uploadPaths) {
+              if (splitPath !== filePath) {
+                await rm(splitPath, { force: true }).catch(() => {});
+              }
+            }
+          }
+        } catch (fileErr) {
+          log.error({ err: fileErr, fileName: file.fileName }, "Failed to process file");
+        }
+      }
+
+      // Group packages if multiple files
+      if (packageIds.length >= 2) {
+        const groupName =
+          upload.groupName ?? upload.files[0].fileName.replace(/\.[^.]+$/, "");
+        const group = await db.packageGroup.create({
+          data: {
+            name: groupName,
+            sourceChannelId: destChannel.id,
+            groupingSource: "MANUAL",
+          },
+        });
+        await db.package.updateMany({
+          where: { id: { in: packageIds } },
+          data: { packageGroupId: group.id },
+        });
+        log.info(
+          { groupId: group.id, groupName, packageCount: packageIds.length },
+          "Created group for uploaded files"
+        );
+      }
+
+      await db.manualUpload.update({
+        where: { id: uploadId },
+        data: { status: "COMPLETED", completedAt: new Date() },
+      });
+
+      log.info(
+        { uploadId, fileCount: upload.files.length, packageCount: packageIds.length },
+        "Manual upload completed"
+      );
+    } finally {
+      await closeTdlibClient(client);
+    }
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    log.error({ err, uploadId }, "Manual upload failed");
+    await db.manualUpload.update({
+      where: { id: uploadId },
+      data: { status: "FAILED", errorMessage: message },
+    });
+  }
+
+  // Clean up uploaded files
+  try {
+    const uploadDir = path.join("/data/uploads", uploadId);
+    await rm(uploadDir, { recursive: true, force: true });
+  } catch {
+    // Best-effort cleanup
+  }
+}
@@ -0,0 +1,306 @@
+import { db } from "./db/client.js";
+import { childLogger } from "./util/logger.js";
+import { downloadFileRange } from "./tdlib/range-download.js";
+import { invokeWithTimeout } from "./tdlib/download.js";
+import { parseZipCentralDirectoryFromTail, MIN_ZIP_TAIL_BYTES } from "./archive/central-directory.js";
+import { fingerprintsMatch, crcFingerprint } from "./archive/fingerprint.js";
+import {
+  findPlaceholderCandidates,
+  getPackageFileCrcs,
+  backfillProvenance,
+  type PlaceholderCandidate,
+} from "./db/queries.js";
+import type { FileEntry } from "./archive/zip-reader.js";
+import { readSevenZListingRanged, type RangedPart } from "./archive/ranged/sevenz-ranged.js";
+import { readRarListingRanged } from "./archive/ranged/rar-ranged.js";
+import { tdlibRangeReader } from "./archive/ranged/range-reader.js";
+import { fullDownloadListing } from "./archive/ranged/fallback.js";
+import type { Client } from "tdl";
+
+const log = childLogger("provenance-backfill");
+
+export interface BackfillArgs {
+  client: Client;
+  destChannelId: string;
+  scannedSourceChannelId: string;
+  fileName: string;
+  fileSize: bigint;
+  archiveType: string;
+  sourceMessageId: bigint;
+  sourceTopicId: bigint | null;
+  sourceCaption: string | null;
+  remoteUniqueId: string | null;
+  creator: string | null;
+  scannedParts: RangedPart[];
+  previewData?: Buffer | null;
+  previewMsgId?: bigint | null;
+}
+
+/**
+ * Read a ZIP central directory from the tail of a (possibly multipart)
+ * archive. `parts` is ordered; only the LAST part carries the EOCD record.
+ * `fileSize` on each part is that part's own size (NOT the whole-archive
+ * total) so the download offset stays within that part's bounds, while
+ * `tailStart` passed to the parser is the logical whole-archive offset
+ * (preceding parts' sizes + the offset within the last part).
+ */
+async function readScannedZipListing(
+  client: Client,
+  parts: { fileId: string; fileSize: bigint }[],
+): Promise<FileEntry[] | null> {
+  if (parts.length === 0) return null;
+  const lastPart = parts[parts.length - 1];
+  const precedingSize = parts.slice(0, -1).reduce((sum, p) => sum + Number(p.fileSize), 0);
+  const lastSize = Number(lastPart.fileSize);
+  for (const tailBytes of [MIN_ZIP_TAIL_BYTES, MIN_ZIP_TAIL_BYTES * 4]) {
+    const partOffset = Math.max(0, lastSize - tailBytes);
+    const downloadLen = Math.min(tailBytes, lastSize);
+    try {
+      const buf = await downloadFileRange(client, lastPart.fileId, partOffset, downloadLen, lastPart.fileSize);
+      const tailStart = precedingSize + partOffset;
+      return parseZipCentralDirectoryFromTail(buf, tailStart);
+    } catch (err) {
+      if (err instanceof RangeError) continue; // try a larger tail
+      log.warn({ err, fileId: lastPart.fileId }, "ranged ZIP listing failed");
+      return null;
+    }
+  }
+  return null;
+}
+
+/**
+ * Resolve the destination copy's message(s) into ranged parts (file id +
+ * size + name), in order, so a multipart destination copy is reconstructed
+ * with correct per-part sizes and names (the last message carries the
+ * EOCD-bearing tail part for ZIP; multipart RAR needs correctly-suffixed
+ * `.partN.rar` names for `unrar` sibling discovery). Cheap-only: any TDLib
+ * failure here degrades the caller to name-size confidence rather than
+ * falling back to a full download.
+ */
+async function resolveDestParts(
+  client: Client,
+  destChatTelegramId: bigint,
+  destMessageIds: bigint[],
+  destMessageId: bigint | null,
+  fallbackFileName: string,
+): Promise<RangedPart[] | null> {
+  const messageIds = destMessageIds.length > 0 ? destMessageIds : destMessageId ? [destMessageId] : [];
+  if (messageIds.length === 0) return null;
+  try {
+    const parts: RangedPart[] = [];
+    for (const msgId of messageIds) {
+      const msg = (await invokeWithTimeout(client, {
+        _: "getMessage",
+        chat_id: Number(destChatTelegramId),
+        message_id: Number(msgId),
+      })) as { content?: { document?: { document?: { id: number; size?: number }; file_name?: string } } };
+      const doc = msg?.content?.document?.document;
+      if (!doc?.id) return null;
+      const fileName = msg?.content?.document?.file_name || fallbackFileName;
+      parts.push({ fileId: String(doc.id), fileSize: BigInt(doc.size ?? 0), fileName });
+    }
+    return parts;
+  } catch (err) {
+    log.warn({ err, destMessageIds: messageIds.map(Number) }, "destination archive part resolution failed");
+    return null;
+  }
+}
+
+async function readScannedListingRanged(
+  archiveType: string,
+  client: Client,
+  parts: RangedPart[],
+): Promise<FileEntry[] | null> {
+  const read = tdlibRangeReader(client);
+  if (archiveType === "ZIP") return readScannedZipListing(client, parts);
+  if (archiveType === "SEVEN_Z") return readSevenZListingRanged(parts, read);
+  if (archiveType === "RAR") return readRarListingRanged(parts, read);
+  return null;
+}
+
+/**
+ * Build the CRC fingerprint entries for a placeholder candidate: start from
+ * its stored PackageFile CRCs, and if those are incomplete (e.g. a rebuild
+ * candidate with fileCount === 0), fall back to a fresh ranged read of the
+ * candidate's own copy in the destination channel (Task 9).
+ */
+async function resolveCandidateFingerprintEntries(
+  client: Client,
+  candidate: PlaceholderCandidate,
+): Promise<FileEntry[]> {
+  const candidateCrcs = await getPackageFileCrcs(candidate.id);
+  let candidateEntries: FileEntry[] = candidateCrcs.map((crc) => ({
+    path: "", fileName: "", extension: null, compressedSize: 0n, uncompressedSize: 0n, crc32: crc,
+  }));
+  const hasDestMessage = candidate.destMessageIds.length > 0 || candidate.destMessageId != null;
+  if (!crcFingerprint(candidateEntries).complete && hasDestMessage && candidate.destChannel) {
+    const destParts = await resolveDestParts(
+      client,
+      candidate.destChannel.telegramId,
+      candidate.destMessageIds,
+      candidate.destMessageId,
+      candidate.fileName,
+    );
+    let destEntries: FileEntry[] | null = null;
+    if (destParts) {
+      const read = tdlibRangeReader(client);
+      destEntries =
+        candidate.archiveType === "ZIP" ? await readScannedZipListing(client, destParts)
+        : candidate.archiveType === "SEVEN_Z" ? await readSevenZListingRanged(destParts, read)
+        : candidate.archiveType === "RAR" ? await readRarListingRanged(destParts, read)
+        : null;
+    }
+    if (destEntries) {
+      candidateEntries = destEntries;
+    }
+  }
+  return candidateEntries;
+}
+
+/**
+ * Classify a fingerprint comparison between two entry sets. "incomplete"
+ * means at least one side is missing CRCs (e.g. an empty file → CRC32 of
+ * zero-length data → null) and the comparison CANNOT be used to confirm or
+ * refute a match — callers must fall back to name+size confidence rather
+ * than treating this as a mismatch.
+ */
+function compareFingerprints(a: FileEntry[], b: FileEntry[]): "match" | "mismatch" | "incomplete" {
+  const fa = crcFingerprint(a);
+  const fb = crcFingerprint(b);
+  if (!fa.complete || !fb.complete) return "incomplete";
+  return fingerprintsMatch(a, b) ? "match" : "mismatch";
+}
+
+export async function tryProvenanceBackfill(
+  args: BackfillArgs,
+): Promise<{ backfilled: boolean; confidence?: "fingerprint" | "name-size" }> {
+  const candidates = await findPlaceholderCandidates(args.destChannelId, args.fileName, args.fileSize);
+  if (candidates.length === 0) return { backfilled: false };
+
+  let scannedEntries: FileEntry[] | null = await readScannedListingRanged(
+    args.archiveType, args.client, args.scannedParts,
+  );
+  // Cheap ranged read failed — fall back to a size-capped full download so the
+  // listing still gets indexed. Only worth it when the candidate lacks a listing.
+  if (!scannedEntries && candidates.some((c) => c.fileCount === 0)) {
+    const totalSize = args.scannedParts.reduce((s, p) => s + p.fileSize, 0n);
+    scannedEntries = await fullDownloadListing({
+      client: args.client, parts: args.scannedParts, archiveType: args.archiveType,
+      totalSize, fileName: args.fileName,
+    });
+  }
+
+  let chosen = candidates[0];
+  let confidence: "fingerprint" | "name-size" = "name-size";
+
+  if (candidates.length > 1) {
+    // Multiple placeholder packages share this name+size. Try to
+    // disambiguate by fingerprint (ZIP only); if we can't uniquely resolve
+    // it, notify instead of guessing which one is the real match.
+    if (args.archiveType === "ZIP" && scannedEntries) {
+      const matches: PlaceholderCandidate[] = [];
+      // Candidates NOT ruled out as a definite (both-complete) mismatch —
+      // used as the name+size fallback pool when the fingerprint can't
+      // confirm a match (e.g. incomplete CRCs on either side).
+      const nonMismatches: PlaceholderCandidate[] = [];
+      for (const c of candidates) {
+        const candidateEntries = await resolveCandidateFingerprintEntries(args.client, c);
+        const comparison = compareFingerprints(scannedEntries, candidateEntries);
+        if (comparison === "match") {
+          matches.push(c);
+          nonMismatches.push(c);
+        } else if (comparison === "incomplete") {
+          nonMismatches.push(c);
+        }
+        // comparison === "mismatch": both sides complete and differ — excluded.
+      }
+      if (matches.length === 1) {
+        chosen = matches[0];
+        confidence = "fingerprint";
+      } else if (matches.length === 0 && nonMismatches.length === 1) {
+        // Fingerprint couldn't confirm (incomplete CRCs), but exactly one
+        // candidate wasn't ruled out as a definite mismatch — fall back to
+        // name+size confidence rather than treating this as unresolved.
+        chosen = nonMismatches[0];
+        confidence = "name-size";
+      } else {
+        await db.systemNotification.create({
+          data: {
+            type: "INTEGRITY_AUDIT",
+            severity: "WARNING",
+            title: `Ambiguous provenance match: ${args.fileName}`,
+            message: `${candidates.length} placeholder packages share this name+size and the fingerprint did not uniquely disambiguate. No provenance was backfilled.`,
+            context: { fileName: args.fileName, candidateIds: candidates.map((c) => c.id) },
+          },
+        });
+        return { backfilled: false };
+      }
+    } else {
+      // Can't disambiguate without a fingerprint — notify, don't guess.
+      await db.systemNotification.create({
+        data: {
+          type: "INTEGRITY_AUDIT",
+          severity: "WARNING",
+          title: `Ambiguous provenance match: ${args.fileName}`,
+          message: `${candidates.length} placeholder packages share this name+size (archive type ${args.archiveType} — no cheap fingerprint). No provenance was backfilled.`,
+          context: { fileName: args.fileName, candidateIds: candidates.map((c) => c.id) },
+        },
+      });
+      return { backfilled: false };
+    }
+  } else if (scannedEntries) {
+    const candidateEntries = await resolveCandidateFingerprintEntries(args.client, chosen);
+    const comparison = compareFingerprints(scannedEntries, candidateEntries);
+    if (comparison === "match") {
+      confidence = "fingerprint";
+    } else if (comparison === "mismatch") {
+      // Both sides' CRCs are complete and differ: NOT the same content
+      // despite name+size. Do not backfill.
+      log.info({ candidateId: chosen.id, fileName: args.fileName }, "fingerprint mismatch — not backfilling");
+      return { backfilled: false };
+    }
+    // comparison === "incomplete": can't confirm or refute by fingerprint —
+    // fall through and backfill on name+size confidence instead.
+  }
+
+  const ok = await backfillProvenance({
+    packageId: chosen.id,
+    destChannelId: args.destChannelId,
+    sourceChannelId: args.scannedSourceChannelId,
+    sourceMessageId: args.sourceMessageId,
+    sourceTopicId: args.sourceTopicId,
+    sourceCaption: args.sourceCaption,
+    remoteUniqueId: args.remoteUniqueId,
+    creator: args.creator,
+    entries: chosen.fileCount === 0 && scannedEntries ? scannedEntries : undefined,
+    previewData: args.previewData ?? undefined,
+    previewMsgId: args.previewMsgId ?? undefined,
+  });
+
+  if (!ok) return { backfilled: false };
+
+  if (confidence === "name-size") {
+    // Lower-confidence backfill: no CRC fingerprint guard confirmed this
+    // match. Record it as an auditable event so name+size-only backfills
+    // can be reviewed after the fact.
+    await db.systemNotification.create({
+      data: {
+        type: "INTEGRITY_AUDIT",
+        severity: "INFO",
+        title: `Provenance backfilled by name+size: ${args.fileName}`,
+        message: `Package ${chosen.id} was matched to a scanned source message by file name and size only (no CRC fingerprint confirmation).`,
+        context: {
+          packageId: chosen.id,
+          fileName: args.fileName,
+          sourceChannelId: args.scannedSourceChannelId,
+        },
+      },
+    });
+  }
+
+  log.info(
+    { candidateId: chosen.id, fileName: args.fileName, confidence, source: args.scannedSourceChannelId },
+    "provenance backfilled",
+  );
+  return { backfilled: true, confidence };
+}
@@ -63,7 +63,7 @@ export async function rebuildPackageDatabase(
    }

    const account = accounts[0];
-    const client = await createTdlibClient({
+    const { client } = await createTdlibClient({
      id: account.id,
      phone: account.phone,
    });
@@ -308,14 +308,15 @@ async function scanDestinationChannel(
    }>(client, {
      _: "searchChatMessages",
      chat_id: Number(chatId),
+      // No topic context for a flat destination scan. TDLib 1.8.64+ replaced
+      // `message_thread_id` / `saved_messages_topic_id` with a single
+      // optional `topic_id`; for a flat scan we just omit it.
      query: "",
      from_message_id: currentFromId,
      offset: 0,
      limit: 100,
      filter: { _: "searchMessagesFilterDocument" },
      sender_id: null,
-      message_thread_id: 0,
-      saved_messages_topic_id: 0,
    });

    if (!result.messages || result.messages.length === 0) break;
@@ -63,7 +63,7 @@ export async function recoverIncompleteUploads(): Promise<void> {
  let client: Client | undefined;

  try {
-    client = await createTdlibClient({ id: account.id, phone: account.phone });
+    ({ client } = await createTdlibClient({ id: account.id, phone: account.phone }));

    // Load the chat list so TDLib can resolve chat IDs
    try {
@@ -78,18 +78,31 @@ export async function recoverIncompleteUploads(): Promise<void> {

    let resetCount = 0;
    let verifiedCount = 0;
+    let unknownCount = 0;
+    let wrongContentCount = 0;
+
+    // Batch size for getMessages. TDLib accepts up to ~100 IDs per call.
+    // Using 100 means 20k packages → ~200 round-trips instead of 20k.
+    const BATCH_SIZE = 100;

    for (const [, channelPackages] of byChannel) {
-      for (const pkg of channelPackages) {
-        const exists = await verifyMessageExists(
+      // Group packages by destChannelId (already done) — within each group,
+      // process in batches via getMessages (plural).
+      for (let i = 0; i < channelPackages.length; i += BATCH_SIZE) {
+        const batch = channelPackages.slice(i, i + BATCH_SIZE);
+        const batchResults = await verifyMessagesBatch(
          client,
          destChannel.telegramId,
-          pkg.destMessageId!
+          batch.map((p) => p.destMessageId!)
        );

-        if (exists) {
+        for (let j = 0; j < batch.length; j++) {
+          const pkg = batch[j];
+          const result = batchResults[j];
+
+          if (result.state === "exists") {
            verifiedCount++;
-        } else {
+          } else if (result.state === "deleted") {
          log.warn(
            {
              packageId: pkg.id,
@@ -100,21 +113,50 @@ export async function recoverIncompleteUploads(): Promise<void> {
          );
          await resetPackageDestination(pkg.id);
          resetCount++;
+        } else if (result.state === "wrong-content") {
+          // The message exists but isn't a document anymore (got cleared /
+          // replaced). Treat as missing so we re-upload.
+          log.warn(
+            {
+              packageId: pkg.id,
+              fileName: pkg.fileName,
+              destMessageId: Number(pkg.destMessageId),
+              contentType: result.contentType,
+            },
+            "Destination message is not a document, resetting package for re-upload"
+          );
+          await resetPackageDestination(pkg.id);
+          wrongContentCount++;
+        } else {
+          // Unknown — TDLib couldn't tell us. Don't reset, but DO count this
+          // so the summary line shows recovery wasn't 100% successful.
+          unknownCount++;
+          log.warn(
+            {
+              packageId: pkg.id,
+              fileName: pkg.fileName,
+              destMessageId: Number(pkg.destMessageId),
+              reason: result.reason.slice(0, 200),
+            },
+            "Could not verify destination message — will retry on next startup"
+          );
+        }
        }
      }
    }

-    if (resetCount > 0) {
    log.info(
-        { resetCount, verifiedCount, totalChecked: packages.length },
-        "Upload recovery complete — packages reset for re-processing"
+      {
+        verifiedCount,
+        resetCount,
+        wrongContentCount,
+        unknownCount,
+        totalChecked: packages.length,
+      },
+      unknownCount === 0
+        ? "Upload recovery complete"
+        : "Upload recovery complete — some packages could not be verified, will retry next startup"
    );
-    } else {
-      log.info(
-        { verifiedCount, totalChecked: packages.length },
-        "Upload recovery complete — all destination messages verified"
-      );
-    }
  } catch (err) {
    log.error({ err }, "Upload recovery failed (non-fatal, will retry next startup)");
  } finally {
@@ -124,15 +166,77 @@ export async function recoverIncompleteUploads(): Promise<void> {
  }
 }

+type VerifyResult =
+  | { state: "exists" }
+  | { state: "deleted" }
+  | { state: "wrong-content"; contentType: string }
+  | { state: "unknown"; reason: string };
+
 /**
- * Check whether a message exists in a Telegram chat.
- * Returns false if the message was deleted or never existed.
+ * Batch version of verifyMessageExists. Calls TDLib's getMessages (plural)
+ * with up to ~100 message IDs at once. Returns one VerifyResult per input
+ * ID, in input order. Missing messages come back as null in TDLib's response
+ * — translated to {state: "deleted"} here.
+ *
+ * Falls back to per-message verification on any error so that one bad batch
+ * doesn't lose all verification for that chunk.
+ */
+async function verifyMessagesBatch(
+  client: Client,
+  chatTelegramId: bigint,
+  messageIds: bigint[]
+): Promise<VerifyResult[]> {
+  try {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const result = (await withFloodWait(
+      () =>
+        client.invoke({
+          _: "getMessages",
+          chat_id: Number(chatTelegramId),
+          message_ids: messageIds.map((id) => Number(id)),
+        }),
+      "getMessages:verify"
+    )) as { messages?: (null | { content?: { _: string } })[] };
+
+    const messages = result.messages ?? [];
+    return messageIds.map((_id, i) => {
+      const m = messages[i];
+      if (!m || !m.content) return { state: "deleted" };
+      if (m.content._ !== "messageDocument") {
+        return { state: "wrong-content", contentType: String(m.content._) };
+      }
+      return { state: "exists" };
+    });
+  } catch (err) {
+    // If the whole batch errors out, fall back to per-message verification.
+    log.warn(
+      { err, batchSize: messageIds.length, chatTelegramId: chatTelegramId.toString() },
+      "getMessages batch failed, falling back to per-message verification"
+    );
+    const out: VerifyResult[] = [];
+    for (const id of messageIds) {
+      out.push(await verifyMessageExists(client, chatTelegramId, id));
+    }
+    return out;
+  }
+}
+
+/**
+ * Check whether a message exists in a Telegram chat and is the document we
+ * uploaded. Returns a discriminated result instead of a bare boolean so the
+ * caller can distinguish "definitely gone" (reset) from "couldn't reach TG"
+ * (leave alone, try again next startup).
+ *
+ * Previous version conflated all non-404 errors with "exists", which masked
+ * recovery completely when TDLib had a degraded connection — the worker
+ * would log "all destination messages verified" even though it had answered
+ * questions it couldn't actually answer.
 */
 async function verifyMessageExists(
  client: Client,
  chatTelegramId: bigint,
  messageId: bigint
-): Promise<boolean> {
+): Promise<VerifyResult> {
  try {
    const result = await withFloodWait(
      () =>
@@ -144,44 +248,37 @@ async function verifyMessageExists(
      "getMessage:verify"
    );

-    // TDLib returns the message object if it exists.
-    // A deleted message may return with content type "messageChatDeleteMessage"
-    // or the call may throw. Check that we got a real message with content.
    // eslint-disable-next-line @typescript-eslint/no-explicit-any
    const msg = result as any;
    if (!msg || !msg.content) {
-      return false;
+      return { state: "deleted" };
    }

-    // Check that the message has document content (our uploads are documents)
-    // A message that exists but has no document content was likely cleared/replaced
    if (msg.content._ !== "messageDocument") {
-      log.debug(
-        {
-          messageId: Number(messageId),
-          contentType: msg.content._,
-        },
-        "Destination message exists but is not a document"
-      );
-      return false;
+      return { state: "wrong-content", contentType: String(msg.content._) };
    }

-    return true;
+    return { state: "exists" };
  } catch (err) {
-    // TDLib throws "Message not found" (error code 404) for deleted messages
-    const message = err instanceof Error ? err.message : String(err);
+    const errMessage = err instanceof Error ? err.message : String(err);
    const code = (err as { code?: number })?.code;

-    if (code === 404 || message.includes("not found") || message.includes("Not Found")) {
-      return false;
+    // Hard "the message is definitely gone" signals from TDLib:
+    //   - HTTP 404
+    //   - "Message not found" / "MESSAGE_ID_INVALID" error strings
+    const lower = errMessage.toLowerCase();
+    if (
+      code === 404 ||
+      lower.includes("message not found") ||
+      lower.includes("message_id_invalid") ||
+      lower.includes("messageidinvalid") ||
+      lower.includes("not found")
+    ) {
+      return { state: "deleted" };
    }

-    // For other errors (network issues, etc.), assume the message exists
-    // to avoid incorrectly resetting packages due to transient failures
-    log.warn(
-      { err, messageId: Number(messageId) },
-      "Could not verify message (assuming it exists)"
-    );
-    return true;
+    // Everything else (network, connection, TDLib internal) is genuinely
+    // unknown — do NOT claim "verified".
+    return { state: "unknown", reason: errMessage };
  }
 }
--- a/Show More
+++ b/Show More